CAPSUL - Chain algorithm in pipelines and execute in many contexts

AttributesFactory (class in capsul.attributes.attributes_factory)

EditableAttributes (class in capsul.attributes.attributes_schema)

ProcessAttributes (class in capsul.attributes.attributes_schema)

copy() (capsul.attributes.attributes_schema.ProcessAttributes method)

copy_to_single() (capsul.attributes.attributes_schema.ProcessAttributes method)

BuiltinProcessCompletionEngineFactory (class in capsul.attributes.completion_engine_factory)

ProcessCompletionEngineIteration (class in capsul.attributes.completion_engine_iteration)

PathCompletionEngine (class in capsul.attributes.completion_engine)

PathCompletionEngineFactory (class in capsul.attributes.completion_engine)

ProcessCompletionEngine (class in capsul.attributes.completion_engine)

get_completion_engine() (capsul.attributes.completion_engine.ProcessCompletionEngine static method)

ProcessCompletionEngineFactory (class in capsul.attributes.completion_engine)

SwitchCompletionEngine (class in capsul.attributes.completion_engine)

FomPathCompletionEngine (class in capsul.attributes.fom_completion_engine)

FomProcessCompletionEngine (class in capsul.attributes.fom_completion_engine)

FomProcessCompletionEngineIteration (class in capsul.attributes.fom_completion_engine)

named_directories() (capsul.engine.database_json.JSONDBEngine method)

named_directory() (capsul.engine.database_json.JSONDBEngine method)

path_metadata() (capsul.engine.database_json.JSONDBEngine method)

set_json_value() (capsul.engine.database_json.JSONDBEngine method)

set_named_directory() (capsul.engine.database_json.JSONDBEngine method)

set_path_metadata() (capsul.engine.database_json.JSONDBEngine method)

json_value() (capsul.engine.database_populse.PopulseDBEngine method)

named_directories() (capsul.engine.database_populse.PopulseDBEngine method)

named_directory() (capsul.engine.database_populse.PopulseDBEngine method)

path_metadata() (capsul.engine.database_populse.PopulseDBEngine method)

set_json_value() (capsul.engine.database_populse.PopulseDBEngine method)

set_named_directory() (capsul.engine.database_populse.PopulseDBEngine method)

set_path_metadata() (capsul.engine.database_populse.PopulseDBEngine method)

named_directories() (capsul.engine.database.DatabaseEngine method)

named_directory() (capsul.engine.database.DatabaseEngine method)

set_named_directory() (capsul.engine.database.DatabaseEngine method)

set_path_metadata() (capsul.engine.database.DatabaseEngine method)

check_notably_invalid_config() (in module capsul.engine.module.axon)

check_notably_invalid_config() (in module capsul.engine.module.fsl)

check_notably_invalid_config() (in module capsul.engine.module.matlab)

activate_configurations() (in module capsul.engine.module.python)

check_notably_invalid_config() (in module capsul.engine.module.python)

check_notably_invalid_config() (in module capsul.engine.module.spm)

select_configurations() (capsul.engine.settings.Settings method)

collection_name() (capsul.engine.settings.SettingsSession static method)

ensure_module_fields() (capsul.engine.settings.SettingsSession method)

get_all_environments() (capsul.engine.settings.SettingsSession method)

fsl_command_with_environment() (in module capsul.in_context.fsl)

add_iterative_process() (capsul.pipeline.pipeline.Pipeline method)

add_optional_output_switch() (capsul.pipeline.pipeline.Pipeline method)

add_processes_selection() (capsul.pipeline.pipeline.Pipeline method)

autoexport_nodes_parameters() (capsul.pipeline.pipeline.Pipeline method)

call_process_method() (capsul.pipeline.pipeline.Pipeline method)

define_groups_as_steps() (capsul.pipeline.pipeline.Pipeline method)

define_pipeline_steps() (capsul.pipeline.pipeline.Pipeline method)

disabled_pipeline_steps_nodes() (capsul.pipeline.pipeline.Pipeline method)

enable_all_pipeline_steps() (capsul.pipeline.pipeline.Pipeline method)

find_empty_parameters() (capsul.pipeline.pipeline.Pipeline method)

get_pipeline_step_nodes() (capsul.pipeline.pipeline.Pipeline method)

get_processes_selection_groups() (capsul.pipeline.pipeline.Pipeline method)

get_processes_selection_nodes() (capsul.pipeline.pipeline.Pipeline method)

get_processes_selections() (capsul.pipeline.pipeline.Pipeline method)

install_links_debug_handler() (capsul.pipeline.pipeline.Pipeline method)

pipeline_definition() (capsul.pipeline.pipeline.Pipeline method)

remove_pipeline_step() (capsul.pipeline.pipeline.Pipeline method)

uninstall_links_debug_handler() (capsul.pipeline.pipeline.Pipeline method)

update_nodes_and_plugs_activation() (capsul.pipeline.pipeline.Pipeline method)

workflow_ordered_nodes() (capsul.pipeline.pipeline.Pipeline method)

ConstructedPipeline (class in capsul.pipeline.pipeline_construction)

PipelineConstructor (class in capsul.pipeline.pipeline_construction)

add_link() (capsul.pipeline.pipeline_construction.PipelineConstructor method)

add_process() (capsul.pipeline.pipeline_construction.PipelineConstructor method)

add_switch() (capsul.pipeline.pipeline_construction.PipelineConstructor method)

set_callback_on_plug() (capsul.pipeline.pipeline_nodes.Node method)

check_requirements() (capsul.pipeline.pipeline_nodes.Node method)

get_connections_through() (capsul.pipeline.pipeline_nodes.Node method)

get_missing_mandatory_parameters() (capsul.pipeline.pipeline_nodes.Node method)

remove_callback_from_plug() (capsul.pipeline.pipeline_nodes.Node method)

get_connections_through() (capsul.pipeline.pipeline_nodes.PipelineNode method)

has_default_value (capsul.pipeline.pipeline_nodes.Plug attribute)

set_callback_on_plug() (capsul.pipeline.pipeline_nodes.ProcessNode method)

get_plug_value() (capsul.pipeline.pipeline_nodes.ProcessNode method)

set_plug_value() (capsul.pipeline.pipeline_nodes.ProcessNode method)

check_requirements() (capsul.pipeline.pipeline_nodes.ProcessNode method)

get_study_config() (capsul.pipeline.pipeline_nodes.ProcessNode method)

is_parameter_protected() (capsul.pipeline.pipeline_nodes.ProcessNode method)

protect_parameter() (capsul.pipeline.pipeline_nodes.ProcessNode method)

remove_callback_from_plug() (capsul.pipeline.pipeline_nodes.ProcessNode method)

requirements() (capsul.pipeline.pipeline_nodes.ProcessNode method)

set_study_config() (capsul.pipeline.pipeline_nodes.ProcessNode method)

_switch_values (capsul.pipeline.pipeline_nodes.Switch attribute)

get_connections_through() (capsul.pipeline.pipeline_nodes.Switch method)

create_output_directories() (in module capsul.pipeline.pipeline_tools)

disable_runtime_steps_with_existing_outputs() (in module capsul.pipeline.pipeline_tools)

dot_graph_from_pipeline() (in module capsul.pipeline.pipeline_tools)

dot_graph_from_workflow() (in module capsul.pipeline.pipeline_tools)

dump_pipeline_state_as_dict() (in module capsul.pipeline.pipeline_tools)

find_plug_connection_destinations() (in module capsul.pipeline.pipeline_tools)

find_plug_connection_sources() (in module capsul.pipeline.pipeline_tools)

get_output_directories() (in module capsul.pipeline.pipeline_tools)

load_pipeline_parameters() (in module capsul.pipeline.pipeline_tools)

nodes_with_existing_outputs() (in module capsul.pipeline.pipeline_tools)

nodes_with_missing_inputs() (in module capsul.pipeline.pipeline_tools)

pipeline_link_color() (in module capsul.pipeline.pipeline_tools)

pipeline_node_colors() (in module capsul.pipeline.pipeline_tools)

save_pipeline_parameters() (in module capsul.pipeline.pipeline_tools)

set_pipeline_state_from_dict() (in module capsul.pipeline.pipeline_tools)

where_is_plug_value_from() (in module capsul.pipeline.pipeline_tools)

write_fake_pipeline() (in module capsul.pipeline.pipeline_tools)

workflow_from_pipeline() (in module capsul.pipeline.pipeline_workflow)

set_study_config() (capsul.pipeline.process_iteration.ProcessIteration method)

topological_sort() (capsul.pipeline.topological_sort.Graph method)

links_from (capsul.pipeline.topological_sort.GraphNode attribute)

links_to_degree (capsul.pipeline.topological_sort.GraphNode attribute)

links_from_degree (capsul.pipeline.topological_sort.GraphNode attribute)

add_link_to() (capsul.pipeline.topological_sort.GraphNode method)

remove_link_to() (capsul.pipeline.topological_sort.GraphNode method)

add_link_from() (capsul.pipeline.topological_sort.GraphNode method)

remove_link_from() (capsul.pipeline.topological_sort.GraphNode method)

is_job() (capsul.pipeline.custom_nodes.strconv.StrConvNode method)

CrossValidationFoldNode (class in capsul.pipeline.custom_nodes.cv_node)

LeaveOneOutNode (class in capsul.pipeline.custom_nodes.loo_node)

copied_inputs (capsul.process.process.FileCopyProcess attribute)

_update_input_traits() (capsul.process.process.FileCopyProcess method)

_get_process_arguments() (capsul.process.process.FileCopyProcess method)

_copy_input_files() (capsul.process.process.FileCopyProcess method)

_nipype_interface (capsul.process.process.NipypeProcess attribute)

_nipype_interface_name (capsul.process.process.NipypeProcess attribute)

set_output_directory() (capsul.process.process.NipypeProcess method)

get_missing_mandatory_parameters() (capsul.process.process.Process method)

make_commandline_argument() (capsul.process.process.Process method)

run_from_commandline() (capsul.process.process.Process static method)

complement_doc() (capsul.process.process.ProcessMeta static method)

run_process_with_distribution() (in module capsul.process.runprocess)

set_process_param_from_str() (in module capsul.process.runprocess)

add_controls_to_ui() (capsul.qt_apps.main_window.CapsulMainWindow method)

onBrowseClicked() (capsul.qt_apps.main_window.CapsulMainWindow method)

onChangeViewClicked() (capsul.qt_apps.main_window.CapsulMainWindow method)

onCloseTabClicked() (capsul.qt_apps.main_window.CapsulMainWindow method)

onCurrentTabChanged() (capsul.qt_apps.main_window.CapsulMainWindow method)

onHelpClicked() (capsul.qt_apps.main_window.CapsulMainWindow method)

onLoadClicked() (capsul.qt_apps.main_window.CapsulMainWindow method)

onLoadSubPipelineClicked() (capsul.qt_apps.main_window.CapsulMainWindow method)

onParametersClicked() (capsul.qt_apps.main_window.CapsulMainWindow method)

onQualityControlClicked() (capsul.qt_apps.main_window.CapsulMainWindow method)

onRunClicked() (capsul.qt_apps.main_window.CapsulMainWindow method)

onRunStatus() (capsul.qt_apps.main_window.CapsulMainWindow method)

onSearchClicked() (capsul.qt_apps.main_window.CapsulMainWindow method)

onStudyConfigClicked() (capsul.qt_apps.main_window.CapsulMainWindow method)

onTreeSelectionChanged() (capsul.qt_apps.main_window.CapsulMainWindow method)

init_window() (capsul.qt_apps.pipeline_viewer_app.PipelineViewerApp method)

find_pipeline_and_process() (in module capsul.qt_apps.utils.find_pipelines)

find_pipelines_from_description() (in module capsul.qt_apps.utils.find_pipelines)

create_board_tree() (capsul.qt_gui.board_widget.BoardWidget method)

create_output_widget() (capsul.qt_gui.board_widget.BoardWidget method)

create_viewer_tree() (capsul.qt_gui.board_widget.BoardWidget method)

ActivationInspector (class in capsul.qt_gui.widgets.activation_inspector)

show() (capsul.qt_gui.widgets.activation_inspector.ActivationInspector method)

ActivationInspectorApp (class in capsul.qt_gui.widgets.activation_inspector)

AttributedProcessWidget (class in capsul.qt_gui.widgets.attributed_process_widget)

CapsulLinkDebuggerView (class in capsul.qt_gui.widgets.links_debugger)

show() (capsul.qt_gui.widgets.links_debugger.CapsulLinkDebuggerView method)

EmbeddedSubPipelineItem (class in capsul.qt_gui.widgets.pipeline_developer_view)

HandleItem (class in capsul.qt_gui.widgets.pipeline_developer_view)

itemChange() (capsul.qt_gui.widgets.pipeline_developer_view.HandleItem method)

focusInEvent() (capsul.qt_gui.widgets.pipeline_developer_view.Link method)

focusOutEvent() (capsul.qt_gui.widgets.pipeline_developer_view.Link method)

keyPressEvent() (capsul.qt_gui.widgets.pipeline_developer_view.Link method)

mousePressEvent() (capsul.qt_gui.widgets.pipeline_developer_view.Link method)

update() (capsul.qt_gui.widgets.pipeline_developer_view.Link method)

NodeGWidget (class in capsul.qt_gui.widgets.pipeline_developer_view)

paint() (capsul.qt_gui.widgets.pipeline_developer_view.NodeGWidget method)

PipelineDeveloperView (class in capsul.qt_gui.widgets.pipeline_developer_view)

PipelineDeveloperView.IterativeProcessInput (class in capsul.qt_gui.widgets.pipeline_developer_view)

PipelineDeveloperView.ProcessModuleInput (class in capsul.qt_gui.widgets.pipeline_developer_view)

PipelineDeveloperView.ProcessNameEdit (class in capsul.qt_gui.widgets.pipeline_developer_view)

PipelineScene (class in capsul.qt_gui.widgets.pipeline_developer_view)

helpEvent() (capsul.qt_gui.widgets.pipeline_developer_view.PipelineScene method)

mousePressEvent() (capsul.qt_gui.widgets.pipeline_developer_view.Plug method)

boxItem (class in capsul.qt_gui.widgets.pipeline_developer_view)

focusInEvent() (capsul.qt_gui.widgets.pipeline_developer_view.boxItem method)

focusOutEvent() (capsul.qt_gui.widgets.pipeline_developer_view.boxItem method)

keyPressEvent() (capsul.qt_gui.widgets.pipeline_developer_view.boxItem method)

PipelineFileWarningWidget (class in capsul.qt_gui.widgets.pipeline_file_warning_widget)

PipelineUserView (class in capsul.qt_gui.widgets.pipeline_user_view)

update() (capsul.qt_gui.widgets.pipeline_user_view.PipelineUserView method)

generate_index_entry() (capsul.sphinxext.layoutdocgen.LayoutHelperWriter method)

rst2html() (capsul.sphinxext.layoutdocgen.LayoutHelperWriter method)

title_for() (capsul.sphinxext.layoutdocgen.LayoutHelperWriter method)

write_index() (capsul.sphinxext.layoutdocgen.LayoutHelperWriter method)

write_installation() (capsul.sphinxext.layoutdocgen.LayoutHelperWriter method)

write_layout() (capsul.sphinxext.layoutdocgen.LayoutHelperWriter method)

generate_api_doc() (capsul.sphinxext.pipelinedocgen.PipelineHelpWriter method)

get_short_name() (capsul.sphinxext.pipelinedocgen.PipelineHelpWriter method)

write_api_docs() (capsul.sphinxext.pipelinedocgen.PipelineHelpWriter method)

write_index() (capsul.sphinxext.pipelinedocgen.PipelineHelpWriter method)

write_main_index() (capsul.sphinxext.pipelinedocgen.PipelineHelpWriter method)

getsource() (capsul.sphinxext.usecasesdocgen.UseCasesHelperWriter method)

write_index() (capsul.sphinxext.usecasesdocgen.UseCasesHelperWriter method)

default() (capsul.study_config.memory.CapsulResultEncoder method)

get_node_class() (in module capsul.study_config.process_instance)

get_node_instance() (in module capsul.study_config.process_instance)

get_process_instance() (in module capsul.study_config.process_instance)

is_pipeline_node() (in module capsul.study_config.process_instance)

input_directory (capsul.study_config.study_config.StudyConfig attribute)

output_directory (capsul.study_config.study_config.StudyConfig attribute)

generate_logging (capsul.study_config.study_config.StudyConfig attribute)

reset_process_counter() (capsul.study_config.study_config.StudyConfig method)

set_trait_value() (capsul.study_config.study_config.StudyConfig method)

get_trait() (capsul.study_config.study_config.StudyConfig method)

get_trait_value() (capsul.study_config.study_config.StudyConfig method)

set_study_configuration() (capsul.study_config.study_config.StudyConfig method)

add_trait() (capsul.study_config.study_config.StudyConfig method)

get_configuration_dict() (capsul.study_config.study_config.StudyConfig method)

get_iteration_pipeline() (capsul.study_config.study_config.StudyConfig method)

get_process_instance() (capsul.study_config.study_config.StudyConfig method)

initialize_modules() (capsul.study_config.study_config.StudyConfig method)

load_module() (capsul.study_config.study_config.StudyConfig method)

read_configuration() (capsul.study_config.study_config.StudyConfig method)

save_configuration() (capsul.study_config.study_config.StudyConfig method)

initialize_module() (capsul.study_config.study_config.StudyConfigModule method)

name (capsul.study_config.study_config.StudyConfigModule property)

default_study_config() (in module capsul.study_config.study_config)

AttributesConfig (class in capsul.study_config.config_modules.attributes_config)

BrainVISAConfig (class in capsul.study_config.config_modules.brainvisa_config)

FomConfig (class in capsul.study_config.config_modules.fom_config)

FreeSurferConfig (class in capsul.study_config.config_modules.freesurfer_config)

FSLConfig (class in capsul.study_config.config_modules.fsl_config)

MatlabConfig (class in capsul.study_config.config_modules.matlab_config)

NipypeConfig (class in capsul.study_config.config_modules.nipype_config)

SmartCachingConfig (class in capsul.study_config.config_modules.smartcaching_config)

ResourceController (class in capsul.study_config.config_modules.somaworkflow_config)

SomaWorkflowConfig (class in capsul.study_config.config_modules.somaworkflow_config)

SPMConfig (class in capsul.study_config.config_modules.spm_config)

fsl_command_with_environment() (in module capsul.subprocess.fsl)

get_nipype_interfaces_versions() (in module capsul.utils.version_utils)

f-strings support = for self-documenting expressions and debugging

ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON 3.11.14 DOCUMENTATION

email.message.Message: Representing an email message using the compat32 API

What is the most efficient way to concatenate many strings together?

Why does a_tuple[i] += [‘item’] raise an exception when the addition works?

What does the slash(/) in the parameter list of a function mean?

Why am I getting an UnboundLocalError when the variable has a value?

PEP 3118: New memoryview implementation and buffer protocol documentation

PEP 476: Enabling certificate verification by default for stdlib http clients

PEP 471 - os.scandir() function – a better and faster directory iterator

PEP 655: Marking individual TypedDict items as required or not-required

Why must ‘self’ be used explicitly in method definitions and calls?

argparse — Parser for command-line options, arguments and sub-commands

functools — Higher-order functions and operations on callable objects

importlib.resources – Package resource reading, opening and access

multiprocessing.shared_memory — Shared memory for direct access across processes

sysconfig — Provide access to Python’s configuration information

# -*- coding: utf-8 -*-
'''
Attributes factory module

Classes
=======
:class:`AttributesFactory`
--------------------------
'''

from __future__ import print_function
from __future__ import absolute_import

from soma.factory import ClassFactory

from capsul.attributes.attributes_schema import AttributesSchema

class AttributesFactory(ClassFactory):
    ''' AttributesFactory holds an attributes schema
    '''
    class_types = {
        'schema': AttributesSchema,
    }

''' A schema defines attrinbutes used within a completion framework

Classes
=======
:class:`AttributesSchema`
-------------------------
:class:`EditableAttributes`
---------------------------
:class:`ProcessAttributes`

Functions
=========
:func:`set_attribute`
---------------------
'''

import six
from importlib import import_module
from pkgutil import iter_modules
from soma.sorted_dictionary import OrderedDict
from soma.controller import Controller
from soma.functiontools import partial, SomaPartial
import traits.api as traits
from capsul.pipeline.pipeline_nodes import ProcessNode

class AttributesSchema(object):
    '''
    An AttributesSchema has a name, which is used as an identifier to specify completion.
    '''

    # Name of the schema. Must be defined in subclasses.
    schema_name = None

    def __init__(self):
        # Instantiate EditableAttributes classes that are defined in schema
        sets = dict((k, getattr(self.__class__, k)()) for k in 
                    dir(self.__class__)
                    if isinstance(getattr(self.__class__, k), type) and 
                       issubclass(getattr(self.__class__, k), EditableAttributes))
        self.attribute_sets = sets

class EditableAttributes(Controller):
    ''' A set of attributes (group) used to define process parameters
    attributes. Attributes are traits in the EditableAttributes Controller.
    '''
    def __str__(self):
        return '<{0}({1})>'.format(self.__class__.__name__, None)

def set_attribute(object, name, value):
    '''
    
    '''
    setattr(object, name, value)

class ProcessAttributes(Controller):
    '''
    This is the base class for managing attributes for a process.

    It stores attributes associated with a Process, for each of its parameters.
    Attribute values can be accessed "globally" (for the whole process) as
    ProcessAttributes instance traits.
    Or each parameter attributes set may be accessed individually, using
    :meth:`get_parameters_attributes()`.

    To define attributes for a process, the programmer may subclass
    ProcessAttributes and define some EditableAttributes in it. Each
    EditableAttributes is a group of attributes (traits in the EditableAttributes instance or subclass).

    A ProcessAttributes subclass should be registered to a factory to be linked
    to a process name, using the `factory_id` class variable.

    See Capsul :ref:`advanced usage doc <completion>` for details.
    '''
    
    def __init__(self, process, schema_dict):
        super(ProcessAttributes, self).__init__()
        self._process = process
        self._schema_dict = schema_dict
        self.editable_attributes = OrderedDict()
        self.parameter_attributes = {}

    def __getinitargs__(self):
        # needed for self.copy()
        return (self._process, self._schema_dict)

    def set_parameter_attributes(self, parameter, schema, editable_attributes,
                                 fixed_attibute_values, allow_list=True):
        '''
        Set attributes associated with a single process parameter.

        Parameters
        ----------
        parameter: str
            process parameter name
        schema: str
            schema used for it (input, output, shared)
        editable_attributes: str, EditableAttributes instance, or list of them
            EditableAttributes or id containing attributes traits
        fixed_attibute_values: dict (str/str)
            values of non-editable attributes
        allow_list: bool
            if True (the default), it the process parameter is a list, then
            attributes are transformed into lists.
        '''
        if parameter in self.parameter_attributes:
            if schema == 'link':
                return  # this is just a lower priority
            raise KeyError('Attributes already set for parameter %s'
                           % parameter)
        process = self._process
        if isinstance(process, ProcessNode):
            process = process.process
        if parameter not in process._instance_traits():
            print('WARNING: parameter', parameter,
                  'not in process', process.name)
            return
        if isinstance(editable_attributes, six.string_types) \
                or isinstance(editable_attributes, EditableAttributes):
            editable_attributes = [editable_attributes]
        parameter_editable_attributes = []
        for ea in editable_attributes:
            add_editable_attributes = False
            if isinstance(ea, six.string_types):
                key = ea
                ea = self.editable_attributes.get(key)
                if ea is None:
                    ea = getattr(self._schema_dict[schema], key)()
                    self.editable_attributes[key] = ea
                    add_editable_attributes = True
            elif isinstance(ea, EditableAttributes):
                key = ea
                if key not in self.editable_attributes:
            else:
                raise TypeError(
                    'Invalid value for editable attributes: {0}'.format(ea))
            parameter_editable_attributes.append(ea)
            if add_editable_attributes:
                is_list = allow_list \
                        and isinstance(process.trait(parameter).trait_type,
                                       traits.List)
                for name in list(ea.user_traits().keys()):
                    # don't use items() since traits may change during iter.
                    trait = ea.trait(name)
                    if is_list:
                        trait = traits.List(trait)
                        ea.remove_trait(name)
                        ea.add_trait(name, trait)
                    if name in self.user_traits():
                        if not is_list \
                                and isinstance(self.trait(name).trait_type,
                                               traits.List):
                            # a process attribute trait may have been changed
                            # into a list. Here we assume attributes are only
                            # strings (at this point - it can be changed at
                            # higher lever afterwards), so change it back into
                            # a single value trait.
                            self.remove_trait(name)
                            self.add_trait(name, trait)
                        # else don't add it again: this way non-list versions
                        # of attributes get priority. If both exist, lists
                        # should get one single value (otherwise there is an
                        # ambiguity or inconsistency), so the attribute should
                        # not be a list.
                    else:
                        self.add_trait(name, trait)
                    f = SomaPartial(set_attribute, ea)
                    self.on_trait_change(f, name)
        self.parameter_attributes[parameter] = (parameter_editable_attributes,
                                                fixed_attibute_values)

    def get_parameters_attributes(self):
        ''' Get attributes for each process parameter
        '''
        pa = {}
        for parameter, trait in six.iteritems(process.user_traits()):
            if trait.output:
                if hasattr(process, 'id'):
                    process_name = process.id
                else:
                    process_name = process.name
                attributes = {
                    'generated_by_process': process_name,
                    'generated_by_parameter': parameter}
            else:
                attributes = {}
            editable_fixed = self.parameter_attributes.get(parameter, ([], {}))
            editable_attributes, fixed_attibute_values = editable_fixed
            for ea in editable_attributes:
                for attribute in ea.user_traits():
                    value = getattr(ea, attribute)
                    attributes[attribute] = value
            attributes.update(fixed_attibute_values)
            if attributes:
                pa[parameter] = attributes
        return pa

    def copy(self, with_values=True):
        ''' overloads :meth:`soma.Controller.copy`
        '''
        other = self.__class__(self._process, self._schema_dict)
        ea_map = {}
        for parameter, pa in six.iteritems(self.parameter_attributes):
            if parameter not in other.parameter_attributes:
                # otherwise assume this has been done in a subclass constructor
                eas, fa = pa
                oeas = []
                for ea in eas:
                    oea = ea_map.get(ea)
                    if oea is None:
                        oea = ea.copy()
                    oeas.append(oea)
                other.set_parameter_attributes(parameter, '', oeas, fa,
                                               allow_list=False)
        # copy the values
        if with_values:
            for name in self.user_traits():
                #print('copy attribs:', self, name, getattr(self, name))
                #print('   to:', other.trait(name).trait_type)
                #if isinstance(other.trait(name).trait_type, traits.List):
                    #print('    ', other.trait(name).inner_traits[0].trait_type)
                #if isinstance(self.trait(name).trait_type, traits.List):
                    #print('    self list:', self.trait(name).inner_traits[0].trait_type)
                setattr(other, name, getattr(self, name))

        return other

    def copy_to_single(self, with_values=True):
        ''' Similar to :meth:`copy`, excepts that it converts list attributes
        into single values. This is useful within the completion system
        infrastructure, to get from an attributes set containing lists
        (process parameters which are lists), a single value allowing to
        determine a single path.

        This method is merely useful to the end user.
        '''
        other = ProcessAttributes(self._process, self._schema_dict)

        ea_map = {}
                eas, fa = pa
                oeas = []
                        oea = EditableAttributes()
                        for name, trait in six.iteritems(ea.user_traits()):
                            if isinstance(trait.trait_type, traits.List):
                                trait = trait.inner_traits[0]
                            oea.add_trait(name, trait)
                        ea_map[ea] = oea
                value = getattr(self, name)
                if isinstance(value, list):
                    if len(value) != 0:
                        value = value[0]
                    else:
                        value = self.trait(name).inner_traits[0].default
                if value is not None:
                    setattr(other, name, value)

'''
Instantiate a default :class:`~capsul.attributes.completion_engine.ProcessCompletionEngine`

Classes
=======
:class:`BuiltinProcessCompletionEngineFactory`
----------------------------------------------
'''

from capsul.attributes.completion_engine import ProcessCompletionEngine, \
    PathCompletionEngine, ProcessCompletionEngineFactory
from capsul.attributes.completion_engine_iteration \
    import ProcessCompletionEngineIteration
from capsul.attributes.fom_completion_engine \
    import FomProcessCompletionEngine, FomPathCompletionEngine, \
    FomProcessCompletionEngineIteration
from capsul.pipeline.process_iteration import ProcessIteration
from soma.utils.weak_proxy import weak_proxy

class BuiltinProcessCompletionEngineFactory(ProcessCompletionEngineFactory):
    '''
    '''
    factory_id = 'builtin'

    def get_completion_engine(self, process, name=None):
        '''
        Factory for ProcessCompletionEngine: get an ProcessCompletionEngine
        instance for a node or process in the context of a given StudyConfig.

        The study_config should specify which completion system(s) is (are)
        used (FOM, ...)
        If nothing is configured, a ProcessCompletionEngine base instance will
        be returned. It will not be able to perform completion at all, but will
        conform to the API.
        '''
        if hasattr(process, 'completion_engine'):
            ce = process.completion_engine
            # just in case the process has been changed in the CE
            if ce and ce.process != process:
                ce.process = weak_proxy(process, ce._clear_node)
            return ce

        study_config = process.get_study_config()

        # FOM
        if 'FomConfig' in study_config.modules and study_config.use_fom:
            try:
                pfom = FomProcessCompletionEngine(process, name)
                if pfom is not None:
                    pfom.create_attributes_with_fom()
                    return pfom
            except KeyError:
                # process not in FOM
                pass

        # iteration
        in_process = process
            in_process = process.process
        if isinstance(in_process, ProcessIteration):
            if isinstance(
                    ProcessCompletionEngine.get_completion_engine(
                        in_process.process),
                    FomProcessCompletionEngine):
                return FomProcessCompletionEngineIteration(process, name)
            else:
                return ProcessCompletionEngineIteration(process, name)

        # standard ProcessCompletionEngine
        return super(BuiltinProcessCompletionEngineFactory,
                     self).get_completion_engine(process, name)

'''
:class:`~capsul.attributes.completion_engine.ProcessCompletionEngine` dealing with process iterations.
This is an internal machinery.

Classes
=======
:class:`ProcessCompletionEngineIteration`
-----------------------------------------
'''

    ProcessCompletionEngineFactory
from capsul.attributes.attributes_schema import ProcessAttributes
from soma.controller import Controller,ControllerTrait
import six
import sys
from six.moves import range

if sys.version_info[0] >= 3:
    xrange = range

class ProcessCompletionEngineIteration(ProcessCompletionEngine):
    '''
    :class:`~capsul.attributes.completion_engine.ProcessCompletionEngine`
    specialization for iterative process.

    Iterated attributes are given by get_iterated_attributes().
    Completion performs a single iteration step, stored in
    self.capsul_iteration_step
    '''
    def __init__(self, process, name=None):
        super(ProcessCompletionEngineIteration, self).__init__(
            process=process, name=name)
        #self.add_trait('capsul_iteration_step', traits.Int(0))
        self.capsul_iteration_step = 0
        #self.iterated_attributes = self.get_iterated_attributes()

    def get_iterated_attributes(self):
        '''
        '''
        process = self.process
        try:
            pattributes = ProcessCompletionEngine.get_completion_engine(
                process.process).get_attribute_values()
        except AttributeError:
            # ProcessCompletionEngine not implemented for this process:
            # no completion
            return []
        param_attributes = pattributes.get_parameters_attributes()
        attribs = set()
        for parameter in process.iterative_parameters:
            attribs.update(list(param_attributes.get(parameter, {}).keys()))
        # if no iterative parameter has been declared, use all attributes
        if not process.iterative_parameters:
            return set(pattributes.user_traits().keys())
        return [param for param in pattributes.user_traits().keys()
                if param in attribs]

    def get_induced_iterative_parameters(self):
        '''Iterating over some parameters, and triggering completion through
        attributes, imply that some other parameters will also vary with the
        iteration.
        Ex: process A has 2 parameters, "input" and "output", which are linked
        by the completion system. If we iterate on A.input, then A.output will
        also change with the iteration: parameter "output" should thus be
        included in iterative parameters: it is induced by the iteration over
        "input".

        This method gives the induced iterative parameters.
        '''
        # 1st get iterated attributes
        attributes = self.get_iterated_attributes()
        # now look on which parameters they are acting
        pattributes = ProcessCompletionEngine.get_completion_engine(
            process.process).get_attribute_values()
        induced_params = []
        for parameter in process.process.user_traits():
            if parameter not in process.iterative_parameters:
                par_attributes = param_attributes.get(parameter)
                if par_attributes:
                    par_attributes = set(par_attributes.keys())
                    if [attribute for attribute in attributes
                            if attribute in par_attributes]:
                        induced_params.append(parameter)

        return induced_params

    def get_attribute_values(self):
        ''' Get attributes Controller associated to a process

        Returns
        -------
        attributes: Controller
        '''
        if 'capsul_attributes' not in self._instance_traits():
            process = self.process
            if isinstance(process, ProcessNode):
                process = process.process
            try:
                pattributes = ProcessCompletionEngine.get_completion_engine(
                    process.process).get_attribute_values()
            except AttributeError:
                # ProcessCompletionEngine not implemented for this process:
                # no completion
                return

            schemas = self._get_schemas()
            attributes = ProcessAttributes(self.process, schemas)

            self.add_trait('capsul_attributes', ControllerTrait(Controller()))
            self.capsul_attributes = attributes
            iter_attrib = self.get_iterated_attributes()
            for attrib, trait in six.iteritems(pattributes.user_traits()):
                if attrib not in iter_attrib:
                    attributes.add_trait(attrib, trait)
            for attrib in iter_attrib:
                trait = pattributes.trait(attrib)
                if trait is not None:
                    attributes.add_trait(
                        attrib,
                        traits.List(traits.Union(
                            trait, traits.Instance(type(traits.Undefined))),
                            output=trait.output))
                value = getattr(pattributes, attrib, None)
                if value is not None and value is not traits.Undefined:
                    setattr(attributes, attrib, [value])
        return self.capsul_attributes

    def iteration_size(self, process_inputs={}):
        try:
            attributes_set = self.get_attribute_values()
            return

        # attributes lists sizes
        sizes = [len(getattr(attributes_set, attribute))
                 for attribute in self.get_iterated_attributes()]
        if sizes:
            size = max(sizes)
        else:
            size = 0
        sizes = []
        for param in process.iterative_parameters:
            value = process_inputs.get(param)
            if value is not None:
                sizes.append(len(value))
            else:
                value = getattr(process, param)
                if value:
                    sizes.append(len(value))

        if sizes:
            psize = max(sizes)
        else:
            psize = 0
        size = max(size, psize)

        return size

    def complete_parameters(self, process_inputs={},
                            complete_iterations=True):

        if not complete_iterations:
            # then do nothing...
            return

        self.completion_progress = 0.

        try:
            self.set_parameters(process_inputs)
            completion_engine = ProcessCompletionEngine.get_completion_engine(
                process.process, self.name)
            step_attributes = completion_engine.get_attribute_values()
            return

        size = self.iteration_size()

        iterated_attributes = self.get_iterated_attributes()
        for attribute in attributes_set.user_traits():
            if attribute not in iterated_attributes:
                setattr(step_attributes, attribute,
                        getattr(attributes_set, attribute))
        parameters = {}
        for parameter in process.regular_parameters:
            parameters[parameter] = getattr(process, parameter)

        # complete each step to get iterated parameters.
        # This is generally "too much" but it's difficult to perform a partial
        # completion only on iterated parameters

        iterative_parameters = dict(
            [(key, []) for key in process.iterative_parameters])

        # propagate forbid_completion
        for param, trait in six.iteritems(process.user_traits()):
            if trait.forbid_completion:
                if hasattr(process.process, 'propagate_metadata'):
                    process.process.propagate_metadata(
                        '', param, {'forbid_completion': True})
                else:
                    process.process.trait(param).forbid_completion = True

        self.completion_progress_total = size
        for it_step in range(size):
            self.capsul_iteration_step = it_step
            for attribute in iterated_attributes:
                iterated_values = getattr(attributes_set, attribute)
                step = min(len(iterated_values) - 1, it_step)
                value = iterated_values[step]
                setattr(step_attributes, attribute, value)
            for parameter in process.iterative_parameters:
                values = getattr(process, parameter)
                if isinstance(values, list) and len(values) > it_step:
                    parameters[parameter] = values[it_step]
            completion_engine.complete_parameters(
                parameters, complete_iterations=complete_iterations)
                value = getattr(process.process, parameter)
                iterative_parameters[parameter].append(value)
            self.completion_progress = it_step + 1
        for parameter, values in iterative_parameters.items():
            try:
                setattr(process, parameter, values)
            except Exception as e:
                print('assign iteration parameter', parameter, ':\n', e,
                      file=sys.stderr)
        for parameter in parameters:
            if parameter in iterative_parameters:
                continue
            try:
                setattr(process, parameter, value)
                print('assign parameter', parameter, ':\n', e,

    def complete_iteration_step(self, step):
        ''' Complete the parameters on the iterated process for a given
        iteration step.
        '''

                else:

        try:
            return
        self.capsul_iteration_step = step
        for attribute in iterated_attributes:
            iterated_values = getattr(attributes_set, attribute)
            step = min(len(iterated_values) - 1, self.capsul_iteration_step)
            value = iterated_values[step]
            setattr(step_attributes, attribute, value)
            values = getattr(process, parameter)
            if len(values) > self.capsul_iteration_step:
                parameters[parameter] = values[self.capsul_iteration_step]
        completion_engine.complete_parameters(parameters)

'''
Completion system for Capsul

Classes
=======
:class:`ProcessCompletionEngine`
--------------------------------
:class:`SwitchCompletionEngine`
-------------------------------
:class:`PathCompletionEngine`
-----------------------------
:class:`ProcessCompletionEngineFactory`
---------------------------------------
:class:`PathCompletionEngineFactory`
------------------------------------
'''

from soma.singleton import Singleton
from soma.controller import Controller, ControllerTrait
from capsul.pipeline.pipeline import Pipeline
from capsul.pipeline.pipeline import Graph
from capsul.pipeline.pipeline_nodes import (
    Node, ProcessNode, Switch, PipelineNode)
from capsul.attributes.attributes_schema import ProcessAttributes, \
    EditableAttributes
from capsul.pipeline import pipeline_tools
from soma.utils.weak_proxy import weak_proxy, get_ref
from soma.functiontools import SomaPartial
from soma.controller.trait_utils import relax_exists_constraint
import six
import sys
import copy

# DEBUG
#ce_calls = 0

class ProcessCompletionEngine(traits.HasTraits):
    ''' Parameters completion from attributes for a process or pipeline node
    instance, in the context of a specific data organization.

    ProcessCompletionEngine can be used directly for a pipeline, which merely
    delegates completion to its nodes, and has to be subclassed for a data
    organization framework on a node.

    To get a completion engine, use:

    ::

        completion_engine = ProcessCompletionEngine.get_completion_engine(
            node, name)

    Note that this will assign permanently the ProcessCompletionEngine object
    to its associated node or process.
    To get and set the attributes set:

    ::

        attributes = completion_engine.get_attribute_values()
        print(attributes.user_traits().keys())
        attributes.specific_process_attribute = 'a value'

    Once attributes are set, to process with parameters completion:

    ::

        completion_engine.complete_parameters()

    It is possible to have complete_parameters() triggered automatically when attributes or switch nodes change. To set this up, use:

    ::

        completion_engine.install_auto_completion()

    ProcessCompletionEngine can (and should) be specialized, at least to
    provide the attributes set for a given process. A factory is used to create
    the correct type of ProcessCompletionEngine for a given process / name:
    :py:class:`ProcessCompletionEngineFactory`

    :py:class:`capsul.attributes.fom_completion_engine.FomProcessCompletionEngine` is a specialization of ``ProcessCompletionEngine`` to manage File Organization Models (FOM).

    Methods
    -------

    get_completion_engine
    get_attribute_values
    complete_parameters
    set_parameters
    attributes_to_path
    get_path_completion_engine
    install_auto_completion
    remove_auto_completion

    '''

        super(ProcessCompletionEngine, self).__init__(
        self.process = weak_proxy(process, self._clear_node)
        self.name = name
        self.completion_ongoing = False
        self.add_trait('completion_progress', traits.Float(0.))
        self.add_trait('completion_progress_total', traits.Float(1.))
        self._rebuild_attributes = False

    def __del__(self):
        self.remove_switch_observers()
        self.remove_auto_completion()

    def _clear_node(self, wr):
        '''Called when the object behind the self.process proxy is about
        to be deleted
        '''
        self.process = None

        ''' Get attributes Controller associated to a process or node

        Returns
        -------
        attributes: ProcessAttributes instance

        The default implementation does nothing for a
        single Process instance, and merges attributes from its children if
        the process is a pipeline.

        '''
        if not self._rebuild_attributes \
                and 'capsul_attributes' in self._instance_traits():
            # we have to use this private HasTraits method _instance_traits()
            # to know if we actually have an instance trait, because
            # class traits are automatically added whenever we access an
            # attribute of another instance which doesn't have it (!)
            return self.capsul_attributes

        schemas = self._get_schemas()

        study_config = self.process.get_study_config()
        engine = study_config.engine

        proc_attr_cls = ProcessAttributes

        if 'capsul.engine.module.attributes' in engine._loaded_modules:
            factory = engine._modules_data['attributes']['attributes_factory']
            names = [self.process.name]
            if hasattr(self.process, 'context_name'):
                names.insert(0, self.process.context_name)
            for name in names:
                try:
                    proc_attr_cls = factory.get('process_attributes', name)
                    found = True
                    break
                except ValueError:
                    pass

        if not hasattr(self, 'capsul_attributes'):
            self.capsul_attributes = proc_attr_cls(self.process, schemas)

        # if no specialized attributes set and process is a pipeline,
        # try building from children nodes
        if proc_attr_cls is ProcessAttributes \
                and isinstance(self.process, (PipelineNode, Pipeline)):
            attributes = self.capsul_attributes
            pipeline = self.process
            if isinstance(pipeline, PipelineNode):
                pipeline = pipeline.process
            name = getattr(pipeline, 'context_name',
                           getattr(self.process, 'context_name',
                                   getattr(pipeline, 'context_name',
                                           pipeline.name)))

            for node_name, node in six.iteritems(pipeline.nodes):
                if node_name == '':
                    continue
                subprocess = None
                if hasattr(node, 'process'):
                    subprocess = node.process
                elif isinstance(node, Switch):
                    subprocess = node
                if subprocess is not None:
                    pname = '.'.join([name, node_name])
                    subprocess_compl = \
                        ProcessCompletionEngine.get_completion_engine(
                            subprocess, pname)
                    try:
                        sub_attributes \
                            = subprocess_compl.get_attribute_values()
                    except Exception:
                        try:
                            subprocess_compl = self.__class__(subprocess)
                            sub_attributes \
                                = subprocess_compl.get_attribute_values()
                        except Exception:
                            continue
                    for attribute, trait \
                            in six.iteritems(sub_attributes.user_traits()):
                        if attribute not in attributes._instance_traits():
                            attributes.add_trait(attribute, trait)
                            # all attributes are optional by default
                            attributes.trait(attribute).optional = True
                            setattr(attributes, attribute,
                                    getattr(sub_attributes, attribute))

            self._get_linked_attributes()

    def _get_linked_attributes(self):
        # for parameters which still do not have attributes, we can try
        # using links: if a linked parameter in a sub-process has
        # attributes, then we can get them here.
        attributes = self.capsul_attributes
        schema = 'link'  # FIXME
        param_attributes = attributes.get_parameters_attributes()
        forbidden_attributes = set(['generated_by_parameter',
                                    'generated_by_process'])
        done_parameters = set(
            [p for p, al in six.iteritems(param_attributes)
              if len([a for a in al if a not in forbidden_attributes])
              != 0])
        traits_types = {str: traits.Str, six.text_type: traits.Str, int: traits.Int,
                        float: traits.Float, list: traits.List}
        pipeline = self.process
        if isinstance(pipeline, PipelineNode):
            pipeline = pipeline.process
        name = pipeline.name
        for pname, trait in six.iteritems(pipeline.user_traits()):
            if pname in done_parameters:
                continue
            plug = pipeline.pipeline_node.plugs.get(pname)
            if plug is None:
                continue
                links = plug.links_from
            else:
                links = plug.links_to
            for link in links:
                node = link[2]
                completion_engine \
                    = ProcessCompletionEngine.get_completion_engine(
                        node, '.'.join([name, link[0]]))
                sub_attributes = completion_engine.get_attribute_values()
                sub_p_attribs = sub_attributes.get_parameters_attributes()
                if link[1] in sub_p_attribs:
                    s_p_attributes = sub_p_attribs[link[1]]
                    if len(s_p_attributes) != 0 \
                            and len([x for x in s_p_attributes.keys()
                                    if x not in forbidden_attributes]) != 0:
                        ea = EditableAttributes()
                        for attribute, value in six.iteritems(
                                s_p_attributes):
                            if attribute not in forbidden_attributes:
                                ttype = traits_types.get(type(value))
                                if ttype is not None:
                                    trait = ttype()
                                else:
                                    trait = value
                                ea.add_trait(attribute, ttype)
                                # all attributes are optional by default
                                ea.trait(attribute).optional = True
                                setattr(ea, attribute, value)

                        attributes.set_parameter_attributes(
                            pname, schema, ea, {})

                    break

    def complete_parameters(self, process_inputs={}, complete_iterations=True):
        ''' Completes file parameters from given inputs parameters, which may
        include both "regular" process parameters (file names) and attributes.

        process_inputs: dict (optional)
            parameters to be set on the process. It may include "regular"
            process parameters, and attributes used for completion. Attributes
            should be in a sub-dictionary under the key "capsul_attributes".
        complete_iterations: bool (optional)
            if False, iteration nodes inside the pipeline will not run their
            own completion. Thus parameters for the iterations will not be
            correctly completed. However this has 2 advantages: 1. it prevents
            modification of the input pipeline, 2. it will not do iterations
            completion which will anyway be done (again) when building a
            workflow in
            `~capsul.pipeline.pipeline_workflow.workflow_from_pipeline`.
        '''
        self.completion_progress_total = 1.
        self.set_parameters(process_inputs)

        # if process is a pipeline, trigger completions for its nodes and
        # sub-pipelines.
        #
        # Note: for now we do so first, so that parameters can be overwritten
        # afterwards by the higher-level pipeline FOM.
        # Ideally we should process the other way: complete high-level,
        # specific parameters first, then complete with lower-level, more
        # generic ones, while blocking already set ones.
        # as this blocking mechanism does not exist yet, we do it this way for
        # now, but it is sub-optimal since many parameters will be set many
        # times.

        def satisfied_deps(node, all_nodes, done):
            for param, plug in node.plugs.items():
                if not plug.output:
                    for link in plug.links_from:
                        snode = link[2]
                        if snode not in done \
                                and (link[0], snode) in all_nodes:
                            return False
            return True

        verbose = False

        pipeline = None
        if isinstance(self.process, PipelineNode):
            pipeline = self.process.process
        elif isinstance(self.process, Pipeline):
        if pipeline:
            attrib_values = self.get_attribute_values().export_to_dict()
            name = getattr(pipeline, 'context_name', pipeline.name)

            # build nodes list
            nodes_list = set([n for n in pipeline.nodes.items()
                              if n[0] != ''
                                  and pipeline_tools.is_node_enabled(
                                      pipeline, n[0], n[1], use_steps=False)])
            init_result = True
            done = set()
            todo = [(node_name, node) for node_name, node in nodes_list
                    if satisfied_deps(node, nodes_list, done)]

            self.completion_progress_total = len(nodes_list) + 0.05
            index = 0

            # process topologically through nodes dependencies
            while todo:
                node_name, node = todo.pop(0)
                done.add(node)

                pname = '.'.join([name, node_name])

                subprocess_compl = \
                    ProcessCompletionEngine.get_completion_engine(node, pname)
                self._install_subprogress_moniotoring(subprocess_compl)
                try:
                    subprocess_compl.complete_parameters(
                        {'capsul_attributes': attrib_values},
                        complete_iterations=complete_iterations)
                except Exception:
                    try:
                        self.__class__(node).complete_parameters(
                            {'capsul_attributes': attrib_values},
                            complete_iterations=complete_iterations)
                        pass
                self._remove_subprogress_moniotoring(subprocess_compl)

                # increase progress notification
                index += 1
                self.completion_progress = index

                # insert downstream nodes in todo list
                for param, plug in node.plugs.items():
                    if not plug.output:
                        continue
                    links = plug.links_to
                    for l in links:
                        dnode = l[2]
                        #print(l[0], dnode in done, dnode in )
                        if dnode not in done \
                                and (l[0], dnode) in nodes_list \
                                and (l[0], dnode) not in todo \
                                and satisfied_deps(dnode, nodes_list,
                                                    done):
                            todo.append((l[0], dnode))
                        # not needed any longer:
                        ## release "exists" property on connected traits
                        #if hasattr(dnode, 'process'):
                            #p = l[2].process
                        #else:
                            #p = l[2]
                        #trait = p.trait(l[1])
                        #if trait:
                            #relax_exists_constraint(trait)
                            ## FIXME this very specific stuff should be
                            ## handled another way at another place...
                            #if hasattr(p, 'process') \
                                    #and hasattr(p.process, 'inputs'):
                                ## MIA custom wrappings of nipype interfaces
                                ## are this way, and do not release the
                                ## exists constrain internally.
                                #relax_exists_constraint(
                                    #p.process.inputs.trait(l[1]))

            if len(done) != len(nodes_list):
                print('Some nodes of the pipeline could not be reached '
                      'through dependencies. The pipeline structure is '
                      'probably wrong:')
                print([nname for nname, n in nodes_list if n not in done])

        attributes = self.get_attribute_values()

        # if some attributes are list, we must separate list and non-list
        # attributes, and use an un-listed controller to get a path
        have_list = any([isinstance(t.trait_type, traits.List)
                         for t in attributes.user_traits().values()])
        if have_list:
            attributes_single = attributes.copy_to_single(with_values=True)
            attributes_list = attributes_single.copy_to_single(
                with_values=True)
        else:
            # no list parameter
            attributes_single = attributes

        # now complete process parameters:
        for pname, trait in six.iteritems(process.user_traits()):
            if trait.forbid_completion \
                    or process.is_parameter_protected(pname):
                # completion has been explicitly disabled on this parameter
                continue
            value = []  # for the try.. except
            try:
                if isinstance(process.trait(pname).trait_type,
                              traits.List):
                    nmax = 0
                    param_att = attributes.parameter_attributes.get(pname)
                    if param_att is None:
                        continue  # no completion for you

                    # FIXME: why [0][0]; check what it is...
                    for a in param_att[0][0] \
                            .user_traits().keys():
                        att_value = getattr(attributes, a)
                        if not isinstance(att_value, list):
                            if att_value is not None:
                                nmax = max(nmax, 1)
                        else:
                            nmax = max(nmax, len(att_value))
                    value = []
                    # param is a list: call iteratively the path completion
                    # for each attributes values set
                    for item in range(nmax):
                        for a, t in six.iteritems(attributes.user_traits()):
                            if isinstance(t.trait_type, traits.List):
                                att_value = getattr(attributes, a)
                                if not isinstance(att_value, list):
                                    setattr(attributes_list, a, att_value)
                                elif len(att_value) == 0:
                                    setattr(attributes_list, a,
                                            t.inner_traits[0].default)
                                else:
                                    if len(att_value) <= item:
                                        item = -1
                                            att_value[item])
                        value.append(
                            self.attributes_to_path(pname, attributes_list))
                    # avoid case of invalid attribute values
                    if value == [None]:
                        value = []
                else:
                    if pname in attributes.parameter_attributes:
                        value = self.attributes_to_path(pname,
                                                        attributes_single)
                    else:
                        value = None  # not in pattern: don't complete
                if value is not None:  # should None be valid ?
                    setattr(process, pname, value)
                if verbose:
                    print('Exception:', e)
                    print('param:', pname)
                    print('value:', repr(value))
                    import traceback
                    traceback.print_exc()
                #pass
        self.completion_progress = self.completion_progress_total

    def attributes_to_path(self, parameter, attributes):
        ''' Build a path from attributes for a given parameter in a process.

        attributes: ProcessAttributes instance (Controller)
        '''
        return self.get_path_completion_engine() \
            .attributes_to_path(self.process, parameter, attributes)

    def set_parameters(self, process_inputs):
        ''' Set the given parameters dict to the given process.
        process_inputs may include regular parameters of the underlying
        process, and attributes (capsul_attributes: dict).
        '''

        # This convenience method only differs from the Controller
        # import_from_dict() method in the way that capsul_attributes items
        # will not completely replace all the attributes values, but only set
        # those specified here, and leave the others in place.
        dst_attributes = self.get_attribute_values()
        attributes = process_inputs.get('capsul_attributes')
        if attributes:
            avail_attrib = set(dst_attributes.user_traits().keys())
            attributes = dict((k, v) for k, v in six.iteritems(attributes)
                              if k in avail_attrib)
            dst_attributes.import_from_dict(attributes)
        process_inputs = dict((k, v) for k, v
                              in six.iteritems(process_inputs)
                              if k != 'capsul_attributes')
            process = self.process.process
        process.import_from_dict(process_inputs)

    def attributes_changed(self, obj, name, old, new):
        ''' Traits changed callback which triggers parameters update.

        This method basically calls complete_parameters() (after some checks).

        Users do not normally have to use it directly, it is used internally
        when install_auto_completion() has been called.

        See :py:meth:`install_auto_completion`
        '''
        if name != 'trait_added' and name != 'user_traits_changed' \
                and self.completion_ongoing is False:
            #setattr(self.capsul_attributes, name, new)
            self.completion_ongoing = True
            self.complete_parameters({'capsul_attributes': {name: new}})
            self.completion_ongoing = False

    def nodes_selection_changed(self, obj, name, old, new):

        '''
        if not self.completion_ongoing:
            self._rebuild_attributes = True
            self.complete_parameters()

    def install_auto_completion(self):
        ''' Monitor attributes changes and switches changes (which may
        influence attributes) and recompute parameters completion when needed.
        '''
        self.get_attribute_values().on_trait_change(
            self.attributes_changed, 'anytrait')

        if isinstance(process, Pipeline):
            for node_name, node in six.iteritems(process.nodes):
                if isinstance(node, Switch):
                    # a switch may change attributes dynamically
                    # so we must be notified if this happens.
                    node.on_trait_change(self.nodes_selection_changed,
                                         'switch')

    def remove_auto_completion(self):
        ''' Remove attributes monitoring and auto-recomputing of parameters.

        Reverts install_auto_completion()
        '''
        if self.process is not None:
            try:
                av = self.process.completion_engine
            except (ReferenceError, AttributeError):
                return
            av.on_trait_change(
                self.attributes_changed, 'anytrait', remove=True)

            if isinstance(process, PipelineNode):
            if isinstance(process, Pipeline):
                for node_name, node in six.iteritems(process.nodes):
                    if isinstance(node, Switch):
                        # a switch may change attributes dynamically
                        # so we must be notified if this happens.
                        node.on_trait_change(self.nodes_selection_changed,
                                            'switch', remove=True)

    def get_path_completion_engine(self):
        ''' Get a PathCompletionEngine object for the given process.
        The default implementation queries PathCompletionEngineFactory,
        but some specific ProcessCompletionEngine implementations may override
        it for path completion at the process level (FOMs for instance).
        '''
        engine_factory = None
            try:
                engine_factory \
                    = engine._modules_data['attributes'] \
                        ['attributes_factory'].get(
                            'path_completion', study_config.path_completion)
            except ValueError:
                pass # not found
        if engine_factory is None:
            engine_factory = PathCompletionEngineFactory()
        return engine_factory.get_path_completion_engine(self.process)

    @staticmethod
    def get_completion_engine(process, name=None):
        ''' Get a ProcessCompletionEngine instance for a given process/node
        within the framework of its StudyConfig factory function.
        '''
        #global ce_calls
        #ce_calls += 1
        study_config = None
        if hasattr(process, 'get_study_config'):
            # switches don't have a study_config at the moment.
            study_config = process.get_study_config()
            engine = study_config.engine
            if 'capsul.engine.module.attributes' in engine._loaded_modules:
                try:
                    engine_factory \
                        = engine._modules_data['attributes'] \
                            ['attributes_factory'].get(
                                'process_completion',
                                study_config.process_completion)
                    pass # not found
            from . import completion_engine_factory
            engine_factory = completion_engine_factory. \
                BuiltinProcessCompletionEngineFactory()
        completion_engine = engine_factory.get_completion_engine(
            process, name=name)
        # I remove the completion_engine cache because when the FOM config
        # changes in StudyConfig, the completion engine may change also,
        # and caching it will make use of an obsolete one.
        # The other option is to remove the caches when the FOM config changes,
        # but this needs to setup many callbacks that we don't know easily
        # when to clear (process deletion etc)
        ## set the completion engine into the process
        if completion_engine is not None:
            process.completion_engine = completion_engine
            process._has_studyconfig_callback = True
            if study_config is not None:
                from capsul.process.process import Process
                if isinstance(process, Process):
                    nclass = Process
                else:
                    nclass = Node
                if not hasattr(nclass, '_remove_completion_engine'):
                    nclass._remove_completion_engine \
                        = ProcessCompletionEngine._remove_completion_engine
                    nclass.__del__ \
                        = ProcessCompletionEngine._del_process_callback
                else:
                    try:
                        # remove former callback, if any
                        study_config.on_trait_change(
                        process._remove_completion_engine,
                        'use_fom,input_fom,output_fom,shared_fom', remove=True)
                        pass
                study_config.on_trait_change(
                    process._remove_completion_engine,
                    'use_fom,input_fom,output_fom,shared_fom')
        return completion_engine

    def _remove_completion_engine(process):
            del process.completion_engine

    def _del_process_callback(process):
        if hasattr(process, 'study_config') \
                and process.study_config is not None \
                and hasattr(process, '_has_studyconfig_callback'):
            try:
                process.study_config.on_trait_change(
                    'use_fom,input_fom,output_fom,shared_fom', remove=True)
            except ImportError:
                # python shutting down
                pass
            del process._has_studyconfig_callback

    def _get_schemas(self):
        ''' Get schemas dictionary from process and its StudyConfig
        '''
        schemas = {}
        try:
            study_config = self.process.get_study_config()
        except ReferenceError:
            # process is deleted
            return schemas
        factory = getattr(engine, '_modules_data', {}).get(
            'attributes', {}).get('attributes_factory', None)
        if factory is not None:
            for dir_name, schema_name \
                    in six.iteritems(study_config.attributes_schemas):
                schemas[dir_name] = factory.get('schema', schema_name)
        return schemas

    def _install_subprogress_moniotoring(self, subprocess_compl):
        monitor_subprocess_progress = getattr(
            self, 'monitor_subprocess_progress', True)
        if monitor_subprocess_progress:
            self._old_monitor_sub = getattr(
                subprocess_compl, 'monitor_subprocess_progress', True)
            subprocess_compl.monitor_subprocess_progress = False
            self._current_progress = self.completion_progress
            self._monitoring_callback = SomaPartial(
                self.__class__._substep_completion_progress, weak_proxy(self),
                subprocess_compl)
            subprocess_compl.on_trait_change(
                self._monitoring_callback, 'completion_progress')

    def _remove_subprogress_moniotoring(self, subprocess_compl):
                self._monitoring_callback, 'completion_progress', remove=True)
            del self._current_progress
            del self._monitoring_callback
            if self._old_monitor_sub:
                del subprocess_compl.monitor_subprocess_progress
            del self._old_monitor_sub

    def _substep_completion_progress(self, substep_completion_engine, obj,
                                     name, old, new):
        sub_completion_rate \
            = substep_completion_engine.completion_progress \
                / substep_completion_engine.completion_progress_total
        self.completion_progress = self._current_progress \
            + sub_completion_rate

    def remove_attributes(self):
        '''Clear attributes controller cache, to allow rebuilding it after
        a change. This is generally a callback attached to switches changes.
        '''
        if 'capsul_attributes' in self._instance_traits():
            self.remove_trait('capsul_attributes')

    def remove_switch_observers(self):
        '''Remove notification callbacks previously set to listen switches
        state changes.
        '''
        try:
                for name, node in six.iteritems(process.nodes):
                    if isinstance(node, Switch) \
                            and hasattr(node, 'completion_engine'):
                        completion_engine = node.completion_engine
                        completion_engine.remove_switch_observer(self)
            # the process has already been destroyed
            pass

class SwitchCompletionEngine(ProcessCompletionEngine):
    ''' Completion engine specislization for a switch. The switch will
    propagate attributes from its selected inputs to corresponding outputs,
    if they can be retrieved from parameters links. Otherwise the countrary
    will be tried (propagated from outputs to inputs).
    '''

        self.remove_switch_observer()

        self.add_trait('capsul_attributes', ControllerTrait(Controller()))
        capsul_attributes = ProcessAttributes(self.process, {})
        self.capsul_attributes = capsul_attributes
        outputs = self.process._outputs
        schema = 'switch'  # FIXME
        name = getattr(self.process, 'context_name', self.name)
        pipeline_name = '.'.join(name.split('.')[:-1])
        if pipeline_name == '':
            pipeline_name = []
        else:
            pipeline_name = [pipeline_name]
        for out_name in outputs:
            in_name = '_switch_'.join((self.process.switch, out_name))
            found = False
            for output, name in ((False, in_name), (True, out_name)):
                plug = self.process.plugs.get(name)
                if plug is None:
                    continue
                if output:
                else:
                    links = plug.links_from
                for link in links:
                    node = link[2]
                        # FIXME: just for now
                        continue
                    if link[0] == '':
                        # link to the parent pipeline: don't call it to avoid
                        # an infinite loop.
                        # Either it will provide attributes by its own, either
                        # we must not take them into account, so skip it.
                        continue
                    proc_name = '.'.join(pipeline_name + [link[0]])
                    completion_engine \
                        = ProcessCompletionEngine.get_completion_engine(
                            node, name=proc_name)
                    attributes = completion_engine.get_attribute_values()
                    try:
                        param_attributes \
                            = attributes.get_parameters_attributes()[link[1]]
                        continue

                    if len(param_attributes) != 0 \
                            and len([x for x in param_attributes.keys()
                                     if x not in forbidden_attributes]) != 0:
                                param_attributes):
                                else:

                        capsul_attributes.set_parameter_attributes(
                            name, schema, ea, {})
                        found = True
                        break
                if found:
                    break
            if found:
                # propagate from input/output to other side
                ea = EditableAttributes()
                for attribute, value in six.iteritems(
                        param_attributes):
                    ttype = traits_types.get(type(value))
                    if ttype is not None:
                        trait = ttype()
                    else:
                        trait = value
                    ea.add_trait(attribute, ttype)
                    # all attributes are optional by default
                    ea.trait(attribute).optional = True
                    setattr(ea, attribute, value)
                if output:
                    capsul_attributes.set_parameter_attributes(
                        in_name, schema, ea, {})
                else:
                        out_name, schema, ea, {})

        self.install_switch_observer()
        return capsul_attributes

    def install_switch_observer(self, observer=None):
        '''Setup a switch change observation, to remove parameters attributes
        when the switch state changes.

        observer: ProcessCompletionEngine instance
            The observer which should change attributes after switch change.
            If not specified, the observer is the switch completion engine
            (self).
            Notification will call the observer remove_attributes() method.
        '''
        if observer is None:
            observer = self
        self.process.on_trait_change(observer.remove_attributes, 'switch')

    def remove_switch_observer(self, observer=None):
        '''Remove notification previously set by install_switch_observer()
        '''
        try:
            self.process.on_trait_change(observer.remove_attributes, 'switch',
                                         remove=True)
        except Exception:
            pass  # probably already deleted object

class PathCompletionEngine(object):
    ''' Implements building of a single path from a set of attributes for a
    specific process / parameter
    '''
    def attributes_to_path(self, process, parameter, attributes):

        This method has to be specialized. The default implementation returns
        None.

        process: Node or Process instance
        '''
        return None

    def allowed_formats(self, process, parameter):
        ''' List of possible formats names associated with a parameter
        '''
        return []

    def allowed_extensions(self, process, parameter):
        ''' List of possible file extensions associated with a parameter
        '''
        return []

class ProcessCompletionEngineFactory(object):
    '''
    Get a :class:`ProcessCompletionEngine` instance
    '''
    factory_id = 'basic'

        '''
        instance for a process in the context of a given StudyConfig.

        The base class implementation returns a base ProcessCompletionEngine
        instance, which is quite incomplete.
        '''
            return ce

        if isinstance(process, Switch):
            return SwitchCompletionEngine(process, name=name)
        return ProcessCompletionEngine(process, name=name)

class PathCompletionEngineFactory(object):
    '''
    Get a :class:`PathCompletionEngine` instance
    '''

    factory_id = 'null'

    def get_path_completion_engine(self, process):
        raise RuntimeError('PathCompletionEngineFactory is pure virtual. '
                           'It must be derived to do actual work.')

'''
Completion engine for File Organization Models (FOM).

Classes
=======
:class:`FomProcessCompletionEngine`
-----------------------------------
:class:`FomPathCompletionEngine`
:class:`FomProcessCompletionEngineIteration`
--------------------------------------------
'''

import os
import six
from traits.api import Str, HasTraits, List, Undefined

from capsul.pipeline.pipeline_nodes import Node, Switch, ProcessNode
    ProcessCompletionEngineFactory, PathCompletionEngine, \
    PathCompletionEngineFactory
from soma.fom import DirectoryAsDict
from soma.path import split_path
from soma.sorted_dictionary import SortedDictionary

class FomProcessCompletionEngine(ProcessCompletionEngine):
    """
    FOM (File Organization Model) implementation of completion engine.

    * A capsul.study_config.StudyConfig also needs to be configured with FOM
      module, and selected FOMS and directories:

    ::

        from capsul.api import StudyConfig
        from capsul.study_config.config_modules.fom_config import FomConfig
        study_config = StudyConfig(modules=StudyConfig.default_modules
                                   + ['FomConfig', 'BrainVISAConfig'])
        study_config.update_study_configuration('study_config.json')

    * Only then a FomProcessCompletionEngine can be created:

    ::

        process = get_process_instance('morphologist')
        fom_completion_engine = FomProcessCompletionEngine(
            process, study_config)

    But generally this creation is handled via the
    ProcessCompletionEngine.get_completion_engine() function:

    ::

        fom_completion_engine = ProcessCompletionEngine.get_completion_engine(
            process)

    Parameters
    ----------
    name: string (optional)
        name of the process in the FOM dictionary. By default the
        process.name variable will be used.

    Methods
    -------
    create_attributes_with_fom
    """
        super(FomProcessCompletionEngine, self).__init__(
        self.input_fom = None
        self.output_fom = None
        self.shared_fom = None

        Returns
        -------
        '''
                and self.trait('capsul_attributes') is not None \
                and hasattr(self, 'capsul_attributes'):

        self.create_attributes_with_fom()

    def create_attributes_with_fom(self):
        """To get useful attributes by the fom"""

        # print('create_attributes_with_fom for', self.process, ', FCE:', self)

        study_config = process.study_config
        modules_data = study_config.modules_data

        #Get attributes in input fom
        id = getattr(process, 'id', None)
        names_search_list = [self.name]
        if id:
            names_search_list.append(id)
        names_search_list += [process.name,
                             getattr(process, 'context_name', '')]

            self.capsul_attributes = ProcessAttributes(self.process, schemas)
        capsul_attributes = self.capsul_attributes

        matching_fom = False
        input_found = False
        output_found = False
        shared_fom = None

        # print('create_attributes_with_fom for', self.process, ', FCE:', self, ', foms:', {n: f.fom_names[-1] for n, f in modules_data.foms.items()})

        foms = SortedDictionary()
        foms.update(modules_data.foms)
        if study_config.auto_fom:
            # in auto-fom mode, also search in additional and non-loaded FOMs
            foms.update(modules_data.all_foms)

        def editable_attributes(attributes, fom):
            ea = EditableAttributes()
            for attribute in attributes:
                if attribute.startswith('fom_'):
                    continue # skip FOM internals
                default_value = fom.attribute_definitions[attribute].get(
                    'default_value', '')
                ea.add_trait(attribute, Str(default_value, optional=True))
            return ea

        sel_foms = {}
        if study_config.input_fom:
            sel_foms['input'] = study_config.input_fom
        if study_config.output_fom:
            sel_foms['output'] = study_config.output_fom
        if study_config.shared_fom:
            sel_foms['shared'] = study_config.shared_fom

        for schema, fom in foms.items():
            if fom is None:
                fom, atp, pta \
                    = study_config.modules['FomConfig'].load_fom(schema)
                foms[schema] = fom

            if schema not in ('input', 'output', 'shared'):
                # exclude incompatible FOMs
                for fs in ('input', 'output', 'shared'):
                    found = False
                    f = modules_data.foms.get(fs)
                    if f is None:
                        if fs == 'shared':
                            if 'shared' in schema:
                                sel_foms['shared'] = schema
                                found = True
                        elif 'shared' not in schema:
                            sel_foms[fs] = schema
                            found = True
                    elif fom is not f:
                        if sum([n in fom.fom_names
                                for n in f.fom_names]) \
                                == len(f.fom_names):
                            # all sub-foms of f included in fom:
                            # possibly keep it
                            if fs != 'shared':
                                # print('extend fom:', schema, 'for', fs, ':', f.fom_names)
                                for name in names_search_list:
                                    fom_patterns = fom.patterns.get(name)
                                    if fom_patterns is not None:
                                        found = True
                                        break
                                #else:
                                    #print('process not found - discard it')
                            else:
                            if found:
                                sel_foms[fs] = schema

        # print('new foms:', sel_foms)

        fom_modified = False

        found = False
        for schema, fom_type in sel_foms.items():
            # print('schema:', schema, fom_type)
            fom = modules_data.all_foms[fom_type]
            atp = modules_data.fom_atp['all'].get(fom_type) \
                or modules_data.fom_atp.get(schema)

            if atp is None:
                continue
            for name in names_search_list:
                fom_patterns = fom.patterns.get(name)
                if fom_patterns is not None:
                    break
            else:
                # print('process', names_search_list, 'not found in', fom_type)
                continue

            found = True
            # print('completion using FOM:', schema, fom_type, 'for', process.id, ', atp:', atp)
            #break

            for parameter in fom_patterns:
                param_attributes = atp.find_discriminant_attributes(
                        fom_parameter=parameter, fom_process=name)
                ea = editable_attributes(param_attributes, fom)
                try:
                        parameter, schema, ea, {})
                except KeyError:
                    # param already registered
                    pass

            if (schema not in modules_data.foms
                    or modules_data.foms[schema] != fom
                    or schema not in modules_data.fom_atp
                    or modules_data.fom_atp[schema] != atp
                    or getattr(study_config, '%s_fom' % schema) != fom_type):
                modules_data.foms[schema] = fom
                modules_data.fom_atp[schema] = atp
                setattr(study_config, '%s_fom' % schema, fom_type)
                fom_modified = True

        if not found:
            # no FOM contains the process
            raise KeyError('No matching FOM for process %s'
                           % repr(names_search_list))

        #if fom_modified:
            # the modif will trigger another completion
            #return

        # in a pipeline, we still must iterate over nodes to find switches,
        # which have their own behaviour.
            name = process.name

            for node_name, node in list(process.nodes.items()):
                    if subprocess is None:
                        continue
                    try:
                        try:
                            continue
                        if attributes.trait(attribute) is None:

        # remember foms for later
        if 'input' in sel_foms:
            self.input_fom = sel_foms['input']
        else:
            self.input_fom = study_config.input_fom
        if 'output' in sel_foms:
            self.output_fom = sel_foms['output']
        else:
            self.output_fom = study_config.output_fom
        if 'shared' in sel_foms:
            self.shared_fom = sel_foms['shared']
        else:
            self.shared_fom = study_config.shared_fom

    def setup_fom(process):
        completion_engine \
            = ProcessCompletionEngine.get_completion_engine(process)
        if not isinstance(completion_engine, FomProcessCompletionEngine):
            return
        if not hasattr(completion_engine, 'input_fom') \
                or completion_engine.input_fom is None:
            if process.study_config.input_fom != completion_engine.input_fom \
                    or process.study_config.output_fom \
                        != completion_engine.output_fom \
                    or process.study_config.shared_fom \
                        != completion_engine.shared_fom:
                # FOMs have changed: rebuild attributes
                completion_engine._rebuild_attributes = True
            completion_engine.create_attributes_with_fom()
        if process.study_config.input_fom != completion_engine.input_fom:
            process.study_config.input_fom = completion_engine.input_fom
        if process.study_config.output_fom != completion_engine.output_fom:
            process.study_config.output_fom = completion_engine.output_fom
        if process.study_config.shared_fom != completion_engine.shared_fom:
            process.study_config.shared_fom = completion_engine.shared_fom

    def path_attributes(self, filename, parameter=None):
        """By the path, find value of attributes"""

        pta = self.process.study_config.modules_data.fom_pta['input']

        # Extract the attributes from the first result returned by
        # parse_directory
        liste = split_path(filename)
        len_element_to_delete = 1
        for element in liste:
            if element != os.sep:
                len_element_to_delete \
                    = len_element_to_delete + len(element) + 1
                new_value = filename[len_element_to_delete:len(filename)]
                try:
                    #import logging
                    #logging.root.setLevel( logging.DEBUG )
                    #path, st, self.attributes = pta.parse_directory(
                    #    DirectoryAsDict.paths_to_dict( new_value),
                    #        log=logging ).next()
                    path, st, attributes = next(pta.parse_directory(
                        DirectoryAsDict.paths_to_dict( new_value) ))
                    break
                except StopIteration:
                    if element == liste[-1]:
                        raise ValueError(
                            '%s is not recognized for parameter "%s" of "%s"'
                            % (new_value, parameter, self.process.name))

        attrib_values = self.get_attribute_values().export_to_dict()
        for att in attributes:
            if att in list(attrib_values.keys()):
                setattr(attrib_values, att, attributes[att])
        return attributes

        '''
        '''
        return FomPathCompletionEngine()

    def _fom_completion_factory(process, name):
        ''' Factoty inserted in attributed_processFactory
        '''
        if study_config is None \
                or 'FomConfig' not in study_config.modules \
                or study_config.use_fom == False:
            #print("no FOM:", study_config, study_config.modules.keys())
            return None  # Non Fom config, no way it could work
        try:
            pfom = FomProcessCompletionEngine(process, name)
            return pfom
        except KeyError:
            # process not in FOM
            pass

class FomPathCompletionEngine(PathCompletionEngine):

        ''' Build a path from attributes

        process: Process instance
        '''
        FomProcessCompletionEngine.setup_fom(process)

        input_fom = process.study_config.modules_data.foms['input']
        output_fom = process.study_config.modules_data.foms['output']
        input_atp = process.study_config.modules_data.fom_atp['input']
        output_atp = process.study_config.modules_data.fom_atp['output']

        #Create completion
        names_search_list = []
        if isinstance(process, Node):
            trait = process.get_trait(parameter)
            if hasattr(process, 'process'):
                if hasattr(process.process, 'context_name'):
                    names_search_list.append(process.process.context_name)
                names_search_list.append(process.process.name)
        else:
            trait = process.trait(parameter)
            name = process.id
            names_search_list.append(name)
        if trait.output:
            atp = output_atp
            fom = output_fom
        else:
            atp = input_atp
            fom = input_fom
                              getattr(process, 'context_name', '')]
        for fname in names_search_list:
            fom_patterns = fom.patterns.get(fname)
            if fom_patterns is not None:
                name = fname
                break
        else:
            raise KeyError('Process not found in FOMs amongst %s' \
                % repr(names_search_list))

        allowed_attributes = set(attributes.user_traits().keys())
        allowed_attributes.discard('parameter')
        allowed_attributes.discard('process_name')
        #allowed_attributes = set(attributes.get_parameters_attributes()[
            #parameter].keys())
        #allowed_attributes.discard('type')
        #allowed_attributes.discard('generated_by_parameter')
        #allowed_attributes.discard('generated_by_process')

        # Select only the attributes that are discriminant for this
        # parameter otherwise other attributes can prevent the appropriate
        # rule to match
        parameter_attributes = atp.find_discriminant_attributes(
            fom_parameter=parameter, fom_process=name)
        d = {i: getattr(attributes, i)
             for i in parameter_attributes
             if i in allowed_attributes
             and getattr(attributes, i) not in (None, Undefined)}
        d['fom_process'] = name
        d['fom_parameter'] = parameter
        d['fom_format'] = 'fom_preferred'
        path_value = None
        #path_values = []
        #debug = getattr(self, 'debug', None)
        for h in atp.find_paths(d):  # , debug=debug):
            path_value = h[0]
            # find_paths() is a generator which can sometimes generate
            # several values (formats). We are only interested in the
            # first one.
            #path_values.append(h[0])
            break

        return path_value

    def open_values_attributes(self, process, parameter):
        ''' Attributes with "open" values, not restricted to a list of possible
        values
        '''
        # print('open_values_attributes', process.id, parameter)

        for schema in ('input', 'output', 'shared'):
            fom = process.study_config.modules_data.foms[schema]
            atp = process.study_config.modules_data.fom_atp[schema]
            # print('fom:', fom.fom_names)
            # print('atp:', atp)

            name = getattr(process, 'id', process.name)
            names_search_list = []
            if hasattr(process, 'id'):
                names_search_list.append(process.id)
            names_search_list += [process.name,
                                  getattr(process, 'context_name', '')]
            for fname in names_search_list:
                fom_patterns = fom.patterns.get(fname)
                    name = fname
                    break
            else:
                continue

            values = atp.find_attributes_values()
            attributes = [k for k, v in six.iteritems(values)
                          if k not in ('fom_name', 'fom_process',
                                       'fom_parameter', 'fom_format')
                            and len(v) == 2 and v[1] == (u'', )]
            return attributes

        '''

        formats = []
            sub_f = atp.allowed_formats_for_parameter(process.name, parameter)
            formats += [f for f in sub_f if f not in formats]
        return formats

        '''

        exts = set()
            sub_e = atp.allowed_extensions_for_parameter(
                process_name=process.name, param=parameter)
            exts.update(sub_e)
        # sort and add dots
        exts2 = sorted(['.%s' % e for e in exts if e])
        if '' in exts:
            exts2.append('')
        return exts2

class FomProcessCompletionEngineIteration(ProcessCompletionEngineIteration):

        subprocess = process.process

        FomProcessCompletionEngine.setup_fom(subprocess)

        input_fom = subprocess.study_config.modules_data.foms['input']
        output_fom = subprocess.study_config.modules_data.foms['output']
        input_atp = subprocess.study_config.modules_data.fom_atp['input']
        output_atp = subprocess.study_config.modules_data.fom_atp['output']

        name = subprocess.id
        names_search_list = (subprocess.id, subprocess.name,
                             getattr(subprocess, 'context_name', ''))
        for fom in (input_fom, output_fom):
                    break
            else:
                continue
            break
        else:

        iter_attrib = set()
            params = list(subprocess.user_traits().keys())
        else:
            params = process.iterative_parameters
        for parameter in params:
            if subprocess.trait(parameter).output:
                atp = output_atp
            else:
                atp = input_atp
            parameter_attributes = set([
                x for x in atp.find_discriminant_attributes(
                    fom_parameter=parameter, fom_process=name)
                if not x.startswith('fom_')])
            iter_attrib.update(parameter_attributes)
        return iter_attrib

#class FomPathCompletionEngineFactory(PathCompletionEngineFactory):

    #factory_id = 'fom'

    #def get_path_completion_engine(self, process):
        #return FomPathCompletionEngine(process)

"""
This module defines the main API to interact with Capsul processes.
In order to execute a process, it is mandatory to have an instance of
:py:class:`CapsulEngine`. Such an instance can be created with factory
:py:func:`capsul_engine`

Classes
=======
:class:`CapsulEngine`

Functions
=========
:func:`database_factory`
------------------------
:func:`capsul_engine`
:func:`activate_configuration`
------------------------------
"""

import importlib
import os
import os.path as osp
import re
import tempfile
import weakref

from traits.api import Dict, String, Undefined

from soma.controller import Controller, controller_to_dict
from soma.serialization import to_json, from_json
from soma.utils.weak_proxy import get_ref

from .database_populse import PopulseDBEngine

from .settings import Settings
from .module import default_modules
from . import run
from .run import WorkflowExecutionError

# FIXME TODO: OBSOLETE

# Questions about API/implementation:

# * execution:
# * workflows are not exposed, they are running a possibly different pipeline (single process case), thus we need to keep track on it
# * logging / history / provenance, databasing
# * retrieving output files with transfers: when ? currently in wait(), should it be a separate method ? should it be asynchronous ?
# * setting output parameters: currently in wait(), should it be a separate method ?
# * disconnections / reconnections client / server
# * actually connect computing resource[s]
# * settings / config:
# * see comments in settings.py
# * GUI and constraints on parameters ?
# * how to handle optional dependencies: ie nipype depends on spm if spm is installed / configured, otherwise we can run other nipype interfaces, but no spm ones
# * integrate soma-workflow config + CE.computing_resource

class CapsulEngine(Controller):
    """
    A CapsulEngine is the mandatory entry point of all software using Capsul.
    It contains objects to store configuration and metadata, defines execution
    environment(s) (possibly remote) and performs pipelines execution.

    A CapsulEngine must be created using capsul.engine.capsul_engine function.
    For instance::

        from capsul.engine import capsul_engine
        ce = capsul_engine()

    Or::

        from capsul.api import capsul_engine

    By default, CapsulEngine only stores necessary configuration. But it may be
    necessary to modify the Python environment globally to apply this
    configuration. For instance, Nipype must be configured globally. If SPM is
    configured in CapsulEngine, it is necessary to explicitly activate the
    configuration in order to modify the global configuration of Nipype for
    SPM. This activation is done by explicitly activating the execution
    context of the capsul engine with the following code, inside a running
    process::

        from capsul.engine import capsul_engine, activate_configuration
        # Nipype is not configured here
        config = capsul_engine.settings.select_configurations(
            'global', {'nipype': 'any'})
        activate_configuration(config)
        # Nipype is configured here

    .. note::

        CapsulEngine is the replacement of the older
        :class:`~capsul.study_config.study_config.StudyConfig`, which is still
        present in Capsul 2.2 for backward compatibility, but will disappear in
        later versions. In Capsul 2.2 both objects exist, and are synchronized
        internally, which means that a StudyConfig object will also create a
        CapsulEngine, and the other way, and modifications in the StudyConfig
        object will change the corresponding item in CapsulEngine and vice
        versa. Functionalities of StudyConfig are moving internally to
        CapsulEngine, StudyConfig being merely a wrapper.

    **Using CapsulEngine**

    It is used to store configuration variables, and to handle execution within
    the configured context. The configuration has 2 independent axes:
    configuration modules, which provide additional configuration variables,
    and "environments" which typically represent computing resources.

    *Computing resources*

    Capsul is using :somaworkflow:`Soma-Workflow <index.html>` to run
    processes, and is thus able to connect and execute on a remote computing
    server. The remote computing resource may have a different configuration
    from the client one (paths for software or data, available external
    software etc). So configurations specific to different computing resources
    should be handled in CapsulEngine. For this, the configuration section is
    split into several configuration entries, one for each computing resource.

    As this is a little bit complex to handle at first, a "global"
    configuration (what we call "environment") is used to maintain all common
    configuration options. It is typically used to work on the local machine,
    especially for users who only work locally.

    Configuration is stored in a database (either internal or persistent),
    through the :class:`~capsul.engine.settings.Settings` object found in
    ``CapsulEngine.settings``.
    Access and modification of settings should occur within a session block
    using ``with capsul_engine.settings as session``. See the
    :class:`~capsul.engine.settings.Settings` class for details.

    ::

        >>> from capsul.api import capsul_engine
        >>> ce = capsul_engine()
        >>> config = ce.settings.select_configurations('global')
        >>> config = ce.global_config
        >>> print(config)
        {'capsul_engine': {'uses': {'capsul.engine.module.fsl': 'ALL',
          'capsul.engine.module.matlab': 'ALL',
          'capsul.engine.module.spm': 'ALL'}}}

    Whenever a new computing resource is used, it can be added as a new
    environment key to all configuration operations.

    Note that the settings store all possible configurations for all
    environments (or computing resources), but are not "activated": this is
    only done at runtime in specific process execution functions: each process
    may need to select and use a different configuration from other ones, and
    activate it individually.

    :class:`~capsul.process.process.Process` subclasses or instances may
    provide their configuration requirements via their
    :meth:`~capsul.process.process.Process.requirements` method. This method
    returns a dictionary of request strings (one element per needed module)
    that will be used to select one configuration amongst the available
    settings entries of each required module.

    *configuration modules*

    The configuration is handled through a set of configuration modules. Each
    is dedicated for a topic (for instance handling a specific external
    software paths, or managing process parameters completion, etc). A module
    adds a settings table in the database, with its own variables, and is able
    to manage runtime configuration of programs, if needed, through its
    ``activate_configurations`` function. Capsul comes with a
    set of predefined modules:
    :class:`~capsul.engine.module.attributes`,
    :class:`~capsul.engine.module.axon`,
    :class:`~capsul.engine.module.fom`,
    :class:`~capsul.engine.module.fsl`,
    :class:`~capsul.engine.module.matlab`,
    :class:`~capsul.engine.module.spm`

    **Methods**
    """

    def __init__(self, database_location, database, require):
        """
        CapsulEngine.__init__(self, database_location, database, config=None)

        The CapsulEngine constructor should not be called directly.
        Use :func:`capsul_engine` factory function instead.
        """
        super(CapsulEngine, self).__init__()

        self._settings = None

        self._database_location = database_location
        self._database = database

        self._loaded_modules = set()
        self.load_modules(require)

        from capsul.study_config.study_config import StudyConfig

        self.study_config = StudyConfig(engine=self)

        self._metadata_engine = from_json(database.json_value("metadata_engine"))

        self._connected_resource = ""

    @property
    def settings(self):
        if self._settings is None:
            self._settings = Settings(self.database.storage)
        return self._settings

    @property
    def database(self):
        return self._database

    @property
    def database_location(self):
        return self._database_location

    @property
    def metadata_engine(self):
        return self._metadata_engine

    @metadata_engine.setter
    def metadata_engine(self, metadata_engine):
        self._metadata_engine = metadata_engine
        self.database.set_json_value("metadata_engine", to_json(self._metadata_engine))

    def load_modules(self, require):
        """
        Call self.load_module for each required module. The list of modules
        to load is located in self.modules (if it is None,
        capsul.module.default_modules is used).
        """
        if require is None:
            require = default_modules

        for module in require:
            self.load_module(module)

    def load_module(self, module_name):
        """
        Load a module if it has not already been loaded (is this case,
        nothing is done)

        A module is a fully qualified name of a Python module (as accepted
        by Python import statement). Such a module must define the two
        following functions (and may define two others, see below):

        def load_module(capsul_engine, module_name):
        def set_environ(config, environ):

        load_module of each module is called once before reading and applying
        the configuration. It can be used to add traits to the CapsulEngine
        in order to define the configuration options that are used by the
        module. Values of these traits are automatically stored in
        configuration in database when self.save() is used, and they are
        retrieved from database before initializing modules.

        set_environ is called in the context of the processing (i.e. on
        the, possibly remote, machine that runs the pipelines). It receives
        the configuration as a JSON compatible dictionary (for instance a
        CapsulEngine attribute `capsul_engine.spm.directory` would be
        config['spm']['directory']). The function must modify the environ
        dictionary to set the environment variables that must be defined
        for pipeline configuration. These variables are typically used by
        modules in capsul.in_context module to manage running external
        software with appropriate configuration.
        """
        module_name = self.settings.module_name(module_name)
        if module_name not in self._loaded_modules:
            self._loaded_modules.add(module_name)
            python_module = importlib.import_module(module_name)
            init_settings = getattr(python_module, "init_settings", None)
            if init_settings is not None:
                init_settings(self)
        return False

    #
    # Method imported from self.database
    #

    # TODO: take computing resource in account in the following methods

    def set_named_directory(self, name, path):
        return self.database.set_named_directory(name, path)

    def named_directory(self, name):
        return self.database.named_directory(name)

    def named_directories(self):
        return self.database.set_named_directories()

    def set_json_value(self, name, json_value):
        return self.database.set_json_value(name, json_value)

    def json_value(self, name):
        return self.database.json_value(name)

    def set_path_metadata(self, path, metadata, named_directory=None):
        return self.database.set_path_metadata(path, metadata, named_directory)

    def path_metadata(self, path, named_directory=None):
        return self.database.set_path_metadata(path, named_directory)

    def import_configs(self, environment, config_dict, cont_on_error=False):
        """
        Import config values from a dictionary as given by
        :meth:`Settings.select_configurations`.

        Compared to :meth:`Settings.import_configs` this method (at
        :class:`CapsulEngine` level) also loads the required modules.
        """
        modules = config_dict.get("capsul_engine", {}).get("uses", {})
        for module in modules:
        self.settings.import_configs(environment, config_dict, cont_on_error)

    #
    # Processes and pipelines related methods
    #
    def get_process_instance(self, process_or_id, **kwargs):
        """
        The only official way to get a process instance is to use this method.
        For now, it simply calls self.study_config.get_process_instance
        but it will change in the future.
        """
        instance = self.study_config.get_process_instance(process_or_id, **kwargs)
        return instance

    def get_iteration_pipeline(
        self,
        pipeline_name,
        node_name,
        process_or_id,
        iterative_plugs=None,
        do_not_export=None,
        make_optional=None,
        **kwargs
    ):
        """Create a pipeline with an iteration node iterating the given
        process.

        pipeline_name: str
            pipeline name
        node_name: str
            iteration node name in the pipeline
        process_or_id: process description
            as in :meth:`get_process_instance`
        iterative_plugs: list (optional)
            passed to :meth:`Pipeline.add_iterative_process`
        do_not_export: list
        make_optional: list

        Returns
        -------
        pipeline: :class:`Pipeline` instance
        """
        from capsul.pipeline.pipeline import Pipeline

        pipeline = Pipeline()
        pipeline.name = pipeline_name
        pipeline.set_study_config(get_ref(self.study_config))
        pipeline.add_iterative_process(
            node_name, process_or_id, iterative_plugs, do_not_export, **kwargs
        )
        pipeline.autoexport_nodes_parameters(include_optional=True)
        return pipeline

    def start(self, process, workflow=None, history=True, get_pipeline=False, **kwargs):
        """
        Asynchronously start the execution of a process or pipeline in the
        connected computing environment. Returns an identifier of
        the process execution and can be used to get the status of the
        execution or wait for its termination.

        TODO:
        if history is True, an entry of the process execution is stored in
        the database. The content of this entry is to be defined but it will
        contain the process parameters (to restart the process) and will be
        updated on process termination (for instance to store execution time
        if possible).

        process: Process or Pipeline instance
        workflow: Workflow instance (optional - if already defined before call)
        history: bool (optional)
            TODO: not implemented yet.
        get_pipeline: bool (optional)
            if True, start() will return a tuple (execution_id, pipeline). The
            pipeline is normally the input pipeline (process) if it is actually
            a pipeline. But if the input process is a "single process", it will
            be inserted into a small pipeline for execution. This pipeline will
            be the one actually run, and may be passed to :meth:`wait` to set
            output parameters.

        Returns
        -------
        execution_id: int
            execution identifier (actually a soma-workflow id)
        pipeline: Pipeline instance (optional)
            only returned if get_pipeline is True.
        """
        return run.start(self, process, workflow, history, get_pipeline, **kwargs)

    def connect(self, computing_resource):
        """
        Connect the capsul engine to a computing resource
        """
        self._connected_resource = computing_resource

    def connected_to(self):
        """
        Return the name of the computing resource this capsul engine is
        connected to or None if it is not connected.
        """
        return self._connected_resource

    def disconnect(self):
        """
        Disconnect from a computing resource.
        """
        self._connected_resource = None

    def executions(self):
        """
        List the execution identifiers of all processes that have been started
        but not disposed in the connected computing resource. Raises an
        exception if the computing resource is not connected.
        """
        raise NotImplementedError()

    def dispose(self, execution_id, conditional=False):
        """
        Update the database with the current state of a process execution and
        free the resources used in the computing resource (i.e. remove the
        workflow from SomaWorkflow).

        If ``conditional`` is set to True, then dispose is only done if the
        configuration does not specify to keep succeeded / failed workflows.
        """
        run.dispose(self, execution_id, conditional=conditional)

    def interrupt(self, execution_id):
        """
        Try to stop the execution of a process. Does not wait for the process
        to be terminated.
        """
        return run.interrupt(self, execution_id)

    def wait(self, execution_id, timeout=-1, pipeline=None):
        """
        Wait for the end of a process execution (either normal termination,
        interruption or error).
        """
        return run.wait(self, execution_id, timeout=timeout, pipeline=pipeline)

    def status(self, execution_id):
        """
        Return a simple value with the status of an execution (queued,
        running, terminated, error, etc.)
        """
        return run.status(self, execution_id)

    def detailed_information(self, execution_id):
        """
        Return complete (and possibly big) information about a process
        execution.
        """
        return run.detailed_information(self, execution_id)

    def call(self, process, history=True, *kwargs):
        return run.call(self, process, history=history, **kwargs)

    def check_call(self, process, history=True, **kwargs):
        return run.check_call(self, process, history=history, **kwargs)

    def raise_for_status(self, status, execution_id=None):
        """
        Raise an exception if a process execution failed
        """
        run.raise_for_status(self, status, execution_id)

_populsedb_url_re = re.compile(r"^\w+(\+\w+)?://(.*)")

class AutoDeleteFileName(str):
    """
    A temporary file name that automatically deletes the file when `cleanup()`
    is called.

    This class creates a temporary file using `tempfile.mkstemp()`, stores its
    file descriptor and path, and behaves like a string (inherits from `str`).
    It is useful when you need to pass a file path around but also want to
    ensure the file is properly deleted later.

    :param **kwargs: Keyword arguments passed to `tempfile.mkstemp()` such as
                     `prefix`,`suffix`, or `dir`.

    :returns (AutoDeleteFileName): An instance that acts like a string path
                                   but can clean up its underlying file.
    """

    def __new__(cls, **kwargs):
        """
        Create a temporary file and return an AutoDeleteFileName instance with
        the file path.

        :param **kwargs: Keyword arguments passed to `tempfile.mkstemp()`.

        :returns (AutoDeleteFileName): The string path of the created
                                       temporary file.
        """
        fd, path = tempfile.mkstemp(**kwargs)
        instance = super().__new__(cls, path)
        instance.path = path
        instance.fd = fd

    def cleanup(self):
        """
        Close and delete the temporary file if it exists.

        This method ensures the file descriptor is closed and the file is
        removed from the filesystem. Errors during cleanup are silently
        ignored.
        """

        try:
            os.close(self.fd)

        except OSError:
            pass

        try:

            if os.path.exists(self.path):
                os.remove(self.path)

            pass

def database_factory(database_location):
    """
    Create a DatabaseEngine from its location string. This location can be
    either a sqlite file path (ending with '.sqlite' or ':memory:' for an
    in memory database for testing) or a populse_db URL, or None.
    """
    global _populsedb_url_re

    engine_directory = None

    if database_location is None:
        database_location = AutoDeleteFileName(
            prefix="populse_db_", suffix=".sqlite"
        )
    match = _populsedb_url_re.match(database_location)
    if match:
        path = match.groups(2)
        _, path = osp.splitdrive(path)
        if path.startswith(os.apth.sep):
            engine_directory = osp.abspath(osp.dirname(path))
        populse_db = database_location
    elif database_location.endswith(".sqlite"):
        populse_db = f"sqlite://{database_location}"
        engine_directory = osp.abspath(osp.dirname(database_location))
    elif database_location == ":memory:":
        populse_db = "sqlite://:memory:"
    else:
        raise ValueError("Invalid database location: %s" % database_location)

    engine = PopulseDBEngine(populse_db)

    if isinstance(database_location, AutoDeleteFileName):
        # attach cleanup so it deletes when engine is done
        weakref.finalize(engine, database_location.cleanup)

    if engine_directory:
        engine.set_named_directory("capsul_engine", engine_directory)
    return engine

def capsul_engine(database_location=None, require=None):
    """
    User factory for creating capsul engines.

    If no database_location is given, it will default to an internal (in-
    memory) database with no persistent settings or history values.

    Configuration is read from a dictionary stored in two database entries.
    The first entry has the key 'global_config' (i.e.
    database.json_value('global_config')), it contains the configuration
    values that are shared by all processing engines. The second entry is
    computing_config`. It contains a dictionary with one item per computing
    resource where the key is the resource name and the value is configuration
    values that are specific to this computing resource.

    Before initialization of the CapsulEngine, modules are loaded. The
    list of loaded modules is searched in the 'modules' value in the
    database (i.e. in database.json_value('modules')) ; if no list is
    defined in the database, capsul.module.default_modules is used.
    """
    # if database_location is None:
    # database_location = osp.expanduser('~/.config/capsul/capsul_engine.sqlite')
    database = database_factory(database_location)
    capsul_engine = CapsulEngine(database_location, database, require=require)
    return capsul_engine

configurations = None
activated_modules = set()

def activate_configuration(selected_configurations):
    """
    Activate a selected configuration (set of modules) for runtime.
    """
    global configurations

    configurations = selected_configurations
    modules = configurations.get("capsul_engine", {}).get("uses", {}).keys()
    for m in modules:
        activate_module(m)

def activate_module(module_name):
    """
    Activate a module configuration for runtime. This function is called by
    activate_configuration() and assumes the global variable
    ``capsul.engine.configurations`` is properly setup.
    """
    global activated_modules

    if module_name not in activated_modules:
        activated_modules.add(module_name)
        module = importlib.import_module(module_name)
        check_configurations = getattr(module, "check_configurations", None)
        complete_configurations = getattr(module, "complete_configurations", None)
        if check_configurations:
            error = check_configurations()
            if error:
                if complete_configurations:
                    complete_configurations()
                    error = check_configurations()
            if error:
                raise EnvironmentError(error)
        activate_configurations = getattr(module, "activate_configurations", None)
        if activate_configurations:
            activate_configurations()

import os
import six
import uuid
import json

from capsul.engine.database import DatabaseEngine

class JSONDBEngine(DatabaseEngine):
    '''
    A JSON dictionary implementation of :py:class:`capsul.engine.database.DatabaseEngine`
    '''
    def __init__(self, json_filename):
        if json_filename is not None:
            self.json_filename = osp.normpath(osp.abspath(json_filename))
        else:
            self.json_filename = None
        self.read_json()
    
    def read_json(self):
        if self.json_filename is not None and osp.exists(self.json_filename):
            self.json_dict = json.load(open(self.json_filename))
            self.modified = False
        else:
            self.json_dict = {}
            self.modified = True
            
    def commit(self):
        if self.modified and self.json_filename is not None:
            parent = osp.dirname(self.json_filename)
            if not osp.exists(parent):
                os.makedirs(parent)
            json.dump(self.json_dict, open(self.json_filename, 'w'), indent=2)

    def rollback(self):
    
        if path:
            path = osp.normpath(osp.abspath(path))
            self.json_dict.setdefault('named_directory', {})[name] = {'name': name,
                                                                      'path': path}
        else:
            named_directory = self.json_dict.get('named_directory')
            if named_directory is not None:
                named_directory.pop(name, None)
                self.modified = True

        return self.json_dict.get('named_directory', {}).get(name, {}).get('path')
    
        return self.json_dict.get('named_directory', {}).values()
    
        self.json_dict.setdefault('json_value', {})[name] = json_value
        self.modified = True

        return self.json_dict.get('json_value', {}).get(name)
    
        metadata = self.check_path_metadata(path, metadata, named_directory)
        path = metadata['path']
        named_directory = metadata['named_directory']
        self.json_dict.setdefault('path_metadata', {})[
            (named_directory, path)] = metadata
            
        named_directory, path = self.check_path(path, named_directory)
        return self.json_dict.get('path_metadata', {}).get((named_directory, path))

from populse_db.storage import Storage

schemas = [
    {
        "version": "1.0.0",
        "schema": {
            "named_directory": [
                {
                    "name": [str, {"primary_key": True}],
                    "path": str,
                }
            ],
            "json_value": [
                {
                    "json_dict": dict,
                }
            ],
            "path_metadata": [
                {
                    "path": [str, {"primary_key": True}],
                    "named_directory": str,
                }
            ],
            "metadata": [
                {
                    "subject": str,
                    "time_point": str,
                    "history": list[str],  # contains a list of execution_id
                }
            ],
        },
    },
]

class PopulseDBEngine(DatabaseEngine):
    def __init__(self, database_engine):
        self.storage = Storage(database_engine)
        with self.storage.schema() as schema:
            schema.add_schema("capsul.engine.database_populse")

        self.close()

    def close(self):
        self.storage = None

        if path:
        with self.storage.data(write=True) as db:
            if path:
                db.named_directory[name] = path
            else:
                del db.named_directory[name]

        with self.storage.data() as db:
            return db.named_directory[name].path.get()

            for row in db.named_directory.search(fields=["name"], as_list=True):
                yield row[0]

            db["json_value"][name].json_dict = json_value

            return db["json_value"][name].json_dict.get()

    def set_path_metadata(self, path, metadata):
        named_directory = metadata.get("named_directory")
        if named_directory:
            base_path = self.named_directory("capsul_engine")
            if base_path:
                if not path.startswith(named_directory):
                    raise ValueError(
                        'Path "%s" is defined as relative to named directory %s but it does not start with "%s"'
                        % (path, named_directory, base_path)
                    )
                path = path[len(base_path) + 1 :]
            else:
                if osp.isabs(path):
                        'Cannot determine relative path for "%s" because its base named directory "%s" is unknown'
                        % (path, named_directory)
                    )
        else:
            if osp.isabs(path):
                for nd in self.named_directories():
                    if path.startswith(nd.path):
                        named_directory = nd.name()
                        path = path[len(nd.path) + 1]
                        break
                else:
                    named_directory = None
            else:
                # capsul_engine is the default named directory for relative paths
                named_directory = "capsul_engine"

        doc = metadata.copy()
        doc["path"] = path
            doc["named_directory"] = named_directory
        with self.db as dbs:
            dbs.add_document("path_metadata", doc)

    def path_metadata(self, path):
        if osp.isabs(path):
            for nd in self.named_directories():
                if path.startswith(nd.path):
                    path = path[len(nd.path) + 1 :]
                    break
            return dbs.get_document("path_metadata", path)

class DatabaseEngine:
    '''
    A :py:class:`DatabaseEngine` is the base class for all engines 
    allowing to store, retrieve and search metadata associated with
    a key that can be either a string or a path (i.e. a file or directory
    name).
    
    To instantiate a :py:class:`DatabaseEngine` one must use the factory 
    To date, two concrete :py:class:`DatabaseEngine` implementations exist:

    - :py:class:`capsul.engine.database_json.JSONDBEngine`
    - :py:class:`capsul.engine.database_populse.PopulseDBEngine`
    
    '''
    
    def check_path_metadata(self, path, metadata, named_directory=None):
        named_directory = metadata.get('named_directory')
        
        doc['path'] = path
        doc['named_directory'] = named_directory
        return doc

    def check_path(self, path, named_directory=None):
        '''
        Find a pair (named_directory, path) given a path and eventually a
        named_directory.
        
        If named_directory is not given, path must be absolute or a 
        ValueError is raised. Then, either the corresponding named 
        directory is found or 'absolute' is used.
        
        If name_directory is given, the path must be relative (unless
        named_directory == 'absolute') or begin with the path of the 
        named directory.
        '''
        if named_directory is None:
                    base_path = nd['path']
                    if path.startswith(base_path):
                        named_directory = nd['name']
                        path = path[len(base_path)+1:]
                        break
                else:
                    named_directory = 'absolute'
            else:
                raise ValueError('Cannot determine base named directory for relative path "%s"' % path)
        else:
            if named_directory == 'absolute':
                if not osp.isabs(path):
                    raise ValueError('Using "absolute" named directory requires an absolute path, not "%s"' % path)
            else:
                base_path = self.named_directory(named_directory)
                if base_path is None:
                    raise ValueError('Unknown named directory "%s"' % named_directory)
                    if not path.startswith(base_path):
                        raise ValueError('Path "%s" is defined as relative to named directory %s but it does not start with "%s"' % (path, named_directory, base_path))
                    path = path[len(base_path)+1:]
        return (named_directory, path)
    
        '''
        Store a json value and associate it with a key given in "name".
        The value can be retrieved with method json_value().
        
        @param name: unique key used to identify and retrieve the value
        @type name: C{string}
        @param json_value: a value to store in the database
        @type  name: any JSON compatible value
        '''

        '''
        Retrieve a value previously stored with set_json_value()
        '''
        
        '''
        Associate an absolute path to a generic name. This allow to always
        use a location independent name for a directory such as 'spm_template'
        and to customize the real absolute path in the configuration. These 
        named directories are used when setting/retrieving path metadata with 
        set_path_metadata() and path_metadata().
        '''
    
        '''
        Return the absolute path of a named directory.
        '''
    
        '''
        List the names of all named directories. This method may return any
        iterable value (list, generator, etc.)
        '''
    
        '''
        Set metadata associated to a path. The metadata are associated to the
        result of self.check_path(path, named_directory). metadata can be any
        JSON serializable value.
        '''
    
        '''
        Retrieve metadata associated with a path.
        '''

'''
Attributes completion config module

'''

from capsul.attributes.attributes_factory import AttributesFactory
from capsul.attributes.attributes_schema import AttributesSchema, \
    ProcessAttributes
from capsul.attributes.completion_engine \
    import ProcessCompletionEngineFactory, PathCompletionEngineFactory
from functools import partial

def init_settings(capsul_engine):
    default_paths = ['capsul.attributes.completion_engine_factory']

    init_att = {
        'attributes_schema_paths': default_paths,
        'attributes_schemas': {},
        'process_completion': 'builtin',
        capsul_engine.settings.config_id_field: 'attributes',
    }

    with capsul_engine.settings as session:
        session.ensure_module_fields('attributes',
            [dict(name='attributes_schema_paths',
                  type='list[str]',
                  description='attributes shchemas modules names'),
             dict(name='attributes_schemas',
                  type='dict',
                  description='attributes shchemas names'),
             dict(name='process_completion',
                  type='str',
                  description='process completion model name'),
             dict(name='path_completion',
                  description='path completion model name'),
            ])
        config = session.config('attributes', 'global')
        if not config:
            session.new_config('attributes', 'global', init_att)

    if not hasattr(capsul_engine, '_modules_data'):
        capsul_engine._modules_data = {}
    data = capsul_engine._modules_data.setdefault('attributes', {})
    factory = AttributesFactory()
    data['attributes_factory'] = factory

    factory.class_types['schema'] = AttributesSchema
    factory.class_types['process_completion'] \
        = ProcessCompletionEngineFactory
    factory.class_types['path_completion'] \
        = PathCompletionEngineFactory
    factory.class_types['process_attributes'] \
        = ProcessAttributes

    factory.module_path = default_paths
    capsul_engine.settings.module_notifiers.setdefault(
        'capsul.engine.module.attributes', []).append(
            partial(_sync_attributes_factory, weakref.proxy(capsul_engine)))

    # link with StudyConfig
    if hasattr(capsul_engine, 'study_config') \
            and 'AttributesConfig' not in capsul_engine.study_config.modules:
        scmod = capsul_engine.study_config.load_module('AttributesConfig', {})
        scmod.initialize_module()
        scmod.initialize_callbacks()

#def check_configurations():
    #'''
    #Checks if the activated configuration is valid to use BrainVisa and returns
    #an error message if there is an error or None if everything is good.
    #'''
    #return None

#def complete_configurations():
    #'''
    #Try to automatically set or complete the capsul.engine.configurations for
    #the attributes module.
    #'''
    #config = capsul.engine.configurations
    #config = config.setdefault('attributes', {})
    #attributes_schema_paths = config.get('attributes_schema_paths', None)
    #if attributes_schema_paths is None:
        #config['attributes_schema_paths'] \
            #= ['capsul.attributes.completion_engine_factory']
    #attributes_schemas = config.get('attributes_schemas', None)
    #if attributes_schemas is None:
        #config['attributes_schemas'] = {}

def _sync_attributes_factory(capsul_engine, param=None, value=None):
    factory = capsul_engine._modules_data['attributes']['attributes_factory']
        if config:
            factory.module_path = config.attributes_schema_paths

def edition_widget(engine, environment, config_id='attributes'):
    ''' Edition GUI for attributes config - see
    :class:`~capsul.qt_gui.widgets.settings_editor.SettingsEditor`
    '''
    from soma.qt_gui.controller_widget import ScrollControllerWidget
    from soma.controller import Controller
    import types
    import traits.api as traits

    def validate_config(widget):
        widget.update_controller()
        controller = widget.controller_widget.controller
        with widget.engine.settings as session:
            conf = session.config(config_id, widget.environment)
            values = {'config_id': config_id}
            values['attributes_schema_paths'] \
                = controller.attributes_schema_paths
            values['attributes_schemas'] = controller.attributes_schemas
            values['process_completion'] = controller.process_completion
            if controller.process_completion is traits.Undefined:
                values['process_completion'] = None
            values['path_completion'] = controller.path_completion
            if controller.path_completion is traits.Undefined:
                values['path_completion'] = None
            if conf is None:
                session.new_config(config_id, widget.environment, values)
            else:
                for k in ('attributes_schema_paths', 'attributes_schemas',
                          'process_completion', 'path_completion'):
                    setattr(conf, k, values[k])

    controller = Controller()
    controller.add_trait(
        'attributes_schema_paths',
        traits.List(traits.Str(), desc='attributes shchemas modules names'))
        'attributes_schemas',
        traits.Dict(str, str, desc='attributes shchemas names'))
        'process_completion',
        traits.Str(desc='process completion model name'))
        'path_completion',
        traits.Str(desc='path completion model name',
                   optional=True))

    conf = engine.settings.select_configurations(
        environment, {'attributes': 'any'})
    if conf:
        aconf = conf.get(
            'capsul.engine.module.attributes', {})
        controller.attributes_schema_paths = aconf.get(
            'attributes_schema_paths', [])
        controller.attributes_schemas = aconf.get(
            'attributes_schemas', {})
        controller.process_completion = aconf.get(
            'process_completion', 'builtin')
        controller.path_completion = aconf.get(
            'path_completion', traits.Undefined)

    widget = ScrollControllerWidget(controller, live=True)
    widget.engine = engine
    widget.environment = environment
    widget.accept = types.MethodType(validate_config, widget)

    return widget

"""
Configuration module which links with `Axon <http://brainvisa.info/axon/user_doc>`_
"""

import os
import six
import capsul.engine

    with capsul_engine.settings as settings:
        settings.ensure_module_fields(
            "axon",
            [
                dict(
                    name="shared_directory",
                    type="str",
                    description="Directory where BrainVisa shared data is installed",
                ),
                dict(
                    name="user_level",
                    type="int",
                    description="0: basic, 1: advanced, 2: expert, or more. "
                    "used to display or hide some advanced features or "
                    "process parameters that would be confusing to a novice "
                    "user",
                ),
            ],
        )

        config = session.config("axon", "global")
            from soma import config as soma_config

            shared_dir = soma_config.BRAINVISA_SHARE

            values = {
                capsul_engine.settings.config_id_field: "axon",
                "shared_directory": shared_dir,
                "user_level": 0,
            }
            session.new_config("axon", "global", values)

    if hasattr(capsul_engine, "study_config"):
        if "BrainVISAConfig" not in capsul_engine.study_config.modules:
            scmod = capsul_engine.study_config.load_module("BrainVISAConfig", {})
            scmod.initialize_module()
            scmod.initialize_callbacks()
        else:
            scmod = capsul_engine.study_config.modules["BrainVISAConfig"]
            scmod.sync_to_engine()

def check_configurations():
    """
    Checks if the activated configuration is valid to use BrainVisa and returns
    an error message if there is an error or None if everything is good.
    """
    shared_dir = capsul.engine.configurations.get("axon", {}).get(
        "shared_directory", ""
    )
    if not shared_dir:
        return "Axon shared_directory is not found"
    return None

def check_notably_invalid_config(conf):
    """
    Checks if the given module config is obviously invalid, for instance if a mandatory path is not filled

    Returns
    -------
    invalid: list
        list of invalid config keys
    """
    invalid = []
    for k in ("shared_directory",):
        if conf.get(k) is None:
            invalid.append(k)
    return invalid

def complete_configurations():
    """
    Try to automatically set or complete the capsul.engine.configurations for
    Axon.
    """
    config = capsul.engine.configurations
    config = config.setdefault("axon", {})
    shared_dir = config.get("shared_directory", None)
    if shared_dir is None:
        from soma import config as soma_config

        shared_dir = soma_config.BRAINVISA_SHARE
        if shared_dir:
            config["shared_directory"] = shared_dir

def edition_widget(engine, environment, config_id="axon"):
    """Edition GUI for axon config - see
    """

            values = {"config_id": config_id, "user_level": controller.user_level}
            if controller.shared_directory in (None, traits.Undefined, ""):
                values["shared_directory"] = None
            else:
                values["shared_directory"] = controller.shared_directory
            else:
                for k in ("shared_directory", "user_level"):

        "shared_directory",
        traits.Directory(desc="Directory where BrainVisa " "shared data is installed"),
    )
        "user_level",
        traits.Int(
            desc="0: basic, 1: advanced, 2: expert, or more. "
            "used to display or hide some advanced features or "
            "process parameters that would be confusing to a novice "
            "user"
        ),
    )

    conf = engine.settings.select_configurations(environment, {"axon": "any"})
    if conf:
        controller.shared_directory = conf.get("capsul.engine.module.axon", {}).get(
            "shared_directory", traits.Undefined
        )
        controller.user_level = conf.get("capsul.engine.module.axon", {}).get(
            "user_level", 0
        )

"""
Config module for :mod:`File Organization models (FOMs) <capsul.attributes.fom_completion_engine>`

Classes
=======
:class:`FomConfig`
------------------
"""

import os
import six
from soma.fom import AttributesToPaths, PathToAttributes
from soma.application import Application

        session.ensure_module_fields(
            "fom",
            [
                dict(name="input_fom", type="str", description="input FOM"),
                dict(name="output_fom", type="str", description="output FOM"),
                dict(name="shared_fom", type="str", description="shared data FOM"),
                dict(
                    name="volumes_format",
                    description="Format used for volumes",
                ),
                dict(
                    name="meshes_format",
                    description="Format used for meshes",
                ),
                dict(
                    name="auto_fom",
                    type="bool",
                    description="Look in all FOMs when a process is not found "
                    "(in addition to the standard share/foms). Note that "
                    "auto_fom looks for the first FOM matching the process to "
                    "get completion for, and does not handle ambiguities. "
                    "Moreover it brings an overhead (typically 6-7 seconds) the "
                    "first time it is used since it has to parse all available "
                    "FOMs.",
                ),
                dict(
                    name="fom_path",
                    type="list[str]",
                    description="list of additional directories where to look "
                    "for FOMs",
                ),
                # FIXME: until directories are included in another config module
                dict(
                    name="input_directory",
                    description="input study data directory",
                ),
                dict(
                    name="output_directory",
                    description="output study data directory",
                ),
            ],
        )

    capsul_engine.load_module("capsul.engine.module.axon")
    capsul_engine.load_module("capsul.engine.module.spm")
    capsul_engine.load_module("capsul.engine.module.attributes")

        config = session.config("fom", "global")
            values = {
                capsul_engine.settings.config_id_field: "fom",
                "auto_fom": True,
                "fom_path": [],
            }
            session.new_config("fom", "global", values)

    if not hasattr(capsul_engine, "_modules_data"):
    store = capsul_engine._modules_data.setdefault("fom", {})
    store["foms"] = {}
    store["all_foms"] = SortedDictionary()
    store["fom_atp"] = {"all": {}}
    store["fom_pta"] = {"all": {}}

    capsul_engine.settings.module_notifiers["capsul.engine.module.fom"] = [
        partial(fom_config_updated, weakref.proxy(capsul_engine), "global")
    ]
        "capsul.engine.module.axon", []
    ).append(partial(config_updated, weakref.proxy(capsul_engine), "global"))
        "capsul.engine.module.spm", []

    if (
        hasattr(capsul_engine, "study_config")
        and "FomConfig" not in capsul_engine.study_config.modules
    ):
        scmod = capsul_engine.study_config.load_module("FomConfig", {})

    update_fom(capsul_engine, "global")

def config_dependencies(config):
    return {
        "axon": "any",
        #'spm': 'any',
        "attributes": "any",
    }

def config_updated(capsul_engine, environment, param=None, value=None):
    if param in (None, "directory", "shared_directory"):
        update_fom(capsul_engine, environment)

def spm_config_updated(capsul_engine, environment, param=None, value=None):
    if param in (None, "directory"):

def fom_config_updated(capsul_engine, environment="global", param=None, value=None):
    if param in ("volumes_format", "meshes_format"):
        update_formats(capsul_engine, environment)
        return

    if param in ("fom_path",):
        reset_foms(capsul_engine, environment)
        return

    # otherwise use update_fom()
    update_fom(capsul_engine, environment, param, value)

def update_fom(capsul_engine, environment="global", param=None, value=None):
    """Load configured FOMs and create FOM completion data"""
    # print('***update_fom ***')
        config = session.config("fom", environment)
        if config is None:
            return

        soma_app = Application("capsul", plugin_modules=["soma.fom"])
        if "soma.fom" not in soma_app.loaded_plugin_modules:
            # WARNING: this is unsafe, may erase configured things, and
            # probably not thread-safe.
            soma_app.initialize()

        if config.fom_path:
            fom_path = [
                p for p in config.fom_path if p not in soma_app.fom_path
            ] + soma_app.fom_path
        else:
            fom_path = list(soma_app.fom_path)

        soma_app.fom_manager.paths = fom_path
        soma_app.fom_manager.fom_files()

        store = capsul_engine._modules_data["fom"]
        if config.auto_fom and len(store["all_foms"]) <= 3:
            for schema in soma_app.fom_manager.fom_files():
                if schema not in store["all_foms"]:
                    store["all_foms"][schema] = None  # not loaded yet.

        foms = (
            ("input", config.input_fom),
            ("output", config.output_fom),
            ("shared", config.shared_fom),
        )
        for fom_type, fom_filename in foms:
            if fom_filename not in ("", None, traits.Undefined):
                fom = store["all_foms"].get(fom_filename)
                if fom is None:
                    fom, atp, pta = load_fom(
                        capsul_engine, fom_filename, config, session, environment
                    )
                else:
                    atp = store["fom_atp"]["all"][fom_filename]
                    pta = store["fom_pta"]["all"][fom_filename]
                store["foms"][fom_type] = fom
                store["fom_atp"][fom_type] = atp
                store["fom_pta"][fom_type] = pta

        # update directories
        directories = {}
        spm = session.config("spm", environment)
        if spm:
            directories["spm"] = spm.directory
        axon = session.config("axon", environment)
        if axon:
            directories["shared"] = axon.shared_directory
        directories["input"] = config.input_directory
        directories["output"] = config.output_directory

        for atp in store["fom_atp"]["all"].values():
            atp.directories = directories

        # backward compatibility for StudyConfig
        capsul_engine.study_config.modules_data.foms = store["foms"]
        capsul_engine.study_config.modules_data.all_foms = store["all_foms"]
        capsul_engine.study_config.modules_data.fom_atp = store["fom_atp"]
        capsul_engine.study_config.modules_data.fom_pta = store["fom_pta"]

def update_formats(capsul_engine, environment):
            return

        if spm:
        if axon:

        with config._storage.data() as data:
            fields = data[config._collection].keys()
        formats = tuple(
            getattr(config, key)
            for key in fields
            if key.endswith("_format") and getattr(config, key) is not None
        )

        for schema, fom in store["all_foms"].items():
                continue

            atp = AttributesToPaths(
                fom,
                selection={},
                directories=directories,
                preferred_formats=set((formats)),
            )
            old_atp = store["fom_atp"]["all"].get(schema)
            store["fom_atp"]["all"][schema] = atp
            if old_atp is not None:
                for t in ("input", "output", "shared"):
                    if store["fom_atp"].get(t) is old_atp:
                        store["fom_atp"][t] = atp

def load_fom(capsul_engine, schema, config, session, environment="global"):
    # print('=== load fom', schema, '===')
    # import time
    # t0 = time.time()
    soma_app = Application("capsul", plugin_modules=["soma.fom"])
    if "soma.fom" not in soma_app.loaded_plugin_modules:
        # WARNING: this is unsafe, may erase configured things, and
        # probably not thread-safe.
        soma_app.initialize()
    old_fom_path = soma_app.fom_path
    if config.fom_path:
        soma_app.fom_path = [
            six.ensure_str(p) for p in config.fom_path if p not in soma_app.fom_path
        ] + soma_app.fom_path
    else:
        soma_app.fom_path = list(soma_app.fom_path)
    try:
        fom = soma_app.fom_manager.load_foms(schema)
    except KeyError:
        return None, None, None
    soma_app.fom_path = old_fom_path
    store = capsul_engine._modules_data["fom"]
    store["all_foms"][schema] = fom

    # Create FOM completion data
    with config._storage.data() as data:
        fields = data[config._collection].keys()
    formats = tuple(
        getattr(config, key)
        for key in fields
        if key.endswith("_format") and getattr(config, key) is not None
    )

    directories = {}
    spm = session.config("spm", environment)
    if spm:
        directories["spm"] = spm.directory
    axon = session.config("axon", environment)
    if axon:
        directories["shared"] = axon.shared_directory
    directories["input"] = config.input_directory
    directories["output"] = config.output_directory

    atp = AttributesToPaths(
        fom, selection={}, directories=directories, preferred_formats=set((formats))
    )
    store["fom_atp"]["all"][schema] = atp
    pta = PathToAttributes(fom, selection={})
    store["fom_pta"]["all"][schema] = pta
    # print('   load fom done:', time.time() - t0, 's')
    return fom, atp, pta

def reset_foms(capsul_engine, environment):
    soma_app.fom_manager.clear_cache()
    update_fom(capsul_engine, environment)

def edition_widget(engine, environment, config_id="fom"):
    """Edition GUI for FOM config - see
    """

            values = {"config_id": config_id}
            for k in (
                "input_fom",
                "output_fom",
                "shared_fom",
                "volumes_format",
                "meshes_format",
                "auto_fom",
                "fom_path",
                "input_directory",
                "output_directory",
            ):
                value = getattr(controller, k)
                if value is traits.Undefined:
                    if k in ("fom_path",):
                        value = []
                    else:
                        value = None
                values[k] = value
            else:
                for k, value in values.items():
                    if k == "config_id":
                        continue

        "input_fom", traits.Str(traits.Undefined, output=False, desc="input FOM")
    )
        "output_fom", traits.Str(traits.Undefined, output=False, desc="output FOM")
    )
        "shared_fom", traits.Str(traits.Undefined, output=False, desc="shared data FOM")
    )
        "volumes_format",
        traits.Str(traits.Undefined, output=False, desc="Format used for volumes"),
    )
        "meshes_format",
        traits.Str(traits.Undefined, output=False, desc="Format used for meshes"),
    )
        "auto_fom",
        traits.Bool(
            True,
            output=False,
            desc="Look in all FOMs when a process is not found (in "
            "addition to the standard share/foms). Note that auto_fom "
            "looks for the first FOM matching the process to get "
            "completion for, and does not handle ambiguities. Moreover "
            "it brings an overhead (typically 6-7 seconds) the first "
            "time it is used since it has to parse all available FOMs.",
        ),
    )
        "fom_path",
        traits.List(
            traits.Directory(output=False),
            desc="list of additional directories where to look for FOMs",
        ),
    )
    # FIXME: until directories are included in another config module
        "input_directory",
        traits.Directory(
            traits.Undefined, output=False, desc="input study data directory"
        ),
    )
        "output_directory",
            traits.Undefined, output=False, desc="output study data directory"
        ),
    )

    conf = engine.settings.select_configurations(environment, {"fom": "any"})
    if conf:
        fconf = conf.get("capsul.engine.module.fom", {})
        controller.input_fom = fconf.get("input_fom", traits.Undefined)
        controller.output_fom = fconf.get("output_fom", traits.Undefined)
        controller.shared_fom = fconf.get("shared_fom", traits.Undefined)
        controller.volumes_format = fconf.get("volumes_format", traits.Undefined)
        controller.meshes_format = fconf.get("meshes_format", traits.Undefined)
        controller.auto_fom = fconf.get("auto_fom", traits.Undefined)
        controller.fom_path = fconf.get("fom_path", traits.Undefined)
        controller.input_directory = fconf.get("input_directory", traits.Undefined)
        controller.output_directory = fconf.get("output_directory", traits.Undefined)

import os
from soma.path import find_in_path
from capsul import engine
import six

            "fsl",
            [
                dict(
                    name="directory",
                    description="Directory where FSL is installed",
                ),
                dict(name="config", type="str", description="path of fsl.sh file"),
                dict(
                    name="prefix",
                    description="Prefix to add to FSL commands",
                ),
            ],
        )

        # init a single config
        config = settings.config("fsl", "global")
            settings.new_config(
                "fsl", "global", {capsul_engine.settings.config_id_field: "fsl"}
            )

    """
    Checks if the activated configuration is valid to run FSL and returns an
    error message if there is an error or None if everything is good.
    """
    fsl_prefix = capsul.engine.configurations.get("capsul.engine.module.fsl", {}).get(
        "prefix", ""
    )
    fsl_config = capsul.engine.configurations.get("capsul.engine.module.fsl", {}).get(
        "config"
    )
    if not fsl_config:
        if not find_in_path("%sbet" % fsl_prefix):
            return 'FSL command "%sbet" cannot be found in PATH' % fsl_prefix
    else:
        if fsl_prefix:
            return "FSL configuration must either use config or prefix but not both"
        if not osp.exists(fsl_config):
            return 'File "%s" does not exist' % fsl_config
        if not fsl_config.endswith("fsl.sh"):
            return 'File "%s" is not a path to fsl.sh script' % fsl_config

    """

    Returns
    -------
    """
    for k in ("directory", "config"):

    """
    Try to automatically set or complete the capsul.engine.configurations for FSL.
    """
    config = config.setdefault("capsul.engine.module.fsl", {})
    fsl_dir = config.get("directory", os.environ.get("FSLDIR"))
    fsl_prefix = config.get("prefix", "")
    if fsl_dir and not fsl_prefix:
        # Try to set fsl_config from FSLDIR environment variable
        fsl_config = "%s/etc/fslconf/fsl.sh" % fsl_dir
        if osp.exists(fsl_config):
            config["config"] = fsl_config
    elif not fsl_prefix:
        # Try to set fsl_prefix by searching fsl-*bet in PATH
        bet = find_in_path("fsl*-bet")
        if bet:
            config["prefix"] = os.path.basename(bet)[:-3]

def activate_configurations():
    """
    Activate the FSL module (set env variables) from the global configurations,
    in order to use them via :mod:`capsul.in_context.fsl` functions
    """
    conf = engine.configurations.get("capsul.engine.module.fsl", {})
    fsl_dir = conf.get("directory")
    if fsl_dir:
        os.environ["FSLDIR"] = six.ensure_str(fsl_dir)
    elif "FSLDIR" in os.environ:
        del os.environ["FSLDIR"]
    fsl_prefix = conf.get("prefix")
    if fsl_prefix:
        os.environ["FSL_PREFIX"] = six.ensure_str(fsl_prefix)
    elif "FSL_PREFIX" in os.environ:
        del os.environ["FSL_PREFIX"]
    fsl_conf = conf.get("config")
    if fsl_conf:
        os.environ["FSL_CONFIG"] = six.ensure_str(fsl_conf)
    elif "FSL_CONFIG" in os.environ:
        del os.environ["FSL_CONFIG"]

def edition_widget(engine, environment, config_id="fsl"):
    """Edition GUI for FSL config - see
    """

            for k in ("directory", "config", "prefix"):
                    value = None
            else:
                        continue

        "directory",
        traits.Directory(traits.Undefined, desc="Directory where FSL is installed"),
    )
        "config",
        traits.File(
            traits.Undefined, output=False, desc="Parameter to specify the fsl.sh path"
        ),
    )
        "prefix", traits.String(traits.Undefined, desc="Prefix to add to FSL commands")
    )

    conf = engine.settings.select_configurations(environment, {"fsl": "any"})
    if conf:
        fconf = conf.get("capsul.engine.module.fsl", {})
        controller.directory = fconf.get("directory", traits.Undefined)
        controller.config = fconf.get("config", traits.Undefined)
        controller.prefix = fconf.get("prefix", traits.Undefined)

import os

# from soma.controller import Controller
# from traits.api import File, Undefined, Instance

            "matlab",
            [
                dict(
                    name="executable",
                    description="Full path of the matlab executable",
                ),
                dict(
                    name="mcr_directory",
                    description="Full path of the matlab MCR directory, for use "
                    "with standalone binaries",
                ),
            ],
        )

    """
    Check if the activated configuration is valid for Matlab and return
    """
    matlab_executable = capsul.engine.configurations.get(
        "capsul.engine.module.matlab", {}
    ).get("executable")
    mcr = capsul.engine.configurations.get("capsul.engine.module.matlab", {}).get(
        "mcr_directory"
    )
    if not matlab_executable and not mcr:
        return "both matlab.executable and matlab.mcr_directory are not defined"
    if matlab_executable and not os.path.exists(matlab_executable):
        return (
            'Matlab executable is defined as "%s" but this path does not exist'
            % matlab_executable
        )

    """

    Returns
    -------
    """
    for k in ("executable", "mcr_directory"):
    if len(invalid) == 1:
        # if one of the paths is filled, then it should be OK.
        # however it may depend on uses (SPM standalone needs mcr_directory,
        # for instance)
        return []

def edition_widget(engine, environment, config_id="matlab"):
    """Edition GUI for matlab config - see
    """

            if controller.executable in (None, traits.Undefined, ""):
                values["executable"] = None
            else:
                values["executable"] = controller.executable
            if controller.mcr_directory in (None, traits.Undefined, ""):
                values["mcr_directory"] = None
            else:
                values["mcr_directory"] = controller.mcr_directory
            else:
                for k in (
                    "executable",
                    "mcr_directory",
                ):
                    # don't check invalid files: they may be valid on a remote
                    # server
                    # if (k == 'mcr_directory' and
                    # values[k] and
                    # not os.path.isdir(values[k])):
                    # raise NotADirectoryError('\nMatlab mcr_directory '
                    #'was not updated:\n{} is '
                    #'not existing!'.format(
                    # values[k]))
                    # elif (k == 'executable' and
                    # not os.path.isfile(values[k])):
                    # raise FileNotFoundError('\nMatlab executable '
                    # else:

        "executable", traits.File(desc="Full path of the matlab executable")
    )
        "mcr_directory",
            desc="Full path of the matlab MCR "
            "directory, or use with standalone "
            "binaries"
        ),
    )

    conf = engine.settings.select_configurations(environment, {"matlab": "any"})
    if conf:
        controller.executable = conf.get("capsul.engine.module.matlab", {}).get(
            "executable", traits.Undefined
        )
        controller.mcr_directory = conf.get("capsul.engine.module.matlab", {}).get(
            "mcr_directory", traits.Undefined
        )

''' Python configuration module for CAPSUL

This config module allows the customization of python executable and python path in process execution. It can (as every config module) assign specific config values for different environments (computing resources, typically).

Python configuration is slightly different from other config modules in the way that it cannot be handled during execution inside a python library: python executable and modules path have to be setup before starting python and loading modules. So the config here sometimes has to be prepared from client side and hard-coded in the job to run.

For this reason, what we call here "python jobs" have a special handling. "python jobs" are :class:`~capsul.process.process.Process` classes defining a :meth:`~capsul.process.process.Process._run_process` method, and not :meth:`~capsul.process.process.Process.get_commandline`. Processing are thus python functions or methods, and need the capsul library to run.

Python jobs are handled in workflow building (:mod:`capsul.pipeline.pipeline_workflow`), and jobs on engine side should not have to bother about it.

The python config module is not mandatory: if no specific configuration is needed, jobs are run using the python command from the path, following the client ``sys.executable`` short name (if the client runs ``/usr/bin/python3``, the engine will try to use ``python3`` from the ``PATH``.

The python config module is used optionally (if there is a config, it is used, otherwise no error is produced), and automatically for all jobs: no need to declare it in jobs :meth:`~capsul.process.process.Process.requirements` method.

Inside process execution, the module is otherwise handled like any other.
'''

import os
import sys

        settings.ensure_module_fields('python',
        [dict(name='executable',
              type='str',
              description='Full path of the python executable'),
        dict(name='path',
              type='list[str]',
              description='paths to prepend to sys.path'),
         ])

    '''
    Activate the python module from the global configurations
    '''
    py_conf = capsul.engine.configurations.get('capsul.engine.module.python')
    if py_conf:
        py_path = py_conf.get('path')
        if py_path:
            sys.path = py_path + [p for p in sys.path if p not in py_path]

    '''

    Returns
    -------
    '''
    for k in ('executable', ):

def edition_widget(engine, environment, config_id='python'):
    ''' Edition GUI for python config - see
    '''
    from soma.qt_gui.qt_backend import Qt

            values = {'config_id': config_id,
                      'path': controller.path}
            if controller.executable in (None,
                                                traits.Undefined, ''):
                values['executable'] = None
            else:
                values['executable'] = controller.executable
            else:
                for k in ('path', 'executable'):

    controller.add_trait('executable',
                         traits.Str(desc='Full path of the python executable'))
    controller.add_trait('path',
                         traits.List(traits.Directory(), [],
                                     desc='paths to prepend to sys.path'))

        environment, {'python': 'any'})
    if conf:
        controller.executable = conf.get(
            'capsul.engine.module.python', {}).get('executable',
                                                   traits.Undefined)
        controller.path = conf.get(
            'capsul.engine.module.python', {}).get('path', [])

import os
import six

# import glob
# import os.path as osp
# import weakref
##import subprocess # Only in case of matlab call (auto_configuration func)

# from traits.api import Directory, Undefined, Instance, String, Bool

# from . import matlab

            "spm",
            [
                dict(
                    description="Directory where SPM is installed",
                ),
                dict(
                    name="version",
                    description="Version of SPM release (8 or 12)",
                ),
                dict(
                    name="standalone",
                    description="If this parameter is set to True, use the "
                    "standalone SPM version, otherwise use Matlab.",
                ),
            ],
        )

    return {"matlab": "any"}

# def set_environ(config, environ):
# spm_config = config.get('spm', {})
# use = spm_config.get('use')
# if  use is True or (use is None and 'directory' in spm_config):
# error_message = check_environ(environ)
# if error_message:
# complete_environ(config, environ)

# raise EnvironmentError(error_message)

# def check_environ(environ):
#'''
# Check if the configuration is valid to run SPM and returns an error
# message if there is an error or None if everything is good.
#'''
# if not environ.get('SPM_DIRECTORY'):
# return 'SPM directory is not defined'
# if not environ.get('SPM_VERSION'):
# return 'SPM version is not defined (maybe %s is not a valid SPM directory)' % environ['SPM_DIRECTORY']
# if not environ.get('SPM_STANDALONE'):
# return 'No selection of SPM installation type : Standalone or Matlab'
# if not osp.isdir(environ['SPM_DIRECTORY']):
# return 'No valid SPM directory: %s' % environ['SPM_DIRECTORY']
# if environ['SPM_STANDALONE'] != 'yes':
# matlab_error = matlab.check_environ(environ)
# if matlab_error:
# return 'Matlab configuration must be valid for SPM: ' + matlab_error
# return None

# def complete_environ(config, environ):
#'''
# Try to automatically complete environment for SPM
#'''
# spm_directory = config.get('spm', {}).get('directory')
# if spm_directory:
# environ['SPM_DIRECTORY'] = spm_directory
# mcr = glob.glob(osp.join(spm_directory, 'spm*_mcr'))
# if mcr:
# fileName = osp.basename(mcr[0])
# inc = 1

# while fileName[fileName.find('spm') + 3:
# fileName.find('spm') + 3 + inc].isdigit():
# environ['SPM_VERSION'] = fileName[fileName.find('spm') + 3:
# fileName.find('spm') + 3
# + inc]
# inc+=1

# environ['SPM_STANDALONE'] = 'yes'

# else:
# environ['SPM_STANDALONE'] = 'no'
## determine SPM version (currently 8 or 12)
# if osp.isdir(osp.join(spm_directory, 'toolbox', 'OldNorm')):
# environ['SPM_VERSION'] = '12'
# elif osp.isdir(osp.join(spm_directory, 'templates')):
# environ['SPM_VERSION'] = '8'
# else:
# environ.pop('SPM_VERSION', None)

# For SPM with MATLAB license, if we want to get the SPM version from a system
# call to matlab:.
#            matlab_cmd = ('addpath("' + capsul_engine.spm.directory + '");'
#                          ' [name, ~]=spm("Ver");'
#                          ' fprintf(2, \"%s\", name(4:end));'
#                          ' exit')
#
#            try:
#                p = subprocess.Popen([capsul_engine.matlab.executable,
#                                       '-nodisplay', '-nodesktop',
#                                       '-nosplash', '-singleCompThread',
#                                       '-batch', matlab_cmd],
#                                     stdin=subprocess.PIPE,
#                                     stdout=subprocess.PIPE,
#                                     stderr=subprocess.PIPE)
#                output, err = p.communicate()
#                rc = p.returncode
#
#            except FileNotFoundError as e:
#                print('\n {0}'.format(e))
#                rc = 111
#
#            except Exception as e:
#                rc = 111
#
#            if (rc != 111) and (rc != 0):
#                print(err)
#
#            if rc == 0:
#                 capsul_engine.spm.version = err.decode("utf-8")
#

    """
    Activate the SPM module (set env variables) from the global configurations,
    in order to use them via :mod:`capsul.in_context.spm` functions
    """
    conf = engine.configurations.get("capsul.engine.module.spm", {})
    mlab_conf = engine.configurations.get("capsul.engine.module.matlab", {})
    spm_dir = conf.get("directory")
    mcr_dir = mlab_conf.get("mcr_directory")
    if spm_dir:
        os.environ["SPM_DIRECTORY"] = six.ensure_str(spm_dir)
    elif "SPM_DIRECTORY" in os.environ:
        del os.environ["SPM_DIRECTORY"]
    spm_version = conf.get("version")
    if spm_version:
        os.environ["SPM_VERSION"] = six.ensure_str(spm_version)
    elif "SPM_VERSION" in os.environ:
        del os.environ["SPM_VERSION"]
    spm_standalone = conf.get("standalone")
    if spm_standalone is not None:
        os.environ["SPM_STANDALONE"] = "yes" if spm_standalone else "no"
        if spm_standalone and mcr_dir:
            os.environ["MCR_HOME"] = mcr_dir
    elif "SPM_STANDALONE" in os.environ:
        del os.environ["SPM_STANDALONE"]

    """

    Returns
    -------
    """
    if conf.get("directory") is None:
        invalid.append("directory")

def edition_widget(engine, environment, config_id="any"):
    """Edition GUI for SPM config - see
    """

            values = {}
            if controller.directory in (None, traits.Undefined, ""):
                values["directory"] = None
            else:
                values["directory"] = controller.directory
            values["standalone"] = controller.standalone
            if controller.version in (None, traits.Undefined, ""):
                values["version"] = None
            else:
                values["version"] = controller.version
            id = "spm%s%s" % (
                controller.version if controller.version != traits.Undefined else "",
                "-standalone" if controller.standalone else "",
            )
            values["config_id"] = id
            query = 'config_id == "%s"' % id
            conf = session.config("spm", widget.environment, selection=query)
                session.new_config("spm", widget.environment, values)
            else:
                for k in ("directory", "standalone", "version"):
                    if k == "directory" and values[k] and not os.path.isdir(values[k]):
                        raise NotADirectoryError(
                            "\nSPM directory was not "
                            "updated:\n{} is not "
                            "existing!".format(values[k])
                        )
                    else:
                        setattr(conf, k, values[k])
            if id != widget.config_id:
                try:
                    session.remove_config("spm", widget.environment, widget.config_id)
                    pass
                widget.config_id = id

            traits.Undefined, output=False, desc="Directory containing SPM."
        ),
    )
        "standalone",
        traits.Bool(False, desc="If True, use the standalone version of SPM."),
    )
        "version",
        traits.Str(
            traits.Undefined,
            desc='Version string for SPM: "12", "8", etc.',
        ),
    )

    conf = None
    if config_id == "any":
        conf = engine.settings.select_configurations(environment, {"spm": "any"})
    else:
        try:
            conf = engine.settings.select_configurations(
                environment, {"spm": 'config_id=="%s"' % config_id}
            )
            pass
    if conf:
        controller.directory = conf.get("capsul.engine.module.spm", {}).get(
            "directory", traits.Undefined
        )
        controller.standalone = conf.get("capsul.engine.module.spm", {}).get(
            "standalone", False
        )
        controller.version = conf.get("capsul.engine.module.spm", {}).get(
            "version", traits.Undefined
        )
        config_id = conf.get("capsul.engine.module.spm", {}).get("config_id", config_id)

    # TODO handle several configs

    widget.config_id = config_id

'''
Implementation of :class:`~capsul.engine.CapsulEngine` processing methods.
They have been moved to this file for clarity.

Running is always using `Soma-Workflow <https://github.com/populse/soma-workflow/>`_.
'''

from traits.api import Undefined
import six
import os
import io

class WorkflowExecutionError(Exception):
    '''
    Exception class raised when a workflow execution fails.
    It holds references to the
    :class:`~soma_workflow.client.WorkflowController` and the workflow id
    '''
    def __init__(self, controller, workflow_id, status=None,
                 workflow_kept=True, verbose=True):
        wk = ''
        wc = ''
        precisions = ''
        if workflow_kept:
            wk = 'not '
            wc = ' from soma_workflow and must be deleted manually'
            self.controller = controller
            self.workflow_id = workflow_id
        if verbose:
            import soma_workflow.client as swclient

            failed_jobs = swclient.Helper.list_failed_jobs(
                workflow_id, controller)
            precisions_list = ['\nFailed jobs: %s' % repr(failed_jobs)]
            from soma_workflow import constants as swc
            aborted_statuses = set(
                [swc.EXIT_UNDETERMINED,
                 swc.EXIT_ABORTED,
                 swc.FINISHED_TERM_SIG,
                 swc.FINISHED_UNCLEAR_CONDITIONS,
                 swc.USER_KILLED])
            aborted_jobs = swclient.Helper.list_failed_jobs(
                workflow_id, controller, include_statuses=aborted_statuses)
            aborted_jobs = [job for job in aborted_jobs
                            if job not in failed_jobs]
            precisions_list.append('Aborted/killed jobs: %s'
                                   % repr(aborted_jobs))
            tmp1 = tempfile.mkstemp(prefix='capsul_swf_job_stdout')
            tmp2 = tempfile.mkstemp(prefix='capsul_swf_job_stderr')
            os.close(tmp1[0])
            os.close(tmp2[0])
            fileio = io.StringIO()
            try:
                controller.log_failed_workflow(workflow_id, file=fileio)
                precisions_list.append(fileio.getvalue())

                #jobs = failed_jobs + aborted_jobs
                #cmds = controller.jobs(jobs)
                #workflow = controller.workflow(workflow_id)
                #for job_id in jobs:
                    #jinfo = cmds[job_id]
                    #if job_id in failed_jobs:
                        #has_run = True
                    #else:
                        #has_run = False
                    #status = controller.job_termination_status(job_id)
                    #controller.retrieve_job_stdouterr(job_id, tmp1[1], tmp2[1])
                    #with open(tmp1[1]) as f:
                        #stdout = f.read()
                    #with open(tmp2[1]) as f:
                        #stderr = f.read()
                    #full_job = [j for j in workflow.jobs
                                #if workflow.job_mapping[j].job_id == job_id][0]
                    #precisions_list += [
                        #'============================================',
                        #'---- failed job info ---',
                        #'* job: %d: %s' % (job_id, jinfo[0]),
                        #'* exit status: %s' % status[0],
                        #'* commandline:',
                        #jinfo[1],
                        #'* exit value: %s' % str(status[1]),
                        #'* term signal: %s' % str(status[2]),
                        #'---- env ----',
                        #repr(full_job.env),
                        #'---- configuration ----',
                        #repr(full_job.configuration),
                        #'---- stdout ----',
                        #stdout,
                        #'---- stderr ----',
                        #stderr,
                        ##'---- full host env ----',
                        ##repr(os.environ)
                    #]
            finally:
                if os.path.exists(tmp1[1]):
                    os.unlink(tmp1[1])
                if os.path.exists(tmp2[1]):
                    os.unlink(tmp2[1])
            precisions = '\n'.join(precisions_list)

        super(WorkflowExecutionError, self).__init__('Error during '
            'workflow execution. Status=%s.\n'
            'The workflow has %sbeen removed%s. %s'
            % (status, wk, wc, precisions))

def start(engine, process, workflow=None, history=True, get_pipeline=False, **kwargs):
    '''
    Asynchronously start the execution of a process or pipeline in the
    connected computing environment. Returns an identifier of
    the process execution and can be used to get the status of the
    execution or wait for its termination.

    TODO:
    if history is True, an entry of the process execution is stored in
    the database. The content of this entry is to be defined but it will
    contain the process parameters (to restart the process) and will be
    updated on process termination (for instance to store execution time
    if possible).

    engine: CapsulEngine
    process: Process or Pipeline instance
    workflow: Workflow instance (optional - if already defined before call)
    history: bool (optional)
        TODO: not implemented yet.
    get_pipeline: bool (optional)
        if True, start() will return a tuple (execution_id, pipeline). The
        pipeline is normally the input pipeline (process) if it is actually
        a pipeline. But if the input process is a "single process", it will
        be inserted into a small pipeline for execution. This pipeline will
        be the one actually run, and may be passed to :meth:`wait` to set
        output parameters.

    Returns
    -------
    execution_id: int
        execution identifier (actually a soma-workflow id)
    pipeline: Pipeline instance (optional)
        only returned if get_pipeline is True.
    '''

    # set parameters values
    for k, v in six.iteritems(kwargs):
        setattr(process, k, v)

    missing = process.get_missing_mandatory_parameters()
    if len(missing) != 0:
        ptype = 'process'
            ptype = 'pipeline'
        raise ValueError('In %s %s: missing mandatory parameters: %s'
                          % (ptype, process.name,
                            ', '.join(missing)))

    # Use soma workflow to execute the pipeline or process in parallel
    # on the local machine

    # Create soma workflow pipeline
    from capsul.pipeline.pipeline_workflow import workflow_from_pipeline
    import soma_workflow.client as swclient

    swm = engine.study_config.modules['SomaWorkflowConfig']
    swm.connect_resource(engine.connected_to())
    controller = swm.get_workflow_controller()
    resource_id = swm.get_resource_id()

    query = 'config_id=="somaworkflow"'
    if resource_id is not None:
        query = f'computing_resource == "{resource_id}"'
    resource_config_d = engine.settings.select_configurations(
        'global', {'somaworkflow': query})
    resource_config = resource_config_d.get(
        'capsul.engine.module.somaworkflow', {})
    has_resource_config = ('capsul.engine.module.somaworkflow'
        in resource_config_d)

    if has_resource_config:
        environment = resource_id
    else:
        environment = 'global'

    if workflow is None:
        workflow = workflow_from_pipeline(process, environment=environment)

    queue = resource_config.get('queue', None)
    max_running_jobs = resource_config.get('max_running_jobs', None)
    max_queued_jobs = resource_config.get('max_queued_jobs', None)
    #if hasattr(engine.study_config.somaworkflow_computing_resources_config,
               #resource_id):
        #res_conf = getattr(
            #engine.study_config.somaworkflow_computing_resources_config,
            #resource_id)
        #queue = res_conf.queue
        #if queue is Undefined:
            #queue = None
    if max_running_jobs is not None:
        controller.config.change_running_jobs_limits(queue, max_running_jobs)
        if controller.config.is_local_resource(
                controller.config._config_parser, resource_id) \
                and controller.config.get_scheduler_type() \
                == 'local_basic':
            controller.scheduler_config.set_max_proc_nb(max_running_jobs)
            controller.scheduler_config.set_proc_nb(max_running_jobs)
    if max_queued_jobs is not None:
        controller.config.change_queue_limits(queue, max_queued_jobs)
            controller.scheduler_config.set_proc_nb(max_queued_jobs)
    workflow_name = process.name
    wf_id = controller.submit_workflow(workflow=workflow, name=workflow_name,
                                       queue=queue)
    swclient.Helper.transfer_input_files(wf_id, controller)

    if get_pipeline:
        return wf_id, workflow.pipeline()
    # else forget the pipeline
    return wf_id

def wait(engine, execution_id, timeout=-1, pipeline=None):
    '''
    Wait for the end of a process execution (either normal termination,
    interruption or error).
    '''
    from soma_workflow import constants

    wf_id = execution_id

    controller.wait_workflow(wf_id, timeout=timeout)
    workflow_status = controller.workflow_status(wf_id)
    if workflow_status != constants.WORKFLOW_DONE:
        # not finished
        return workflow_status

    # get output values
    if pipeline:
        proc_map = {}
        todo = [pipeline]
        while todo:
            process = todo.pop(0)
                todo += [n.process for n in process.nodes.values()
                         if n is not process.pipeline_node
                             and isinstance(n, ProcessNode)]
            else:
                proc_map[id(process)] = process

        eng_wf = controller.workflow(wf_id)
        for job in eng_wf.jobs:
            if job.has_outputs:
                out_params = controller.get_job_output_params(
                    eng_wf.job_mapping[job].job_id)
                if out_params:
                    process = proc_map.get(job.process_hash)
                    if process is None:
                        # iteration or non-process job
                        continue
                    for param in list(out_params.keys()):
                        if process.trait(param) is None:
                            del out_params[param]
                    try:
                        process.import_from_dict(out_params)
                    except Exception as e:
                        print('error while importing outputs in', process.name)
                        print('outputs:', out_params)
                        print(e)

    # TODO: should we transfer if the WF fails ?
    swclient.Helper.transfer_output_files(wf_id, controller)
    return status(engine, execution_id)

def interrupt(engine, execution_id):
    '''
    Try to stop the execution of a process. Does not wait for the process
    to be terminated.
    '''
    controller.stop_workflow(execution_id)

def status(engine, execution_id):
    '''
    Return a simple value with the status of an execution (queued,
    running, terminated, error, etc.)
    '''

    workflow_status = controller.workflow_status(execution_id)
    if workflow_status == constants.WORKFLOW_DONE:
        # finished, but in which state ?
        elements_status = controller.workflow_elements_status(execution_id)
        failed_jobs = [element for element in elements_status[0]
                       if element[1] == constants.FAILED
                       or (element[1] == constants.DONE and
                           (element[3][0]
                            not in (constants.FINISHED_REGULARLY, None)
                            or element[3][1] != 0))]
        if len(failed_jobs) != 0:
            workflow_status = 'workflow_failed'

    return workflow_status

def detailed_information(engine, execution_id):
    '''
    Return complete (and possibly big) information about a process
    execution.
    '''
    elements_status = controller.workflow_elements_status(execution_id)

    return elements_status

def dispose(engine, execution_id, conditional=False):
    '''
    Update the database with the current state of a process execution and
    free the resources used in the computing resource (i.e. remove the
    workflow from SomaWorkflow).

    If ``conditional`` is set to True, then dispose is only done if the
    configuration does not specify to keep succeeded / failed workflows.
    '''
    keep = False
    if conditional:
        if engine.study_config.somaworkflow_keep_succeeded_workflows:
            keep = True
        else:
            keep = False
            if engine.study_config.somaworkflow_keep_failed_workflows:
                # must see it it failed or not
                from soma_workflow import constants

                status = engine.status(execution_id)
                if status != constants.WORKFLOW_DONE:
                    keep = True
    if not keep:
        swm = engine.study_config.modules['SomaWorkflowConfig']
        swm.connect_resource(engine.connected_to())
        controller = swm.get_workflow_controller()
        controller.delete_workflow(execution_id)
        # TODO: update engine DB

def call(engine, process, history=True, **kwargs):
    eid, pipeline = engine.start(process, workflow=None, history=history, get_pipeline=True,
                                 **kwargs)
    status = engine.wait(eid, pipeline=pipeline)
    engine.dispose(eid, conditional=True)
    return status

def check_call(engine, process, history=True, **kwargs):
                                 **kwargs)
    try:
        engine.raise_for_status(status, eid)
    finally:
        engine.dispose(eid, conditional=True)

def raise_for_status(engine, status, execution_id=None):
    '''
    Raise an exception if a process execution failed
    '''

    if status != constants.WORKFLOW_DONE:
        raise WorkflowExecutionError(
            controller, execution_id, status,
            engine.study_config.somaworkflow_keep_failed_workflows)

"""
This module provides classes to store CapsulEngine settings for several execution environment and choose a configuration for a given execution environment. Setting management in Capsul has several features that makes it different from classical ways to deal with configuration:

* CapsulEngine must be able to deal with several configurations for the same software. For instance, one can configure both SPM 8 and SPM 12 and choose later the one to use.
* A single pipeline may use various configurations of a software. For instance a pipeline could compare the results of SPM 8 and SPM 12.
* Settings definition must be modular. It must be possible to define possible settings values either in Capsul (for well known for instance) or in external modules that can be installed separately.
* Capsul must deal with module dependencies. For instance the settings of SPM may depends on the settings of Matlab. But this dependency exists only if a non standalone SPM version is used. Therefore, dependencies between modules may depends on settings values.
* CapsulEngine settings must provide the possibility to express a requirement on settings. For instance a process may require to have version of SPM greater or equal to 12.
* The configuration of a module can be defined for a specific execution environment. Settings must allow to deal with several executions environments (e.g. a local machine and a computing cluster). Each environment may have a different configuration (for instance the SPM installation directory is not the same on the local computer and on a computing cluster).

To implement all these features, it was necessary to have a settings storage system providing a query language to express requirements such as ``spm.version >= 12``. Populse_db was thus chosen as the storage and query system for settings. Some of the settings API choices have been influenced by populse_db API.

CapsulEngine settings are organized in modules. Each module defines and documents the schema of values that can be set for its configuration. Typically, a module is dedicated to a software. For instance the module for SPM accepts configurations containing a version (a string), an install directory (a string), a standalone/matlab flag (a boolean), etc. This schema is used to record configuration documents for the module. There can be several configuration documents per module. Each document corresponds to a full configuration of the module (for instance a document for SPM 8 configuration and another for SPM 12, or one for SPM 12 standalone and another for SPM 12 with matlab).

Settings cannot be used directly to configure the execution of a software. It is necessary to first select a single configuration document for each module. This configurations selection step is done by the :meth:`Settings.select_configurations` method.

"""

#
# Questions about settings
# How to automatically build a GUI to input settings values ?
# * default values
# * mandatory / optional settings
# * dependencies between settings
#
# all these were "simple" using Controllers but are in a way "too free" now,
# and "too constrained" in another way (table columns: not dicts or structures)
#
# config modules might provide validation functions (check consistency,
# completeness of config values), and maybe a Controller to build a GUI ?
#
# How to store additional data indirectly linked with settings values ?
# Some modules were using objects shared through StudyConfig.modules_data
# (or CapsulEngine).
#
# doc on modules activation / use ?
#

from uuid import uuid4
import sys

class Settings:
    """
    Main class for the management of CapsulEngine settings. Since these
    settings are always stored in a populse_db database, it is necessary to
    activate a settings session in order to read or modify settings. This is
    done by using a with clause::

        # Create a CapsulEngine
        with ce.settings as settings:
            # Read or modify settings here
            conf = settings.new_config('spm', 'global',
                                       {'version': '12', 'standalone': True})
            # modify value
            conf.directory = '/usr/local/spm12-standalone'
    """

    global_environment = "global"
    collection_prefix = "settings/"
    environment_field = "config_environment"
    config_id_field = "config_id"

    def __init__(self, populse_db):
        """
        Create a settings instance using the given populse_db instance
        """
        self.populse_db = populse_db
        self.module_notifiers = {}

    def __enter__(self):
        """
        Starts a session to read or write settings
        """

        return SettingsSession(self.populse_db, module_notifiers=self.module_notifiers)

    def __exit__(self, *args):
        pass

    def module_name(module_name):
        """
        Return a complete module name (which must be a valid Python module
        name) given a possibly abbreviated module name. This method must
        be used whenever a module name is written by a user (for instance
        in a configuration file.
        This method add the prefix `'capsul.engine.module.'` if the module
        name does not contain a dot.
        """
        if "." not in module_name:
            module_name = "capsul.engine.module." + module_name
        return module_name

    def select_configurations(self, environment, uses=None, check_invalid_mods=False):
        """
        Select a configuration for a given environment. A configuration is
        a dictionary whose keys are module names and values are
        configuration documents. The returned set of configuration per module
        can be activaded with `capsul.api.activate_configuration()`.

        The `uses` parameter determine which modules
        must be included in the configuration. If not given, this method
        considers all configurations for every module defined in settings.
        This parameter is a dictionary whose keys are a module name and
        values are populse_db queries used to select module.

        The environment parameter defines the execution environment in which
        the configurations will be used. For each module, configurations are
        filtered with the query. First, values are searched in the given
        environment and, if no result is found, the `'global'` environment
        (the value defined in `Settings.global_environment`) is used.

        If `check_invalid_mods` is True, then each selected config module is
        checked for missing values and discarded if there are.

        example
        -------
        To select a SPM version greater than 8 for an environment called
        `'my_environment'` one could use the following code::

            config = ce.select_configurations('my_environment',
                                              uses={'spm': 'version > 8'})
        """
        configurations = {}
        with self as settings:
            if uses is None:
                uses = {}
                with settings._storage.data() as data:
                    for collection in data.collection_names():
                        if collection.startswith(Settings.collection_prefix):
                            module_name = collection[len(Settings.collection_prefix) :]
                            uses[module_name] = "ALL"
            uses_stack = list(uses.items())
            while uses_stack:
                module, query = uses_stack.pop(-1)
                # append environment to config_id if it is given in the filter
                if 'config_id=="' in query:
                    i = query.index('config_id=="')
                    i = query.index('"', i + 12)
                    query = "%s-%s%s" % (query[:i], environment, query[i:])

                module = self.module_name(module)
                if module in configurations:
                    continue
                configurations.setdefault("capsul_engine", {}).setdefault("uses", {})[
                    module
                ] = query
                selected_config = None
                query_env = environment
                full_query = '%s == "%s" AND (%s)' % (
                    Settings.environment_field,
                    environment,
                    ("ALL" if query == "any" else query),
                )
                collection = "%s%s" % (Settings.collection_prefix, module)
                    if data.has_collection(collection):
                        docs = data[collection].search(full_query)
                    else:
                        docs = []

                if check_invalid_mods:
                    # filter out invalid configs
                    doc_t = docs
                    docs = []
                    for doc in doc_t:
                        mod = sys.modules[module]
                        invalid = []
                        if hasattr(mod, "check_notably_invalid_config"):
                            invalid = mod.check_notably_invalid_config(doc)
                        if len(invalid) == 0:
                            docs.append(doc)

                if len(docs) == 1:
                    selected_config = docs[0]
                elif len(docs) > 1:
                    if query == "any":
                        selected_config = docs[0]
                    else:
                        raise EnvironmentError(
                            "Cannot create configurations "
                            'for environment "%s" because settings returned '
                            "%d instances for module %s"
                            % (environment, len(docs), module)
                        )
                else:
                    full_query = '%s == "%s" AND (%s)' % (
                        Settings.environment_field,
                        Settings.global_environment,
                        ("ALL" if query == "any" else query),
                    )
                    query_env = Settings.global_environment
                    with settings._storage.data() as data:
                        if data.has_collection(collection):
                            docs = data[collection].search(full_query)
                        else:
                            docs = []

                    if check_invalid_mods:
                        # filter out invalid configs
                        doc_t = docs
                        docs = []
                        for doc in doc_t:
                            mod = sys.modules[module]
                            invalid = []
                            if hasattr(mod, "check_notably_invalid_config"):
                                invalid = mod.check_notably_invalid_config(doc)
                            if len(invalid) == 0:
                                docs.append(doc)

                    if len(docs) == 1:
                    elif len(docs) > 1:
                        if query == "any":
                            selected_config = docs[0]
                        else:
                            raise EnvironmentError(
                                "Cannot create "
                                'configurations for environment "%s" because '
                                "global settings returned %d instances for "
                                "module %s" % (query_env, len(docs), module)
                            )
                if selected_config:
                    # Remove values that are None
                    items = getattr(selected_config, "_items", None)
                    if items is None:
                        # older populse_db 1.x
                        items = selected_config.items
                    selected_config = dict(items())
                    if "config_id" in selected_config:
                        selected_config["config_id"] = selected_config["config_id"][
                            : -len(query_env) - 1
                        ]
                    for k, v in list(items()):
                        if v is None:
                            del selected_config[k]
                    configurations[module] = selected_config
                    python_module = importlib.import_module(module)
                    config_dependencies = getattr(
                        python_module, "config_dependencies", None
                    )
                    if config_dependencies:
                        d = config_dependencies(selected_config)
                        if d:
                            uses_stack.extend(
                                [(Settings.module_name(k), v) for k, v in d.items()]
                            )

        return configurations

    def export_config_dict(self, environment=None):
        conf = {}
        if environment is None:
            environment = self.get_all_environments()
        elif isinstance(environment, str):
            environment = [environment]
        with self as session:
            modules = []
            with session._storage.data() as data:
                for collection in data.collection_names():
                    if collection.startswith(Settings.collection_prefix):
                        module_name = collection[len(Settings.collection_prefix) :]
                        modules.append(module_name)

                for env in environment:
                    env_conf = {}
                    for module in modules:
                        mod_conf = {}
                        for config in session.configs(module, env):
                            id = f"{config._id}-{env}"
                            doc = data[session.collection_name(module)][id].get()
                            if "config_id" in doc:
                                doc["config_id"] = doc["config_id"][: -len(env) - 1]
                            mod_conf[config._id] = doc
                        if mod_conf:
                            env_conf[module] = mod_conf

                    env_conf["capsul_engine"] = {"uses": {m: "ALL" for m in modules}}

                    conf[env] = env_conf

        return conf

        """
        :meth:`select_configurations`.

        Compared to :meth:`CapsulEngine.import_configs` this method (at
        :class:`Settings` level) does not load the required modules.
        """

            for module in modules:
                mod_dict = config_dict.get(module, {})
                if mod_dict:
                    if "config_id" in mod_dict:
                        # select_configurations() shape: 1 config per module
                        mod_dict = {mod_dict["config_id"]: mod_dict}
                    # otherwise several config are indexed by their config_id
                    for config_id, config in mod_dict.items():
                        conf = session.config(
                            module, environment, 'config_id == "%s"' % config_id
                        )
                        try:
                            if conf:
                                values = {
                                    k: v
                                    for k, v in config.items()
                                    if k not in ("config_id", "config_environment")
                                }
                                if values:
                                    conf.set_values(values)
                            else:
                                session.new_config(module, environment, config)
                        except Exception as e:
                            if not cont_on_error:
                                raise
                            print(e)
                            print(
                                "Capsul config import fails for " "environment:",
                                environment,
                                ", module:",
                                module,
                                ", config:",
                                config,
                                file=sys.stderr,
                            )

    def get_all_environments(self):
        """
        Get all environment values in the database
        """
            return session.get_all_environments()

class SettingsSession:
    """
    Settings use/modification session, returned by "with settings as session:"
    """

    def __init__(self, populse_db_storage, module_notifiers=None):
        """
        SettingsSession are created with Settings.__enter__ using a `with`
        statement.
        """
        self._storage = populse_db_storage
        if module_notifiers is None:
            self.module_notifiers = {}
        else:
            self.module_notifiers = module_notifiers

    def collection_name(module):
        """
        Return the name of the populse_db collection corresponding to a
        settings module. The result is the full name of the module
        prefixed by Settings.collection_prefix (i.e. `'settings/'`).
        """
        module = Settings.module_name(module)
        collection = "%s%s" % (Settings.collection_prefix, module)
        return collection

    def ensure_module_fields(self, module, fields):
        """
        Make sure that the given module exists in settings and create the given fields if they do not exist. `fields` is a list of dictionaries with three items:
        - name: the name of the key
        - type: the data type of the field (in populse_db format)
        - description: the documentation of the field
        """
        collection = self.collection_name(module)
        with self._storage.schema() as schema:
            schema.add_collection(collection, Settings.config_id_field)
            schema.add_field(collection, Settings.environment_field, "str", index=True)
            for field in fields:
                name = field["name"]
                schema.add_field(
                    collection,
                    name,
                    field_type=field["type"],
                    description=field["description"],
                )

    def new_config(self, module, environment, values):
        """
        Creates a new configuration document for a module in the given
        environment. Values is a dictionary used to set values for the
        document. The document mut have a unique string identifier in
        the `Settings.config_id_field` (i.e. `'config_id'`), if None is
        given in `values` a unique random value is created (with
        `uuid.uuid4()`).
        """
        document = {Settings.environment_field: environment}
        document.update(values)
        id = document.get(Settings.config_id_field)
        if id is None:
            id = str(uuid4())
        with self._storage.data(write=True) as data:
            if data is not None:
                data[collection][f"{id}-{environment}"] = document
            else:
                print("Data session is None, unable to insert document.")
        config = SettingsConfig(
            self._storage,
            collection,
            id,
            environment,
            notifiers=self.module_notifiers.get(Settings.module_name(module), []),
        )
        config.notify()
        return config

    def remove_config(self, module, environment, config_id):
        """
        Removes a configuration (document in the database) for a given module /
        environment, identified by its `Settings.config_id_field` value.
        """
        id = "%s-%s" % (config_id, environment)
            del data[collection][id]

    def configs(self, module, environment, selection=None):
        """
        Returns a generator that iterates over all configuration
        documents created for the given module and environment.
        """
        with self._storage.data() as data:
            if data.has_collection(collection):
                if selection:
                    if "config_id" in selection:
                        x = selection.find("config_id")
                        x = selection.find('"', x + 9)
                        x = selection.find('"', x + 1)
                        selection = "%s-%s%s" % (
                            selection[:x],
                            environment,
                            selection[x:],
                        )
                    full_query = f'{Settings.environment_field} == "{environment}" AND ({selection})'
                    docs = data[collection].search(full_query)
                else:
                    docs = data[collection].search(
                        f'{Settings.environment_field}=="{environment}"'
                    )
                for d in docs:
                    id = d[Settings.config_id_field]
                    id = id[: -len(environment) - 1]
                    yield SettingsConfig(
                        self._storage,
                        collection,
                        id,
                        environment,
                        notifiers=self.module_notifiers.get(
                            Settings.module_name(module), []
                        ),
                    )

    def config(self, module, environment, selection=None, any=True):
        """
        Selects configurations (like in :meth:`configs`) and ensures at most
        one one is selected

        module: str
            module name
        environment: str
            environment id ('global' etc)
        selection: str (optional)
            to select the configuration
        any: bool (optional)
            When more than one config is found, if ``any`` is True (default),
            return any of them (the first one). If ``any`` is False, return
            None.

        Returns
        -------
        config: SettingsConfig instance or None
            None if no matching config is found or more than one if any is
            False
        """
        configs = list(self.configs(module, environment, selection))
        if len(configs) == 0:
            return None
        if len(configs) == 1 or any:
            return configs[0]

        """
        """
        environments = set()
            for collection in data.collection_names():
                if collection.startswith(Settings.collection_prefix):
                    rows = data[collection].search(
                        fields=[Settings.environment_field], as_list=True, distinct=True
                    )
                    environments.update([row[0] for row in rows])
        return environments

class SettingsConfig:
    def __init__(self, populse_session, collection, id, environment, notifiers=[]):
        super(SettingsConfig, self).__setattr__("_storage", populse_session)
        super(SettingsConfig, self).__setattr__("_collection", collection)
        super(SettingsConfig, self).__setattr__("_environment", environment)
        super(SettingsConfig, self).__setattr__("_id", id)
        super(SettingsConfig, self).__setattr__("_notifiers", notifiers)

    def __setattr__(self, name, value):
        id = "%s-%s" % (
            super(SettingsConfig, self).__getattribute__("_id"),
            super(SettingsConfig, self).__getattribute__("_environment"),
        )
        if getattr(self, name) != value:
            with self._storage.data(write=True) as data:
                data[self._collection][id][name] = value
            # notify change for listeners
            self.notify(name, value)

    def __getattr__(self, name):
        if name in ("_id", "_environment"):
            return super(SettingsConfig, self).__getattribute__(name)
            value = data[self._collection][f"{self._id}-{self._environment}"][
                name
            ].get()
        if name == "config_id":
            return value[: -len(self._environment) - 1]
        return value

    def set_values(self, values):
        id = "%s-%s" % (self._id, self._environment)
            old = data[self._collection][id].get(fields=values.keys(), as_list=True)
            mod_values = {k: v for o, (k, v) in zip(old, values.items()) if o != v}
            if mod_values:
                data[self._collection][id].update(mod_values)
                for name, value in mod_values.items():
                    self.notify(name, value)

    def notify(self, name=None, value=None):
        for notifier in self._notifiers:
            notifier(name, value)

'''
Specific subprocess-like functions to call FSL taking into account 
configuration stored in ExecutionContext. To functions and class in
this module it is mandatory to activate an ExecutionContext (using a
with statement). For instance::

   from capsul.engine import capsul_engine
   from capsul.in_context.fsl import fsl_check_call

   ce = capsul_engine()
   with ce:
       fsl_check_call(['bet', '/somewhere/myimage.nii'])

For calling FSL command with this module, the first argument of
command line must be the FSL executable without any path nor prefix. 
Prefix are used in Neurodebian install. For instance on Ubuntu 16.04 
Neurodebian FSL commands are prefixed with "fsl5.0-".
The appropriate path and eventually prefix are added from the configuration
of the ExecutionContext.
'''

import os
import soma.subprocess
import six

from soma.utils.env import parse_env_lines

'''
If this variable is set, it contains FS runtime env variables, allowing to run directly freesurfer commands from this process.
'''
fsl_runtime_env = None

def fsl_command_with_environment(command, use_prefix=True,
                                 use_runtime_env=True):
    '''
    Given an FSL command where first element is a command name without
    any path or prefix (e.g. "bet"). Returns the appropriate command to
    call taking into account the FSL configuration stored in the
    activated ExecutionContext.

    Usinfg :func`fsl_env` is an alternative to this.
    '''
    if use_prefix:
        fsl_prefix = os.environ.get('FSL_PREFIX', '')
    else:
        fsl_prefix = ''
    if use_runtime_env and fsl_runtime_env:
        c0 = list(osp.split(command[0]))
        c0[-1] = '%s%s' % (fsl_prefix, c0[-1])
        c0 = osp.join(*c0)
        cmd = [c0] + command[1:]
        return cmd

    fsl_dir = os.environ.get('FSLDIR')
        dir_prefix = '%s/bin/' % fsl_dir
    else:
        dir_prefix = ''
    fsl_config = os.environ.get('FSL_CONFIG')
    if fsl_prefix and not os.path.isdir(dir_prefix):

    if fsl_config:
        fsldir = osp.dirname(osp.dirname(osp.dirname(fsl_config)))
        shell = os.environ.get('SHELL', '/bin/sh')
        if shell.endswith('csh'):
            cmd = [shell, '-c', 
                'setenv FSLDIR "{0}"; setenv PATH "{0}/bin:$PATH"; source {0}/etc/fslconf/fsl.csh;exec {1}{2} '.format(
                    fsldir, fsl_prefix, command[0]) + \
                    ' '.join("'%s'" % i.replace("'", "\\'") for i in command[1:])]
        else:
                'export FSLDIR="{0}"; export PATH="{0}/bin:$PATH"; . {0}/etc/fslconf/fsl.sh;exec {1}{2} '.format(
    else:
        cmd = ['%s%s%s' % (dir_prefix,
                           fsl_prefix, 
                           command[0])] + command[1:]
    return cmd

def fsl_env():
    '''
    get FSL env variables by running the setup script in a separate bash
    process
    '''
    global fsl_runtime_env

    if fsl_runtime_env is not None:
        return fsl_runtime_env

    kwargs = {}
        cmd = ['env']
        if fsl_dir:
            kwargs = {'env': {'PATH': os.pathsep.join(
                ['%s/bin' % fsl_dir, os.environ.get('PATH', '')])}}
    else:
        cmd = fsl_command_with_environment(['env'], use_prefix=False,
                                           use_runtime_env=False)
    new_env = soma.subprocess.check_output(cmd, **kwargs).decode(
        'utf-8').strip()
    new_env = parse_env_lines(new_env)
    env = {}
    for l in new_env:
        name, val = l.strip().split('=', 1)
        name = six.ensure_str(name)
        val = six.ensure_str(val)
        if name not in ('_', 'SHLVL') and (name not in os.environ
                                           or os.environ[name] != val):
            env[name] = val
    # add PATH
        fsl_bin = osp.join(fsl_dir, 'bin')
        env['PATH'] = os.pathsep.join([fsl_bin, os.environ.get('PATH', '')])
    # cache dict
    fsl_runtime_env = env
    return env

class FslPopen(soma.subprocess.Popen):
    '''
    Equivalent to Python subprocess.Popen for FSL commands
    '''
    def __init__(self, command, **kwargs):
        cmd = fsl_command_with_environment(command)
        env = fsl_env()
        if 'env' in kwargs:
            env = dict(env)
            env.update(kwargs['env'])
            kwargs = dict(kwargs)
            del kwargs['env']
        super(FslPopen, self).__init__(cmd, env=env, **kwargs)

def fsl_call(command, **kwargs):
    '''
    Equivalent to Python subprocess.call for FSL commands
    '''
    cmd = fsl_command_with_environment(command)
    env = fsl_env()
    if 'env' in kwargs:
        env = dict(env)
        env.update(kwargs['env'])
        kwargs = dict(kwargs)
        del kwargs['env']
    return soma.subprocess.call(cmd, env=env, **kwargs)

def fsl_check_call(command, **kwargs):
    '''
    Equivalent to Python subprocess.check_call for FSL commands
    '''
    return soma.subprocess.check_call(cmd, env=env, **kwargs)

def fsl_check_output(command, **kwargs):
    '''
    Equivalent to Python subprocess.check_output for FSL commands
    '''
    return soma.subprocess.check_output(cmd, env=env, **kwargs)

'''
Specific subprocess-like functions to call SPM taking into account

   from capsul.in_context.spm import spm_check_call

   with ce:
      spm_check_call(spm_batch_filename)

For calling SPM command with this module, the first argument of
command line must be the SPM batch file to execute with Matlab.
'''

from __future__ import absolute_import, print_function

import glob
import os

def spm_command(spm_batch_filename):
    if os.environ.get('SPM_STANDALONE') == 'yes':
        if spm_batch_filename is not None:
            # Check that batch file exists and raise appropriate error if
            # not
            open(spm_batch_filename)
        spm_directory = os.environ.get('SPM_DIRECTORY', '')
        if spm_directory is not None:
            spm_directory = osp.expandvars(spm_directory.strip())
        mcr_dir = os.environ.get('MCR_HOME')
        if mcr_dir is not None:
            mcr_dir = osp.expandvars(mcr_dir.strip())
        if mcr_dir is None:
            mcr_glob = osp.join(spm_directory, 'mcr', 'v*')
            mcr_dir = glob.glob(mcr_glob)
            if not mcr_dir:
                raise ValueError('Cannot find Matlab MCR dir: %s' % mcr_glob)
            mcr_dir = mcr_dir[0]
            print('---- BATCH SMP ----')
            print(open(spm_batch_filename).read())
            print('-------------------')
        cmd = [osp.join(spm_directory,
                        'run_spm%s.sh' % os.environ.get('SPM_VERSION',
                                                        '')),
               mcr_dir]
            cmd += ['batch', spm_batch_filename]
    else:
        raise NotImplementedError('Running SPM with matlab is not '
                                  'implemented yet')
    return cmd

class SPMPopen(soma.subprocess.Popen):
    '''
    Equivalent to Python subprocess.Popen for SPM batch
    '''

    def __init__(self, spm_batch_filename, **kwargs):
        cmd = spm_command(spm_batch_filename)
        super(SPMPopen, self).__init__(cmd, **kwargs)

def spm_call(spm_batch_filename, **kwargs):
    '''
    Equivalent to Python subprocess.call for SPM batch
    '''
    cmd = spm_command(spm_batch_filename)
    return soma.subprocess.call(cmd, **kwargs)

def spm_check_call(spm_batch_filename, **kwargs):
    '''
    Equivalent to Python subprocess.check_call for SPM batch
    '''
    return soma.subprocess.check_call(cmd, **kwargs)

def spm_check_output(spm_batch_filename, **kwargs):
    '''
    Equivalent to Python subprocess.check_output for SPM batch
    '''
    return soma.subprocess.check_output(cmd, **kwargs)

if __name__ == '__main__':
    from capsul.api import capsul_engine
    import tempfile

    ce = capsul_engine()
    ce.global_config.spm.directory = '/casa/spm12_standalone'
    with ce:
        batch = tempfile.NamedTemporaryFile(suffix='.m')
        batch.write("fprintf(1, '%s', spm('dir'));")
        batch.flush()
        spm_call(batch.name)

''' Pipeline main class module

Classes
=======
:class:`Pipeline`
-----------------
'''

# System import
import logging
from copy import deepcopy
import os
import shutil
import sys
import six
from six.moves import zip
from collections import OrderedDict

# Define the logger
logger = logging.getLogger(__name__)

# Trait import
try:
    from traits.api import (File, Enum, Bool,
                            Event, Directory, Trait, List, Set)
except ImportError:
    import enthought.traits.api as traits
    from enthought.traits.api import (File, Enum, Bool,
                                      Event, Directory, Trait, List, Set)

# Capsul import
from capsul.process.process import Process, NipypeProcess
from .topological_sort import GraphNode
from .topological_sort import Graph
from .pipeline_nodes import Plug
from .pipeline_nodes import ProcessNode
from .pipeline_nodes import PipelineNode
from .pipeline_nodes import Switch
from .pipeline_nodes import OptionalOutputSwitch

# Soma import
from soma.controller import ControllerTrait
from soma.utils.functiontools import SomaPartial

class Pipeline(Process):
    """ Pipeline containing Process nodes, and links between node parameters.

    A Pipeline is normally subclassed, and its :py:meth:`pipeline_definition`
    method is overloaded to define its nodes and links.
    :py:meth:`pipeline_definition` will be called by the pipeline constructor.

    ::

        from capsul.pipeline import Pipeline

        class MyPipeline(Pipeline):

          def pipeline_definition(self):
              self.add_process('proc1', 'my_toolbox.my_process1')
              self.add_process('proc2', 'my_toolbox.my_process2')
              self.add_switch('main_switch', ['in1', 'in2'], ['out1', 'out2'])
              self.add_link('proc1.out1->main_switch.in1_switch_out1')
              self.add_link('proc1.out2->main_switch.in1_switch_out2')
              self.add_link('proc2.out1->main_switch.in2_switch_out1')
              self.add_link('proc2.out1->main_switch.in2_switch_out2')

    After execution of :py:meth:`pipeline_definition`, the inner nodes
    parameters which are not connected will be automatically exported to the
    parent pipeline, with names prefixed with their process name, unless they
    are listed in a special "do_not_export" list (passed to
    :py:meth:`add_process` or stored in the pipeline instance).

    >>> pipeline = MyPipeline()
    >>> print(pipeline.proc1_input)
    <undefined>

    **Nodes**

    A pipeline is made of nodes, and links between their parameters. Several
    types of nodes may be part of a pipeline:

    .. currentmodule:: capsul.pipeline

    * process nodes (:py:class:`pipeline_nodes.ProcessNode`) are the leaf nodes
      which represent actual processing bricks.
    * pipeline nodes (:py:class:`pipeline_nodes.PipelineNode`) are
      sub-pipelines which allow to reuse an existing pipeline within another
      one
    * switch nodes (:py:class:`pipeline_nodes.Switch`) allows to select values
      between several possible inputs. The switch mechanism also allows to
      select between several alternative processes or processing branches.
    * iterative process (:py:class:process_iteration.ProcessIteration`)
      represent parallel processing of the same pipeline on a set of
      parameters.

    .. currentmodule:: capsul.pipeline.pipeline

    Note that you normally do not instantiate these nodes explicitly when
    building a pipeline. Rather programmers may call the
    :py:meth:`add_process`, :py:meth:`add_switch`,
    :py:meth:`add_iterative_process` methods.

    **Nodes activation**

    Pipeline nodes may be enabled or disabled. Disabling a node will trigger
    a global pipeline nodes activation step, where all nodes which do not form
    a complete chain will be inactive. This way a branch may be disabled by
    disabling one of its nodes. This process is used by the switch system,
    which allows to select one processing branch between several, and disables
    the unselected ones.

    **Pipeline steps**

    Pipelines may define execution steps: they are user-oriented groups of
    nodes that are to be run together, or disabled together for runtime
    They are intended to allow partial, or step-by-step execution. They do not
    work like the nodes enabling mechanism described above.

    Steps may be defined within the :py:meth:`pipeline_definition` method.
    See :py:meth:`add_pipeline_step`.

    Note also that pipeline steps only act at the highest level: if a
    sub-pipeline has disabled steps, they will not be taken into account in the
    higher level pipeline execution, because executing by steps a part of a
    sub-pipeline within the context of a higher one does generally not make
    sense.

    **Main methods**

    * :meth:`pipeline_definition`
    * :meth:`add_process`
    * :meth:`add_switch`
    * :meth:`add_custom_node`
    * :meth:`add_iterative_process`
    * :meth:`add_optional_output_switch`
    * :meth:`add_processes_selection`
    * :meth:`add_link`
    * :meth:`remove_link`
    * :meth:`export_parameter`
    * :meth:`autoexport_nodes_parameters`
    * :meth:`add_pipeline_step`
    * :meth:`define_pipeline_steps`
    * :meth:`define_groups_as_steps`
    * :meth:`remove_pipeline_step`
    * :meth:`enable_all_pipeline_steps`
    * :meth:`disabled_pipeline_steps_nodes`
    * :meth:`get_pipeline_step_nodes`
    * :meth:`find_empty_parameters`
    * :meth:`count_items`

    Attributes
    nodes: dict {node_name: node}
        a dictionary containing the pipeline nodes and where the pipeline node
        name is ''
    workflow_list: list
        a list of ordered nodes that can be executed
    workflow_repr: str
        a string representation of the workflow list <node_i>-><node_i+1>

    """

    #Methods
    #-------
    #pipeline_definition
    #add_trait
    #add_process
    #add_switch
    #add_link
    #remove_link
    #export_parameter
    #workflow_ordered_nodes
    #workflow_graph
    #update_nodes_and_plugs_activation
    #parse_link
    #parse_parameter
    #find_empty_parameters
    #count_items
    #define_pipeline_steps
    #add_pipeline_step
    #remove_pipeline_step
    #disabled_pipeline_steps_nodes
    #get_pipeline_step_nodes
    #enable_all_pipeline_steps
    #"""

    _doc_path = 'api/pipeline.html#pipeline'

    selection_changed = Event()
    
    # The default value for do_autoexport_nodes_parameters is stored in the
    # pipeline class. This makes it possible to change this default value
    # in derived classes (for instance in DynamicPipeline).
    do_autoexport_nodes_parameters = True
    
    # By default nodes_activation trait is hidden in user interface. Changing
    # this value to False will make it visible.
    hide_nodes_activation = True

    def __init__(self, autoexport_nodes_parameters=None, **kwargs):
        """ Initialize the Pipeline class

        autoexport_nodes_parameters: bool
            if True (default) nodes containing pipeline plugs are automatically
            exported.
        """
        # Inheritance
        super(Pipeline, self).__init__(**kwargs)
        super(Pipeline, self).add_trait(
            'nodes_activation',
            ControllerTrait(Controller(), hidden=self.hide_nodes_activation))

        # Class attributes
        # this one is only useful to maintain subprocesses/subpipelines life
        self.list_process_in_pipeline = []
        self.nodes_activation = Controller()
        self.nodes = SortedDictionary()
        self._invalid_nodes = set()
        self._skip_invalid_nodes = set()
        # Get node_position from the Pipeline class if it is
        # defined
        node_position = getattr(self,'node_position', None)
        if node_position:
            self.node_position = node_position.copy()
        else:
            self.node_position = {}
        
        ###############add by Irmage OM ######################### 
        node_dimension = getattr(self,'node_dimension', None)
        if node_dimension:
            self.node_dimension = node_dimension.copy()
        else:
            self.node_dimension = {}
        #########################################################
            
        self.pipeline_node = PipelineNode(self, '', self)
        self.nodes[''] = self.pipeline_node
        self.do_not_export = set()
        self.parent_pipeline = None
        self._disable_update_nodes_and_plugs_activation = 1
        self._must_update_nodes_and_plugs_activation = False
        self.pipeline_definition()

        self.workflow_repr = ""
        self.workflow_list = []

        if autoexport_nodes_parameters is None:
            autoexport_nodes_parameters = self.do_autoexport_nodes_parameters
        if autoexport_nodes_parameters:
            self.autoexport_nodes_parameters()

        # Refresh pipeline activation
        self._disable_update_nodes_and_plugs_activation -= 1
        self.update_nodes_and_plugs_activation()

    ##############
    # Methods    #

    def pipeline_definition(self):
        """ Define pipeline structure, nodes, sub-pipelines, switches, and
        links.

        This method should be overloaded in subclasses, it does nothing in the
        base Pipeline class.
        """
        pass

    def autoexport_nodes_parameters(self, include_optional=False):
        """ Automatically export nodes plugs to the pipeline.

        Some parameters can be explicitly preserved from exportation if they
        are listed in the pipeline "do_not_export" variable (list or set).

        include_optional: bool (optional)
            If True (the default), optional plugs are not exported
            Exception: optional output plugs of switches are exported
            (otherwise they are useless). It should probably be any single
            output plug of a node.
        """
        for node_name, node in six.iteritems(self.nodes):
            if node_name == "":
                continue
            for parameter_name, plug in six.iteritems(node.plugs):
                if parameter_name in ("nodes_activation", "selection_changed",
                                      "pipeline_steps", "visible_groups"):
                    continue
                if (((node_name, parameter_name) not in self.do_not_export and
                    ((plug.output and not plug.links_to) or
                     (not plug.output and not plug.links_from)) and
                    (include_optional
                     or (plug.output and isinstance(node, Switch))
                     or not self.nodes[node_name].get_trait(
                          parameter_name).optional))):

                    self.export_parameter(node_name, parameter_name)

    def add_trait(self, name, trait):
        """ Add a trait to the pipeline

        name: str (mandatory)
            the trait name
        trait: trait instance (mandatory)
            the trait we want to add
        """
        # Add the trait
        super(Pipeline, self).add_trait(name, trait)
        #self.get(name)

        # If we insert a user trait, create the associated plug
        if getattr(self, 'pipeline_node', False) and self.is_user_trait(trait):
            output = bool(trait.output)
            optional = bool(trait.optional)
            plug = Plug(output=output, optional=optional)
            self.pipeline_node.plugs[name] = plug
            plug.on_trait_change(self.update_nodes_and_plugs_activation,
                                 'enabled')

    def remove_trait(self, name):
        """ Remove a trait to the pipeline

        """
        if name in self.traits():
            trait = self.traits()[name]

        # If we remove a user trait, clear/remove the associated plug
            if (self.is_user_trait(trait) and
              name in self.pipeline_node.plugs):
                plug = self.pipeline_node.plugs[name]
                links_to_remove = []
            # use intermediary links_to_remove to avoid modifying the links set
            # while iterating on it...
                for link in plug.links_to:
                    dst = '%s.%s' % (link[0], link[1])
                    links_to_remove.append('%s->%s' % (name, dst))
                for link in plug.links_from:
                    src = '%s.%s' % (link[0], link[1])
                    links_to_remove.append('%s->%s' % (src, name))
                for link in links_to_remove:    
                    self.remove_link(link)
                del self.pipeline_node.plugs[name]

        # Remove the trait
        super(Pipeline, self).remove_trait(name)

    def reorder_traits(self, names):
        """ Reimplementation of :class:`~soma.controller.controller.Controller`
        method :meth:`~~soma.controller.controller.Controller.reorder_traits`
        so that we also reorder the pipeline node plugs.
        """
        pnames = names
        if 'nodes_activation' not in names:
            # keep nodes_activation first since it's normally the first added
            pnames = ['nodes_activation'] + list(names)
        super(Pipeline, self).reorder_traits(pnames)
        old_keys = self.pipeline_node.plugs.sortedKeys
        self.pipeline_node.plugs.sortedKeys \
            = [k for k in names if k in old_keys] \
              + [k for k in old_keys if k not in names]

    def _make_subprocess_context_name(self, name):
        ''' build full contextual name on process instance
        '''
        pipeline_name = getattr(self, 'context_name', None)
        if pipeline_name is None:
            pipeline_name = self.name
        context_name = '.'.join([pipeline_name, name])
        return context_name

    def _set_subprocess_context_name(self, process, name):
        ''' set full contextual name on process instance
        '''
        process.context_name = '.'.join([pipeline_name, name])
        # do it recursively if process is a pipeline
            todo = [process]
                cur_proc = todo.pop(0)
                for nname, node in six.iteritems(cur_proc.nodes):
                    if nname == '':
                        continue
                    sub_proc = getattr(node, 'process', None)
                    if sub_proc is not None:
                        sub_proc.context_name \
                            = '.'.join([cur_proc.context_name, nname])
                        if isinstance(sub_proc, Pipeline):
                            todo.append(sub_proc)

    def add_process(self, name, process, do_not_export=None,
                    make_optional=None, inputs_to_copy=None,
                    inputs_to_clean=None, skip_invalid=False, **kwargs):
        """ Add a new node in the pipeline

        **Note about invalid nodes:**

        A pipeline can typically offer alternatives (through a switch) to
        different algorithmic nodes, which may have different dependencies, or
        may be provided through external modules, thus can be missing. To handle
        this, Capsul can be telled that a process node can be invalid (or
        missing) without otherwise interfering the rest of the pipeline. This is
        done using the "skip_invalid" option. When used, the process to be added
        is tested, and if its instantiation fails, it will not be added in the
        pipeline, but will not trigger an error. Instead the missing node will
        be marked as "allowed invalid", and links and exports built using this
        node will silently do nothing. thus the pipeline will work normally,
        without the invalid node.

        Such nodes are generally gathered through a switch mechanism. However
        the switch inputs should be restricted to actually available nodes. The
        recommended method is to check that nodes have actually been added in
        the pipeline. Then links can be made normally as if the nodes were all
        present::

            self.add_process('method1', 'module1.Module1', skip_invalid=True)
            self.add_process('method2', 'module2.Module2', skip_invalid=True)
            self.add_process('method3', 'module3.Module3', skip_invalid=True)

            input_params = [n for n in ['method1', 'method2', 'method3']
                            if n in self.nodes]
            self.add_switch('select_method', input_params, 'output')

            self.add_link('method1.input->select_method.method1_switch_output')
            self.add_link('method2.input->select_method.method2_switch_output')
            self.add_link('method3.input->select_method.method3_switch_output')

        A last note about invalid nodes:

        When saving a pipeline (through the :class:`graphical editor
        <capsul.qt_gui.widgets.pipeline_developper_view.PipelineDeveloperView>`
        typically), missing nodes *will not be saved* because they are not
        actually in the pipeline. So be careful to save only pipelines with full
        features.

            the node name (has to be unique).
        process: Process (mandatory)
            the process we want to add. May be a string ('module.process'), a
            process instance or a class.
        do_not_export: list of str (optional)
            a list of plug names that we do not want to export.
        make_optional: list of str (optional)
        inputs_to_copy: list of str (optional)
            a list of item to copy.
        inputs_to_clean: list of str (optional)
            a list of temporary items.
        skip_invalid: bool
            if True, if the process is failing (cannot be instantiated), don't
            throw an exception but instead don't insert the node, and mark
            it as such in order to make add_link() to also silently do nothing.
            This option is useful for optional process nodes which may or may
            not be available depending on their dependencies, typically in a
            switch offering several alternative methods.
        """
        # Unique constrains
        make_optional = set(make_optional or [])
        do_not_export = set(do_not_export or [])
        do_not_export.update(kwargs)

        # Check the unicity of the name we want to insert
        if name in self.nodes:
            raise ValueError("Pipeline cannot have two nodes with the "
                             "same name : {0}".format(name))

        # It is necessary not to import study_config.process_instance at 
        # the module level because there are circular dependencies between
        # modules. For instance, Pipeline class needs get_process_instance
        # which needs create_xml_pipeline which needs Pipeline class.
        from capsul.study_config.process_instance import get_process_instance
        if skip_invalid:
            self._skip_invalid_nodes.add(name)
        # Create a process node
        try:
            process = get_process_instance(process,
                                           study_config=self.study_config,
                                           **kwargs)
            if skip_invalid:
                process = None
                self._invalid_nodes.add(name)
                return
            else:
                raise
        # set full contextual name on process instance
        self._set_subprocess_context_name(process, name)

        # Update the kwargs parameters values according to process
        # default values
        for k, v in six.iteritems(process.default_values):
            kwargs.setdefault(k, v)

        # Update the list of files item to copy
        if inputs_to_copy is not None and hasattr(process, "inputs_to_copy"):
            process.inputs_to_copy.extend(inputs_to_copy)
        if inputs_to_clean is not None and hasattr(process, "inputs_to_clean"):
            process.inputs_to_clean.extend(inputs_to_clean)

        # Create the pipeline node
            node = process.pipeline_node
            node.name = name
            node.pipeline = self
            process.parent_pipeline = weak_proxy(self)
        else:
            node = ProcessNode(self, name, process)
        self.nodes[name] = node

        # If a default value is given to a parameter, change the corresponding
        # plug so that it gets activated even if not linked
        for parameter_name in kwargs:
            if parameter_name in process.traits():
                node.plugs[parameter_name].has_default_value = True
                make_optional.add(parameter_name)

        # Change plug default properties
        for parameter_name in node.plugs:
            # Do not export plug
            if (parameter_name in do_not_export or
                    parameter_name in make_optional):
                self.do_not_export.add((name, parameter_name))

            # Optional plug
            if parameter_name in make_optional:
                node.plugs[parameter_name].optional = True
                process = getattr(node, 'process')
                if process is not None:
                    process.trait(parameter_name).optional = True
            # forbid_completion
            if process.trait(parameter_name).forbid_completion:
                self.propagate_metadata(node, parameter_name,
                                        {'forbid_completion': True})

        # Create a trait to control the node activation (enable property)
        self.nodes_activation.add_trait(name, Bool)
        setattr(self.nodes_activation, name, node.enabled)

        # Observer
        self.nodes_activation.on_trait_change(self._set_node_enabled, name)

        # Add new node in pipeline process list to keep its life
        self.list_process_in_pipeline.append(process)

    def remove_node(self, node_name):
        """ Remove a node from the pipeline
        """
        node = self.nodes[node_name]
        for plug_name, plug in six.iteritems(node.plugs):
            if not plug.output:
                for link_def in list(plug.links_from):
                    src_node, src_plug = link_def[:2]
                    link_descr = '%s.%s->%s.%s' \
                                 % (src_node, src_plug, node_name, plug_name)
                    self.remove_link(link_descr)
            else:
                for link_def in list(plug.links_to):
                    dst_node, dst_plug = link_def[:2]
                                 % (node_name, plug_name, dst_node, dst_plug)
        del self.nodes[node_name]
        if hasattr(node, 'process'):
            self.list_process_in_pipeline.remove(node.process)
            self.nodes_activation.on_trait_change(
                self._set_node_enabled, node_name, remove=True)
            self.nodes_activation.remove_trait(node_name)

    def add_iterative_process(self, name, process, iterative_plugs=None,
                              do_not_export=None, make_optional=None,
                              inputs_to_copy=None, inputs_to_clean=None,
                              **kwargs):
        """ Add a new iterative node in the pipeline.

        process: Process or str (mandatory)
            the process we want to add.
        iterative_plugs: list of str (optional)
            a list of plug names on which we want to iterate.
            If None, all plugs of the process will be iterated.
        """
        if isinstance(process, str):
            # It is necessary not to import study_config.process_instance at
            # the module level because there are circular dependencies between
            # modules. For instance, Pipeline class needs get_process_instance
            # which needs create_xml_pipeline which needs Pipeline class.
            from capsul.study_config.process_instance \
                import get_process_instance
            process = get_process_instance(process)
        if iterative_plugs is None:
            forbidden = set(['nodes_activation', 'selection_changed',
                             'pipeline_steps', 'visible_groups'])
            iterative_plugs = [pname for pname in process.user_traits()
                               if pname not in forbidden]

        from .process_iteration import ProcessIteration
        context_name = self._make_subprocess_context_name(name)
        self.add_process(
            name,
            ProcessIteration(process, iterative_plugs,
                              study_config=self.study_config,
                              context_name=context_name),
            do_not_export, make_optional, **kwargs)
        return

    def call_process_method(self, process_name, method,
                            *args, **kwargs):
        """ Call a method of a process previously added
        with add_process or add_iterative_process.

        process_name: str (mandatory)
            name given to the process node.
        method: str (mandatory)
            name of the method to call.
        """
        return getattr(self.nodes[process_name].process, method)(*args,
                                                                 **kwargs)
    
    def add_switch(self, name, inputs, outputs, export_switch=True,
                   make_optional=(), output_types=None, switch_value=None,
                   opt_nodes=None):
        """ Add a switch node in the pipeline

            name for the switch node (has to be unique)
        inputs: list of str (mandatory)
            names for switch inputs.
            Switch activation will select amongst them.
            Inputs names will actually be a combination of input and output,
            in the shape "input_switch_output".
            This behaviour is needed when there are several outputs, and thus
            several input groups.
        outputs: list of str (mandatory)
            names for outputs.
        export_switch: bool (optional)
            if True, export the switch trigger to the parent pipeline with
            ``name`` as parameter name
        make_optional: sequence (optional)
            list of optional outputs.
            These outputs will be made optional in the switch output. By
            default they are mandatory.
        output_types: sequence of traits (optional)
            If given, this sequence should have the same size as outputs. It
            will specify each switch output parameter type (as a standard
            trait). Input parameters for each input block will also have this
            type.
        switch_value: str (optional)
            Initial value of the switch parameter (one of the inputs names).
            Defaults to 1st input.
        opt_nodes: bool or list
            tells that switch values are node names, and some of them may be
            optional and missing. In such a case, missing nodes are not added
            as inputs. If a list is passed, then it is a list of node names
            which length should match the number of inputs, and which order
            tells nodes related to inputs (in case inputs names are not
            directly node names).

        Examples
        --------
        >>> pipeline.add_switch('group_switch', ['in1', 'in2'],
                                ['out1', 'out2'])

        will create a switch with 4 inputs and 2 outputs:
        inputs: "in1_switch_out1", "in2_switch_out1", "in1_switch_out2",
        "in2_switch_out2"
        outputs: "out1", "out2"

        See Also
        --------
        capsul.pipeline.pipeline_nodes.Switch
        """
            raise ValueError("Pipeline cannot have two nodes with the same "
                             "name: {0}".format(name))

        if opt_nodes:
            if opt_nodes is True:
                opt_nodes = inputs
            opt_inputs = list(zip(inputs, opt_nodes))
            # filter inputs
            inputs = [i for i, n in opt_inputs if n in self.nodes]
        # Create the node
        node = Switch(self, name, inputs, outputs, make_optional=make_optional,
                      output_types=output_types)
            node._optional_input_nodes = opt_inputs

        # Export the switch controller to the pipeline node
        if export_switch:
            self.export_parameter(name, "switch", name)

        if switch_value:
            node.switch = switch_value

        self._set_subprocess_context_name(node, name)
        study_config = getattr(self, 'study_config', None)
        if study_config:
            node.set_study_config(study_config)

    def add_optional_output_switch(self, name, input, output=None):
        """ Add an optional output switch node in the pipeline

        An optional switch activates or disables its input/output link
        according to the output value. If the output value is not None or
        Undefined, the link is active, otherwise it is inactive.

        This kind of switch is meant to make a pipeline output optional, but
        still available for temporary files values inside the pipeline.

        Ex:

        A.output -> B.input

        B.input is mandatory, but we want to make A.output available and
        optional in the pipeline outputs. If we directlty export A.output, then
        if the pipeline does not set a value, B.input will be empty and the
        pipeline run will fail.

        Instead we can add an OptionalOutputSwitch between A.output and
        pipeline.output. If pipeline.output is set a valid value, then A.output
        and B.input will have the same valid value. If pipeline.output is left
        Undefined, then A.output and B.input will get a temporary value during
        the run.

        Add an optional output switch node in the pipeline

        input: str (mandatory)
            name for switch input.
            Switch activation will select between it and a hidden input,
            "_none". Inputs names will actually be a combination of input and
            output, in the shape "input_switch_output".
        output: str (optional)
            name for output. Default is the switch name

        Examples
        --------
        >>> pipeline.add_optional_output_switch('out1', 'in1')
        >>> pipeline.add_link('node1.output->out1.in1_switch_out1')

        See Also
        --------
        capsul.pipeline.pipeline_nodes.OptionalOutputSwitch
        """

        if output is None:
            output = name
        node = OptionalOutputSwitch(self, name, input, output)

    def add_custom_node(self, name, node_type, parameters=None,
                        make_optional=(), do_not_export=None, **kwargs):
        """
        Inserts a custom node (Node subclass instance which is not a Process)
        in the pipeline.

        node_type: str or Node subclass or Node instance
            node type to be built. Either a class (Node subclass) or a Node
            instance (the node will be re-instantiated), or a string
            describing a module and class.
        parameters: dict or Controller or None
            configuration dict or Controller defining parameters needed to
            build the node. The controller should be obtained using the node
            class's `configure_node()` static method, then filled with the
            desired values.
            If not given the node is supposed to be built with no parameters,
            which will not work for every node type.
        make_optional: list or tuple
            parameters names to be made optional
        kwargs: default values of node parameters
        """
        # It is necessary not to import study_config.process_instance at
        from capsul.study_config.process_instance import get_node_instance
        node = get_node_instance(node_type, self, parameters, name=name,
                                 **kwargs)
        if node is None:
            raise ValueError(
                "could not build a Node of type '%s' with the given parameters"
                % node_type)

                trait = node.trait(parameter_name)
                    trait.optional = True

        return node

    def parse_link(self, link, check=True):
        """ Parse a link coming from export_parameter method.

        link: str
            the link description of the form
            'node_from.plug_name->node_to.plug_name'
        check: bool
            if True, check that the node and plug exist

        Returns
        -------
        output: tuple
            tuple containing the link description and instances

        Examples
        --------
        >>> Pipeline.parse_link("node1.plug1->node2.plug2")
        "node1", "plug1", <instance node1>, <instance plug1>,
        "node2", "plug2", <instance node2>, <instance plug2>

        For a pipeline node:

        >>> Pipeline.parse_link("plug1->node2.plug2")
        "", "plug1", <instance pipeline>, <instance plug1>,
        """
        # Split source and destination node descriptions
        source, dest = link.split("->")

        # Parse the source and destination parameters
        err = None
        try:
            source_node_name, source_plug_name, source_node, source_plug = \
                self.parse_parameter(source, check=check)
        except ValueError:
            err = sys.exc_info()
            source_node_name, source_plug_name, source_node, source_plug \
                = (None, None, None, None)
        try:
            dest_node_name, dest_plug_name, dest_node, dest_plug = \
                self.parse_parameter(dest, check=check)
            if err or (source_node is not None and source_plug is not None):
                raise
            dest_node_name, dest_plug_name, dest_node, dest_plug \
            err = None
        if err and dest_node is not None and dest_plug is not None and check:
            six.reraise(*err)

        return (source_node_name, source_plug_name, source_node, source_plug,
                dest_node_name, dest_plug_name, dest_node, dest_plug)

    def parse_parameter(self, name, check=True):
        """ Parse parameter of a node from its description.

        name: str
            the description plug we want to load 'node.plug'

        Returns
        -------
            tuple containing the plug description and instances
        """
        # Parse the plug description
        nodes = name.split('.')
        plug_name = nodes[-1]
        nodes.pop(-1)

        node = self.pipeline_node
        node_name = ''
        for pnode in nodes:
            node = node.process.nodes.get(pnode)
            node_name = pnode
            if node is None:
                if node_name in self._invalid_nodes:
                    node = None
                    plug = None
                    break
                else:
                    raise ValueError("{0} is not a valid node name".format(
                                    node_name))

        # Check if plug exists
        plug = None
        if node is not None:
            if plug_name not in node.plugs:
                if plug_name not in node.invalid_plugs:
                    # adhoc search: look for an invalid node which is the
                    # beginning of the plug name: probably an auto_exported one
                    # from an invalid node
                    err = True
                    if hasattr(node, 'process') \
                            and hasattr(node.process, '_invalid_nodes'):
                        invalid = node.process._invalid_nodes
                        for ip in invalid:
                            if plug_name.startswith(ip + '_'):
                                err = False
                                node.invalid_plugs.add(plug_name)
                                break
                    if err and check:
                            "'{0}' is not a valid parameter name for "
                            "node '{1}'".format(
                                plug_name, (node_name if node_name
                                            else "pipeline")))
            else:
                plug = node.plugs[plug_name]
        return node_name, plug_name, node, plug

    def add_link(self, link, weak_link=False, allow_export=False, value=None):
        """ Add a link between pipeline nodes.

        If the destination node is a switch, force the source plug to be not
        optional.

        link: str or list/tuple
            link description. Its shape should be:
            "node.output->other_node.input".
            If no node is specified, the pipeline itself is assumed.
            Alternatively the link can be
            (source_node, source_plug_name, dest_node, dest_plug_name)
        weak_link: bool
            this property is used when nodes are optional,
            the plug information may not be generated.
        allow_export: bool
            if True, if the link links from/to the pipeline node with a plug
            name which doesn't exist, the plug will be created, and the
            function will act exactly like export_parameter. This may be a more
            convenient way of exporting/connecting pipeline plugs to several
            nodes without having to export the first one, then link the others.
        value: any
            if given, set this value instead of the source plug value
        """
        check = True
        if allow_export:
            check = False
        if isinstance(link, six.string_types):
            # Parse the link
            (source_node_name, source_plug_name, source_node,
            source_plug, dest_node_name, dest_plug_name, dest_node,
            dest_plug) = self.parse_link(link, check=check)
        else:
            (source_node, source_plug_name, dest_node, dest_plug_name) = link
            source_plug = source_node.plugs[source_plug_name]
            dest_plug = dest_node.plugs[dest_plug_name]
            source_node_name = [k for k, n in six.iteritems(self.nodes)
                                if n is source_node][0]
            dest_node_name = [k for k, n in six.iteritems(self.nodes)
                              if n is dest_node][0]

            if source_plug is None:
                source_node.process.export_parameter(
                    dest_node_name, dest_plug_name, source_plug_name)
                return
            elif dest_plug is None:
                dest_node.process.export_parameter(
                    source_node_name, source_plug_name, dest_plug_name)
                return

        if source_node is None \
                or dest_node is None or source_plug is None \
                or dest_plug is None:
            # link from/to an invalid node
            return

        # Assure that pipeline plugs are not linked
        if (not source_plug.output and source_node is not self.pipeline_node):
              raise ValueError(
                  "Cannot link from an input plug: {0}".format(link))
        if (source_plug.output and source_node is self.pipeline_node):
            raise ValueError("Cannot link from a pipeline output "
                             "plug: {0}".format(link))
        if (dest_plug.output and dest_node is not self.pipeline_node):
            raise ValueError("Cannot link to an output plug: {0}".format(link))
        if (not dest_plug.output and dest_node is self.pipeline_node):
            raise ValueError("Cannot link to a pipeline input "

        # the destination of the link should not expect an already existing
        # file value, since it will come as an output from the source.
        trait = dest_node.get_trait(dest_plug_name)
        relax_exists_constraint(trait)

        # Propagate the plug value from source to destination
        if value is None:
            value = source_node.get_plug_value(source_plug_name)
        if value is not None:
            dest_node.set_plug_value(dest_plug_name, value)

        # Update plugs memory of the pipeline
        source_plug.links_to.add((dest_node_name, dest_plug_name, dest_node,
                                  dest_plug, weak_link))
        dest_plug.links_from.add((source_node_name, source_plug_name,
                                  source_node, source_plug, weak_link))

        # Set a connected_output property
        if (isinstance(dest_node, ProcessNode) and
                isinstance(source_node, ProcessNode)):
            source_trait = source_node.process.trait(source_plug_name)
            dest_trait = dest_node.process.trait(dest_plug_name)
            if source_trait.output and not dest_trait.output:
                dest_trait.connected_output = True

        # Propagate the description in case of destination switch node
        if isinstance(dest_node, Switch):
            source_trait = source_node.get_trait(source_plug_name)
            dest_trait = dest_node.trait(dest_plug_name)
            dest_trait.desc = source_trait.desc
            dest_node._switch_changed(getattr(dest_node, "switch"),
                                      getattr(dest_node, "switch"))

        # if completion is forbidden on source or dest, propagate the other
        # side
        forbid_completion \
            = source_node.get_trait(source_plug_name).forbid_completion
        if forbid_completion:
            self.propagate_metadata(source_node, source_plug_name,
                                    {'forbid_completion': True})
        elif trait.forbid_completion:
            self.propagate_metadata(dest_node, dest_plug_name,

        # Observer
        source_node.connect(source_plug_name, dest_node, dest_plug_name)
        dest_node.connect(dest_plug_name, source_node, source_plug_name)

    def remove_link(self, link):
        """ Remove a link between pipeline nodes

        """
            dest_plug) = self.parse_link(link)
        else:

        if source_node is None or dest_node is None or source_plug is None \
            return

        source_plug.links_to.discard((dest_node_name, dest_plug_name,
                                      dest_node, dest_plug, True))
                                      dest_node, dest_plug, False))
        dest_plug.links_from.discard((source_node_name, source_plug_name,
                                      source_node, source_plug, True))
                                      source_node, source_plug, False))

            if dest_trait.connected_output:
                dest_trait.connected_output = False  # FIXME

        # Observer
        source_node.disconnect(source_plug_name, dest_node, dest_plug_name)
        dest_node.disconnect(dest_plug_name, source_node, source_plug_name)

    def export_parameter(self, node_name, plug_name,
                         pipeline_parameter=None, weak_link=False,
                         is_enabled=None, is_optional=None,
                         allow_existing_plug=None):
        """ Export a node plug at the pipeline level.

        node_name: str (mandatory)
            the name of node containing the plug we want to export
        plug_name: str (mandatory)
            the node plug name we want to export
        pipeline_parameter: str (optional)
            the name to access this parameter at the pipeline level.
            Default None, the plug name is used
        weak_link: bool (optional)
            this property is used when nodes are weak,
            **FIXME:** what does it exactly mean ?
        is_enabled: bool (optional)
            a property to specify that it is not a user-parameter
            automatic generation)
        is_optional: bool (optional)
            sets the exported parameter to be optional
        allow_existing_plug:bool (optional)
            the same pipeline plug may be connected to several process plugs
        """
        # If a tuned name is not specified, used the plug name
        if not pipeline_parameter:
            pipeline_parameter = plug_name

        # Get the node and parameter
        node = self.nodes.get(node_name)
        if node is None and node_name in self._invalid_nodes:
            # export an invalid plug: mark it as invalid
            self.pipeline_node.invalid_plugs.add(pipeline_parameter)
            return

        # Make a copy of the trait
        source_trait = node.get_trait(plug_name)

        # Check if the plug name is valid
        if source_trait is None:
            raise ValueError("Node {0} ({1}) has no parameter "
                             "{2}".format(node_name, node.name, plug_name))

        # Check the pipeline parameter name is not already used
        if (pipeline_parameter in self.user_traits() and
                                               not allow_existing_plug is True):
                "Parameter '{0}' of node '{1}' cannot be exported to pipeline "
                "parameter '{2}'".format(
                    plug_name, node_name or 'pipeline_node',
                    pipeline_parameter))

        trait = self._clone_trait(source_trait)

        # Set user enabled parameter only if specified
        # Important because this property is automatically set during
        # the nipype interface wrappings
        if is_enabled is not None:
            trait.enabled = bool(is_enabled)

        # Change the trait optional property
        if is_optional is not None:
            trait.optional = bool(is_optional)

        # Now add the parameter to the pipeline
        if not pipeline_parameter in self.user_traits():
            self.add_trait(pipeline_parameter, trait)

        # Propagate the parameter value to the new exported one
        try:
            self.set_parameter(pipeline_parameter,
                               node.get_plug_value(plug_name))
        except traits.TraitError:
            pass

        # Do not forget to link the node with the pipeline node

            link_desc = "{0}.{1}->{2}".format(
                node_name, plug_name, pipeline_parameter)
            self.add_link(link_desc,  weak_link)
        else:
            link_desc = "{0}->{1}.{2}".format(
                pipeline_parameter, node_name, plug_name)
            self.add_link(link_desc, weak_link)

    def _set_node_enabled(self, node_name, is_enabled):
        """ Method to enable or disabled a node

            the node name
        is_enabled: bool (mandatory)
            the desired property
        """
        if node:
            node.enabled = is_enabled

    def propagate_metadata(self, node, param, metadata):
        """
        Set metadata on a node parameter, and propagate these values to the
        connected plugs.

        Typically needed to propagate the "forbid_completion" metadata to avoid
        manuyally set values to be overridden by completion.

        node may be a Node instance or a node name
        """
        if isinstance(node, str):
            node = self.nodes[node]
        todo = [(node, param, True)]
        done = set()
            node, param, force = todo.pop(0)
            done.add((node, param))

            # set metadata on node param trait
            trait = node.get_trait(param)
            modif = False
            for k, v in six.iteritems(metadata):
                if getattr(trait, k) != v:
                    setattr(trait, k, v)
                    modif = True

            if modif or force:
                # get connected plugs
                plug = node.plugs[param]
                for link in list(plug.links_from) + list(plug.links_to):
                    nn, pn, n, p, weak_link = link
                    if (n, pn) not in done:
                        todo.append((n, pn, False))

    def all_nodes(self):
        """ Iterate over all pipeline nodes including sub-pipeline nodes.

        Returns
        -------
        nodes: Generator of Node
            Iterates over all nodes
        """
        for node in six.itervalues(self.nodes):
            yield node
            if (isinstance(node, PipelineNode) and
               node is not self.pipeline_node):
                for sub_node in node.process.all_nodes():
                    if sub_node is not node:
                        yield sub_node

    def _check_local_node_activation(self, node):
        """ Try to activate a node and its plugs according to its
        state and the state of its direct neighbouring nodes.

        node: Node (mandatory)
            node to check

        Returns
        -------
        plugs_activated: list
            list of (plug_name,plug) containing all plugs that have been
            activated
        """
        plugs_activated = []
        # If a node is disabled, it will never be activated
        if node.enabled:
            # Try to activate input plugs
            node_activated = True
            if node is self.pipeline_node:
                # For the top-level pipeline node, all enabled plugs
                # are activated
                for plug_name, plug in six.iteritems(node.plugs):
                    if plug.enabled:
                        if not plug.activated:
                            plug.activated = True
                            plugs_activated.append((plug_name, plug))
            else:
                # Look for input plugs that can be activated
                    if plug.output:
                        # ignore output plugs
                        continue
                    if plug.enabled and not plug.activated:
                        if plug.has_default_value:
                        else:
                            # Look for a non weak link connected to an
                            # activated plug in order to activate the plug
                            for nn, pn, n, p, weak_link in plug.links_from:
                                if not weak_link and p.activated:
                                    plug.activated = True
                                    plugs_activated.append((plug_name, plug))
                                    break
                    # If the plug is not activated and is not optional the
                    # whole node is deactivated
                    if not plug.activated and not plug.optional:
                        node_activated = False
            if node_activated:
                node.activated = True
                # If node is activated, activate enabled output plugs
                    if plug.output and plug.enabled:
        return plugs_activated

    def _check_local_node_deactivation(self, node):
        """ Check plugs that have to be deactivated according to node
        activation state and to the state of its direct neighbouring nodes.

        Returns
        -------
        plugs_deactivated: list
            deactivated
        """
        def check_plug_activation(plug, links):
            # After the following for loop, plug_activated can have three
            # values:
            #  True  if there is a non weak link connected to an
            #        activated plug
            #  False if there are non weak links that are all connected
            #        to inactive plugs
            #  None if there is no non weak links
            plug_activated = None
            # weak_activation will be True if there is at least one
            # weak link connected to an activated plug
            weak_activation = False
            for nn, pn, n, p, weak_link in links:
                if weak_link:
                    weak_activation = (weak_activation or p.activated)
                else:
                    if p.activated:
                        plug_activated = True
                        break
                    else:
                        plug_activated = False
            if plug_activated is None:
                # plug is connected only with weak links therefore
                # they are used to define its activation state
                plug_activated = weak_activation
            return plug_activated

        plugs_deactivated = []
        # If node has already been  deactivated there is nothing to do
        if node.activated:
            deactivate_node = bool(
                [plug for plug in six.itervalues(node.plugs)
                 if plug.output])
            for plug_name, plug in six.iteritems(node.plugs):
                # Check all activated plugs
                try:
                    if plug.activated:
                        # A plug with a default value is always activated
                            continue
                        output = plug.output
                        if (isinstance(node, PipelineNode) and
                          node is not self.pipeline_node and output):
                            plug_activated = (
                                check_plug_activation(plug, plug.links_to) and
                                check_plug_activation(plug, plug.links_from))
                        else:
                            if node is self.pipeline_node:
                                output = not output
                            if output:
                                plug_activated = check_plug_activation(
                                    plug, plug.links_to)
                            else:
                                    plug, plug.links_from)

                        # Plug must be deactivated, record it in result and
                        # check if this deactivation also deactivate the node
                        if not plug_activated:
                            plug.activated = False
                            plugs_deactivated.append((plug_name, plug))
                            if not (plug.optional or
                                    node is self.pipeline_node):
                                node.activated = False
                                break
                finally:
                    # this must be done even if break or continue has been
                    # encountered
                    if plug.output and plug.activated:
                        deactivate_node = False
            if deactivate_node:
                node.activated = False
                        plug.activated = False
                        plugs_deactivated.append((plug_name, plug))
        return plugs_deactivated

    def delay_update_nodes_and_plugs_activation(self):
        if self.parent_pipeline is not None:
            # Only the top level pipeline can manage activations
            self.parent_pipeline.delay_update_nodes_and_plugs_activation()
            return
        if self._disable_update_nodes_and_plugs_activation == 0:
            self._must_update_nodes_and_plugs_activation = False
        self._disable_update_nodes_and_plugs_activation += 1

    def restore_update_nodes_and_plugs_activation(self):
            self.parent_pipeline.restore_update_nodes_and_plugs_activation()
            return
        if self._disable_update_nodes_and_plugs_activation == 0 and \
                self._must_update_nodes_and_plugs_activation:
            self.update_nodes_and_plugs_activation()

    def update_nodes_and_plugs_activation(self):
        """ Reset all nodes and plugs activations according to the current
        state of the pipeline (i.e. switch selection, nodes disabled, etc.).
        Activations are set according to the following rules.
        """
        if not hasattr(self, 'parent_pipeline'):
            # self is being initialized (the call comes from self.__init__).
            return
            self.parent_pipeline.update_nodes_and_plugs_activation()
            return
        if self._disable_update_nodes_and_plugs_activation:
            self._must_update_nodes_and_plugs_activation = True
            return

        debug = getattr(self, '_debug_activations', None)
        if debug:
            debug = open(debug, 'w')
            print(self.id, file=debug)

        # Remember all links that are inactive (i.e. at least one of the two
        # plugs is inactive) in order to execute a callback if they become
        # active (see at the end of this method)
        inactive_links = []
        for node in self.all_nodes():
            for source_plug_name, source_plug in six.iteritems(node.plugs):
                for nn, pn, n, p, weak_link in source_plug.links_to:
                    if not source_plug.activated or not p.activated:
                        inactive_links.append((node, source_plug_name,
                                               source_plug, n, pn, p))

        # Initialization : deactivate all nodes and their plugs
            node.activated = False
                plug.activated = False

        # Forward activation : try to activate nodes (and their input plugs)
        # and propagate activations neighbours of activated plugs

        # Starts iterations with all nodes
        nodes_to_check = set(self.all_nodes())
        iteration = 1
        while nodes_to_check:
            new_nodes_to_check = set()
            for node in nodes_to_check:
                node_activated = node.activated
                for plug_name, plug in self._check_local_node_activation(node):
                    if debug:
                        print('%d+%s:%s' % (
                            iteration, node.full_name, plug_name), file=debug)
                    for nn, pn, n, p, weak_link in \
                            plug.links_to.union(plug.links_from):
                        if not weak_link and p.enabled:
                            new_nodes_to_check.add(n)
                if (not node_activated) and node.activated:
                    if debug:
                        print('%d+%s' % (iteration, node.full_name),
                              file=debug)
            nodes_to_check = new_nodes_to_check
            iteration += 1

        # Backward deactivation : deactivate plugs that should not been
        # activated and propagate deactivation to neighbouring plugs
                # Test plugs deactivation according to their input/output
                # state
                test = self._check_local_node_deactivation(node)
                if test:
                    for plug_name, plug in test:
                        if debug:
                            print('%d-%s:%s' % (
                                iteration, node.full_name, plug_name),
                                file=debug)
                        for nn, pn, n, p, weak_link in \
                                plug.links_from.union(plug.links_to):
                            if p.activated:
                                new_nodes_to_check.add(n)
                    if not node.activated:
                        # If the node has been deactivated, force deactivation
                        # of all plugs that are still active and propagate
                        # this deactivation to neighbours
                        if node_activated and debug:
                            print('%d-%s' % (iteration, node.full_name),
                                  file=debug)
                        for plug_name, plug in six.iteritems(node.plugs):
                            if plug.activated:
                                plug.activated = False
                                if debug:
                                    print('%d=%s:%s' % (
                                        iteration, node.full_name, plug_name),
                                        file=debug)
                                for nn, pn, n, p, weak_link in \
                                        plug.links_from.union(plug.links_to):
                                    if p.activated:
                                        new_nodes_to_check.add(n)

        # Denis 2020/01/03: I don't understand the reason for hiding
        # parameters of inactive plugs: they still get a value (default or
        # forced). So I comment the following out until we make it clear why
        # this was done this way.
        #
        ## Update processes to hide or show their traits according to the
        ## corresponding plug activation
        #for node in self.all_nodes():
            #if isinstance(node, ProcessNode):
                #traits_changed = False
                #for plug_name, plug in six.iteritems(node.plugs):
                    #trait = node.process.trait(plug_name)
                    #if plug.activated:
                        #if getattr(trait, "hidden", False):
                            #trait.hidden = False
                            #traits_changed = True
                    #else:
                        #if not getattr(trait, "hidden", False):
                            #trait.hidden = True
                #if traits_changed:
                    #node.process.user_traits_changed = True

        # Execute a callback for all links that have become active.
        for node, source_plug_name, source_plug, n, pn, p in inactive_links:
            if (source_plug.activated and p.activated):
                value = node.get_plug_value(source_plug_name)
                node._callbacks[(source_plug_name, n, pn)](value)

        # Refresh views relying on plugs and nodes selection
            if isinstance(node, PipelineNode):
                node.process.selection_changed = True

    def workflow_graph(self, remove_disabled_steps=True,
                       remove_disabled_nodes=True):
        """ Generate a workflow graph

        Returns
        -------
        graph: topological_sort.Graph
            graph representation of the workflow from the current state of
            the pipeline
        remove_disabled_steps: bool (optional)
            When set, disabled steps (and their children) will not be included
            in the workflow graph.
            Default: True
        remove_disabled_nodes: bool (optional)
            When set, disabled nodes will not be included in the workflow
            graph.
        """

        def insert(pipeline, node_name, node, plug, dependencies, plug_name,
                   links, output=None):
            """ Browse the plug links and add the correspondings edges
            to the node.
            """

            if output is None:
                process = getattr(node, 'process', node)
                trait = process.trait(plug_name)
                output = trait.output
                if output:
                    if isinstance(trait.trait_type, (File, Directory)) \
                            and trait.input_filename is not False:
                        output = False
                    elif isinstance(trait.trait_type, List) \
                            and isinstance(trait.inner_traits[0],
                                          (File, Directory)) \

            # Main loop
            for (dest_node_name, dest_plug_name, dest_node, dest_plug,
                 weak_link) in plug.links_to:

                # Ignore the link if it is pointing to a node in a
                # sub-pipeline or in the parent pipeline
                if pipeline.nodes.get(dest_node_name) is not dest_node:
                    continue

                # Plug need to be activated
                if dest_node.activated:

                    # If plug links to an inert node (switch...), we need to
                    # address the node plugs
                    if isinstance(dest_node, ProcessNode):
                        dependencies.add((node_name, dest_node_name))
                        if output:
                            links.setdefault(dest_node, {})[dest_plug_name] \
                                = (node, plug_name)
                    elif isinstance(dest_node, Switch):
                        conn = dest_node.connections()
                        for c in conn:
                            if c[0] == dest_plug_name:
                                insert(pipeline, node_name, node,
                                       dest_node.plugs[c[1]],
                                       dependencies, plug_name, links, output)
                                break
                    else:
                        for switch_plug in six.itervalues(dest_node.plugs):
                            insert(pipeline, node_name, node, switch_plug,
                                   dependencies, plug_name, links, output)

        # Create a graph and a list of graph node edges
        graph = Graph()
        dependencies = set()
        links = {}

        if remove_disabled_steps:
            steps = getattr(self, 'pipeline_steps', Controller())
            disabled_nodes = set()
            for step, trait in six.iteritems(steps.user_traits()):
                if not getattr(steps, step):
                    disabled_nodes.update(
                        [self.nodes[node] for node in trait.nodes])

        # Add activated Process nodes in the graph

            # Do not consider the pipeline node
                continue

            # Select only active Process nodes
            if (node.activated or not remove_disabled_nodes) \
                    and isinstance(node, ProcessNode) \
                    and (not remove_disabled_steps
                         or node not in disabled_nodes):

                # If a Pipeline is found: the meta graph node parameter
                # contains a sub Graph
                if isinstance(node.process, Pipeline):
                    gnode = GraphNode(
                        node_name, node.process.workflow_graph(False))
                    gnode.meta.pipeline = node.process
                    graph.add_node(gnode)

                # If a Process or an iterative node is found: the meta graph
                # node parameter contains a list with one process node or
                # a dynamic structure that cannot be processed yet.
                else:
                    graph.add_node(GraphNode(node_name, [node]))

                # Add node edges

                    # Consider only active pipeline node plugs
                        insert(self, node_name, node, plug, dependencies,
                               plug_name, links)

        # Add edges to the graph
        for d in dependencies:
            if graph.find_node(d[0]) and graph.find_node(d[1]):
                graph.add_link(d[0], d[1])

        graph.param_links = links

        return graph

    def workflow_ordered_nodes(self, remove_disabled_steps=True):
        """ Generate a workflow: list of process node to execute

        Returns
        -------
        workflow_list: list of Process
            an ordered list of Processes to execute
        """
        graph = self.workflow_graph(remove_disabled_steps)

        # Start the topologival sort
        ordered_list = graph.topological_sort()

        def walk_workflow(wokflow, workflow_list):
            """ Recursive function to go through pipelines' graphs
            """
            # Go through all the workflows
            for sub_workflow in wokflow:

                # If we have already a flatten graph structure just add it
                if isinstance(sub_workflow[1], list):
                    workflow_list.extend(sub_workflow[1])

                # Otherwise we need to call the topological sort in order to
                # sort the graph and than flat the graph structure
                else:
                    flat_structure = sub_workflow[1].topological_sort()
                    walk_workflow(flat_structure, workflow_list)

        # Generate the output workflow representation
        self.workflow_repr = "->".join([x[0] for x in ordered_list])
        logger.debug("Workflow: {0}". format(self.workflow_repr))

        # Generate the final workflow by flattening graphs structures
        workflow_list = []
        walk_workflow(ordered_list, workflow_list)

        return workflow_list

    def _check_temporary_files_for_node(self, node, temp_files):
        """ Check temporary outputs and allocate files for them.

        Temporary files or directories will be appended to the temp_files list,
        and the node parameters will be set to temp file names.

        This internal function is called by the sequential execution,
        _run_process() (also used through __call__()).
        The pipeline state will be restored at the end of execution using
        _free_temporary_files().

        node: Node
            node to check temporary outputs on
        temp_files: list
            list of temporary files for the pipeline execution. The list will
            be modified (completed).
        """
        process = getattr(node, 'process', None)
        if process is not None and isinstance(process, NipypeProcess):
            #nipype processes do not use temporaries, they produce output
            # file names
            return

            value = node.get_plug_value(plug_name)
            if not plug.activated or not plug.enabled:
                continue
            trait = node.get_trait(plug_name)
            if not trait.output:
                continue
            if hasattr(trait, 'input_filename') \
                    and trait.input_filename is False:
                continue
            if hasattr(trait, 'inner_traits') \
                    and len(trait.inner_traits) != 0 \
                    and isinstance(trait.inner_traits[0].trait_type,
                                   (traits.File, traits.Directory)):
                if len([x for x in value if x in ('', traits.Undefined)]) == 0:
                    continue
            elif value not in (traits.Undefined, '') \
                    or ((not isinstance(trait.trait_type, traits.File)
                          and not isinstance(trait.trait_type, traits.Directory))
                         or len(plug.links_to) == 0):
                continue
            # check that it is really temporary: not exported
            # to the main pipeline
            if self.pipeline_node in [link[2]
                                      for link in plug.links_to]:
                # it is visible out of the pipeline: not temporary
                continue
            # if we get here, we are a temporary.
            if isinstance(value, list):
                if trait.inner_traits[0].trait_type is traits.Directory:
                    new_value = []
                    tmpdirs = []
                    for i in range(len(value)):
                        if value[i] in ('', traits.Undefined):
                            tmpdir = tempfile.mkdtemp(suffix='capsul_run')
                            new_value.append(tmpdir)
                            tmpdirs.append(tmpdir)
                        else:
                            new_value.append(value[i])
                    temp_files.append((node, plug_name, tmpdirs, value))
                    node.set_plug_value(plug_name, new_value)
                else:
                    tmpfiles = []
                    if trait.inner_traits[0].allowed_extensions:
                        suffix = 'capsul' + trait.allowed_extensions[0]
                    else:
                        suffix = 'capsul'
                            tmpfile = tempfile.mkstemp(suffix=suffix)
                            tmpfiles.append(tmpfile[1])
                            os.close(tmpfile[0])
                            new_value.append(tmpfile[1])
                        else:
                    temp_files.append((node, plug_name, tmpfiles, value))
            else:
                if trait.trait_type is traits.Directory:
                    tmpdir = tempfile.mkdtemp(suffix='capsul_run')
                    temp_files.append((node, plug_name, tmpdir, value))
                    node.set_plug_value(plug_name, tmpdir)
                else:
                    if trait.allowed_extensions:
                    else:
                    tmpfile = tempfile.mkstemp(suffix=suffix)
                    node.set_plug_value(plug_name, tmpfile[1])
                    os.close(tmpfile[0])
                    temp_files.append((node, plug_name, tmpfile[1], value))

    def _free_temporary_files(self, temp_files):
        """ Delete and reset temp files after the pipeline execution.

        This internal function is called at the end of _run_process()
        (sequential execution)
        """
        #
        for node, plug_name, tmpfiles, value in temp_files:
            node.set_plug_value(plug_name, value)
            if not isinstance(tmpfiles, list):
                tmpfiles = [tmpfiles]
            for tmpfile in tmpfiles:
                if os.path.isdir(tmpfile):
                    try:
                        shutil.rmtree(tmpfile)
                    except OSError:
                        pass
                else:
                    try:
                        os.unlink(tmpfile)
                        pass
                # handle additional files (.hdr, .minf...)
                # TODO
                if os.path.exists(tmpfile + '.minf'):
                    try:
                        os.unlink(tmpfile + '.minf')
                        pass

    def _run_process(self):
        '''
        Pipeline execution is managed by StudyConfig class.
        This method must not be called.
        '''
        raise NotImplementedError('Pipeline execution is managed by '
            'StudyConfig class. This method must not be called.')

    def find_empty_parameters(self):
        """ Find internal File/Directory parameters not exported to the main
        input/output parameters of the pipeline with empty values. This is
        meant to track parameters which should be associated with temporary
        files internally.

        Returns
        -------
        list
            Each element is a list with 3 values: node, parameter_name,
            optional
        """
        empty_params = []
        # walk all activated nodes, recursively
        nodes = [(node_name, node)
                 for node_name, node in six.iteritems(self.nodes)
                 if node_name != '' and node.enabled and node.activated]
        while nodes:
            node_name, node = nodes.pop(0)
            if hasattr(node, 'process'):
                process = node.process
                if isinstance(process, Pipeline):
                    nodes += [(cnode_name, cnode)
                        for cnode_name, cnode in six.iteritems(process.nodes)
                        if cnode_name != '' and cnode.enabled
                        and cnode.activated]
            else:
                process = node
            # check output plugs; input ones don't work with generated
            # temporary files (unless they are connected with an output one,
            # which will do the job)
                if not plug.enabled or not plug.output or \
                        (not plug.activated and plug.optional):
                    continue
                parameter = process.trait(plug_name)
                if hasattr(parameter, 'inner_traits') \
                        and len(parameter.inner_traits) != 0:
                    # list trait
                    t = parameter.inner_traits[0]
                    if not isinstance(t.trait_type, (File, Directory)):
                        continue
                elif not isinstance(parameter.trait_type, (File, Directory)) \
                        or (parameter.output
                            and parameter.input_filename is False):
                    # a file with its filename as an output is OK
                    continue
                value = getattr(process, plug_name)
                    if len(value) == 0 \
                            or len([item for item in value
                                    if item in ('', traits.Undefined)]) == 0:
                        continue # non-empty list or all values non-empty
                    # here we have null values
                elif value != '' and value is not traits.Undefined:
                    continue # non-null value: not an empty parameter.
                optional = bool(parameter.optional)
                valid = True
                links = list(plug.links_from.union(plug.links_to))
                if len(links) == 0:
                    if optional:
                        # an optional, non-connected output can stay empty
                        continue
                # check where this plug is linked
                while links:
                    link = links.pop(0)
                    oplug = link[3]
                        if link[2] == self.nodes['']:
                            # linked to the main node: keep it as is
                            valid = False
                            break
                        if hasattr(link[2], 'process'):
                            lproc = link[2].process
                            ltrait = lproc.trait(link[1])
                            if ltrait.output \
                                    and ltrait.input_filename is False:
                                # connected to an output file which filename
                                # is actually an output: it will be generated
                                # by the process, thus is not a temporary
                                valid = False
                                break
                        # linked to an output plug of an intermediate pipeline:
                        # needed only if this pipeline plug is used later,
                        # or mandatory
                        if oplug.optional:
                            links += oplug.links_to
                    optional &= bool(oplug.optional)
                if valid:
                    empty_params.append((node, plug_name, optional))
        return empty_params

    def count_items(self):
        """ Count pipeline items to get its size.

        Returns
        -------
        items: tuple
            (nodes_count, processes_count, plugs_count, params_count,
            links_count, enabled_nodes_count, enabled_procs_count,
            enabled_links_count)
        """
        nodes = list(self.nodes.values())
        plugs_count = 0
        params_count = len([param
            for param_name, param in six.iteritems(self.user_traits())
            if param_name not in ('nodes_activation', 'selection_changed')])
        nodes_count = 0
        links_count = 0
        procs = set()
        nodeset = set()
        enabled_nodes_count = 0
        enabled_procs_count = 0
        enabled_links_count = 0
            node = nodes.pop(0)
            nodeset.add(node)
            nodes_count += 1
            if node.enabled and node.activated:
                enabled_nodes_count += 1
            plugs_count += len(node.plugs)
            links_count += sum([len(plug.links_to) + len(plug.links_from)
                for plug in six.itervalues(node.plugs)])
            enabled_links_count += sum(
                [len([pend for pend in plug.links_to
                        if pend[3].enabled and pend[3].activated])
                    + len([pend for pend in plug.links_from
                    for plug in six.itervalues(node.plugs)
                    if plug.enabled and plug.activated])
            if hasattr(node, 'nodes'):
                sub_nodes = [sub_node
                    for sub_node in node.nodes.values()
                    if sub_node not in nodeset and sub_node not in nodes]
                nodes += sub_nodes
            elif hasattr(node, 'process'):
                if node.process in procs:
                    continue
                procs.add(node.process)
                if node.enabled and node.activated:
                    enabled_procs_count += 1
                params_count += len([param
                    for param_name, param
                    in six.iteritems(node.process.user_traits())
                    if param_name not in (
                        'nodes_activation', 'selection_changed')])
                if hasattr(node.process, 'nodes'):
                    sub_nodes = [sub_node
                        for sub_node in node.process.nodes.values()
                        if sub_node not in nodeset and sub_node not in nodes]
                    nodes += sub_nodes
            elif hasattr(node, 'user_traits'):
                    for param_name, param in six.iteritems(node.user_traits())
                        'nodes_activation', 'selection_changed', 'activated',
                        'enabled', 'name')])
        return nodes_count, len(procs), plugs_count, params_count, \
            links_count, enabled_nodes_count, enabled_procs_count, \
            enabled_links_count

    def pipeline_state(self):
        """ Return an object composed of basic Python objects that contains
        the whole structure and state of the pipeline. This object can be
        given to compare_to_state method in order to get the differences with
        a previously stored state. This is typically used in tests scripts.

        Returns
        -------
        pipeline_state: dictionary
            todo
        """
        result = {}
            plugs_list = []
            node_dict = dict(name=node.name,
                             enabled=node.enabled,
                             activated=node.activated,
                             plugs=plugs_list)
            result[node.full_name] = node_dict
                links_to_dict = {}
                links_from_dict = {}
                plug_dict = dict(enabled=plug.enabled,
                                 activated=plug.activated,
                                 output=plug.output,
                                 optional=plug.optional,
                                 has_default_value=plug.has_default_value,
                                 links_to=links_to_dict,
                                 links_from=links_from_dict)
                plugs_list.append((plug_name, plug_dict))
                for nn, pn, n, p, weak_link in plug.links_to:
                    link_name = '%s:%s' % (n.full_name, pn)
                    links_to_dict[link_name] = weak_link
                for nn, pn, n, p, weak_link in plug.links_from:
                    links_from_dict[link_name] = weak_link
        return result

    def compare_to_state(self, pipeline_state):
        """ Returns the differences between this pipeline and a previously
        recorded state.

        Returns
        -------
        differences: list
            each element is a human readable string explaining one difference
            (e.g. 'node "my_process" is missing')
        """
        result = []
        
        def compare_dict(ref_dict, other_dict):
            for ref_key, ref_value in six.iteritems(ref_dict):
                if ref_key not in other_dict:
                    yield '%s = %s is missing' % (ref_key, repr(ref_value))
                else:
                    other_value = other_dict.pop(ref_key)
                    if ref_value != other_value:
                        yield '%s = %s differs from %s' % (ref_key,
                                                           repr(ref_value),
                                                           repr(other_value))
            for other_key, other_value in six.iteritems(other_dict):
                yield '%s=%s is new' % (other_key, repr(other_value))

        pipeline_state = deepcopy(pipeline_state)
            node_name = node.full_name
            node_dict = pipeline_state.pop(node_name, None)
            if node_dict is None:
                result.append('node "%s" is missing' % node_name)
            else:
                plugs_list = OrderedDict(node_dict.pop('plugs'))
                result.extend('in node "%s": %s' % (node_name, i) for i in
                              compare_dict(dict(name=node.name,
                                                enabled=node.enabled,
                                                activated=node.activated),
                                           node_dict))
                ref_plug_names = list(node.plugs)
                other_plug_names = list(plugs_list.keys())
                if ref_plug_names != other_plug_names:
                    if sorted(ref_plug_names) == sorted(other_plug_names):
                        result.append('in node "%s": plugs order = %s '
                                      'differs from %s' %
                                      (node_name, repr(ref_plug_names),
                                       repr(other_plug_names)))
                    else:
                        result.append('in node "%s": plugs list = %s '
                        # go to next node
                        continue
                    plug_dict = plugs_list[plug_name]
                    links_to_dict = plug_dict.pop('links_to')
                    links_from_dict = plug_dict.pop('links_from')
                    result.extend('in plug "%s:%s": %s' %
                        (node_name,plug_name,i) for i in
                        compare_dict(dict(enabled=plug.enabled,
                                          activated=plug.activated,
                                          output=plug.output,
                                          optional=plug.optional,
                                          has_default_value=
                                              plug.has_default_value),
                                          plug_dict))
                    for nn, pn, n, p, weak_link in plug.links_to:
                        link_name = '%s:%s' % (n.full_name, pn)
                        if link_name not in links_to_dict:
                            result.append('in plug "%s:%s": missing link to %s'
                                          % (node_name, plug_name, link_name))
                        else:
                            other_weak_link = links_to_dict.pop(link_name)
                            if weak_link != other_weak_link:
                                result.append('in plug "%s:%s": link to %s is'
                                              '%sweak' % (node_name, plug_name,
                                                          link_name, (' not'
                                                          if weak_link else 
                                                          '')))
                    for link_name, weak_link in six.iteritems(links_to_dict):
                        result.append('in plug "%s:%s": %slink to %s is new' %
                            (node_name,plug_name, (' weak' if weak_link else 
                            ''),link_name))
                    for nn, pn, n, p, weak_link in plug.links_from:
                        if link_name not in links_from_dict:
                            result.append('in plug "%s:%s": missing link from '
                                          '%s' % (node_name,
                                                  plug_name, link_name))
                        else:
                            other_weak_link = links_from_dict.pop(link_name)
                                result.append('in plug "%s:%s": link from %s '
                                              'is%sweak' % (node_name, 
                                                            plug_name,
                                                            link_name,(' not'
                                                            if weak_link else
                                                            '')))
                    for link_name, weak_link in six.iteritems(links_from_dict):
                        result.append('in plug "%s:%s": %slink from %s is new'
                                      % (node_name,plug_name,(' weak' if
                                          weak_link else ''),link_name))

        for node_name in pipeline_state:
            result.append('node "%s" is new' % node_name)

    def install_links_debug_handler(self, log_file=None, handler=None,
                                    prefix=''):
        """ Set callbacks when traits value change, and follow plugs links to
        debug links propagation and problems in it.

        log_file: str (optional)
            file-like object to write links propagation in.
            If none is specified, a temporary file will be created for it.
        handler: function (optional)
            Callback to be processed for debugging. If none is specified, the
            default pipeline debugging function will be used. This default
            handler prints traits changes and links to be processed in the
            log_file.
            The handler function will receive a prefix string, a node,
            and traits parameters, namely the object (process) owning the
            changed value, the trait name and value in this object.
        prefix: str (optional)
            prefix to be prepended to traits names, typically the parent
            pipeline full name

        Returns
        -------
        log_file: the file object where events will be written in
        """

        if log_file is None:
            log_file_s = tempfile.mkstemp()
            class AutodeDelete(object):
                def __init__(self, file_object):
                    self.file_object = file_object
                def __del__(self):
                    try:
                        self.file_object.close()
                    except IOError:
                        pass
                    try:
                        os.unlink(self.file_object.name)
                        pass
            os.close(log_file_s[0])
            log_file = open(log_file_s[1], 'w')
            self._log_file_del = AutodeDelete(log_file)

        self._link_debugger_file = log_file
        if prefix != '' and not prefix.endswith('.'):
            prefix = prefix + '.'
        # install handler on nodes
            node_prefix = prefix + node_name
            if node_prefix != '' and not node_prefix.endswith('.'):
                node_prefix += '.'
            if handler is None:
                custom_handler = node._value_callback_with_logging
            else:
                custom_handler = handler
            sub_process = None
            sub_pipeline = False
                sub_process = node.process
            if hasattr(sub_process, 'nodes') and sub_process is not self:
                sub_pipeline = sub_process
            if sub_pipeline:
                sub_pipeline.install_links_debug_handler(
                    log_file=log_file,
                    handler=handler,
                    prefix=node_prefix)
            else:
                # replace all callbacks
                callbacks = list(node._callbacks.items())
                for element, callback in callbacks:
                    source_plug_name, dest_node, dest_plug_name = element
                    value_callback = SomaPartial(
                        custom_handler, log_file, node_prefix,
                        source_plug_name,
                        dest_node, dest_plug_name)
                    node.remove_callback_from_plug(source_plug_name, callback)
                    node._callbacks[element] = value_callback
                    node.set_callback_on_plug(source_plug_name, value_callback)

        return log_file

    def uninstall_links_debug_handler(self):
        """ Remove links debugging callbacks set by install_links_debug_handler
        """

                sub_pipeline.uninstall_links_debug_handler()
            else:
                for element, callback in list(node._callbacks.items()):
                        node._value_callback, source_plug_name,

        if hasattr(self, '_link_debugger_file'):
            del self._link_debugger_file
        if hasattr(self, '_log_file_del'):
            del self._log_file_del

    def define_pipeline_steps(self, steps):
        '''Define steps in the pipeline.
        Steps are pipeline portions that form groups, and which can be enabled
        or disabled on a runtime basis (when building workflows).

        Once steps are defined, their activation may be accessed through the
        "step" trait, which has one boolean property for each step:

        Ex:

        ::

            steps = OrderedDict()
            steps['preprocessings'] = [
                'normalization',
                'bias_correction',
                'histo_analysis']
            steps['brain_extraction'] = [
                'brain_segmentation',
                'hemispheres_split']
            pipeline.define_pipeline_steps(steps)

        >>> print(pipeline.pipeline_steps.preprocessings)
        True

        >>> pipeline.pipeline_steps.brain_extraction = False

        See also add_pipeline_step()

        steps: dict or preferably OrderedDict or SortedDictionary (mandatory)
            The steps dict keys are steps names, the values are lists of nodes
            names forming the step.
        '''
        for step_name, nodes in six.iteritems(steps):
            self.add_pipeline_step(step_name, nodes)

    def add_pipeline_step(self, step_name, nodes, enabled=True):
        '''Add a step definition to the pipeline (see also define_steps).

        Steps are groups of pipeline nodes, which may be disabled at runtime.
        They are normally defined in a logical order regarding the workflow
        streams. They are different from pipelines in that steps are purely
        virtual groups, they do not have parameters.

        Disabling a step acts differently as the pipeline node activation: 
        other nodes are not inactivated according to their dependencies.
        Instead, those steps are not run.

        step_name: string (mandatory)
            name of the new step
        nodes: list or sequence
            nodes contained in the step (Node instances)
        enabled: bool (optional)
            initial state of the step
        '''
        if 'pipeline_steps' not in self.user_traits():
            super(Pipeline, self).add_trait(
                'pipeline_steps',
                ControllerTrait(Controller(), desc=
                    'Steps are groups of pipeline nodes, which may be '
                    'disabled at runtime. They are normally defined in a '
                    'logical order regarding the workflow streams. They are '
                    'different from sub-pipelines in that steps are purely '
                    'virtual groups, they do not have parameters. To activate '
                    'or disable a step, just do:\n'
                    'pipeline.steps.my_step = False\n'
                    '\n'
                    'To get the nodes list in a step:\n'
                    'pipeline.get_step_nodes("my_step")'))
            self.trait('pipeline_steps').expanded = False
            self.trait('pipeline_steps').optional = True
            self.pipeline_steps = Controller()
        self.pipeline_steps.add_trait(step_name, Bool(nodes=nodes))
        trait = self.pipeline_steps.trait(step_name)
        setattr(self.pipeline_steps, step_name, enabled)

    def remove_pipeline_step(self, step_name):
        '''Remove the given step
        '''
        if 'pipeline_steps' in self.user_traits():
            self.pipeline_steps.remove_trait(step_name)

    def disabled_pipeline_steps_nodes(self):
        '''List nodes disabled for runtime execution

        Returns
        -------
        disabled_nodes: list
            list of pipeline nodes (Node instances) which will not run in
            a workflow created from this pipeline state.
        '''
        steps = getattr(self, 'pipeline_steps', Controller())
        disabled_nodes = []
        for step, trait in six.iteritems(steps.user_traits()):
            if not getattr(steps, step, True):
                # disabled step
                nodes = trait.nodes
                disabled_nodes.extend([self.nodes[node] for node in nodes])
        return disabled_nodes

    def get_pipeline_step_nodes(self, step_name):
        '''Get the nodes in the given pipeline step
        '''
        return self.pipeline_steps.trait(step_name).nodes

    def enable_all_pipeline_steps(self):
        '''Set all defined steps (using add_step() or define_steps()) to be
        enabled. Useful to reset the pipeline state after it has been changed.
        '''
            setattr(steps, step, True)

    def _change_processes_selection(self, selection_name, selection_group):
        self.delay_update_nodes_and_plugs_activation()
        for group, processes in \
                six.iteritems(self.processes_selection[selection_name]):
            enabled = (group == selection_group)
            for node_name in processes: 
                self.nodes[node_name].enabled = enabled
        self.restore_update_nodes_and_plugs_activation()

    def add_processes_selection(self, selection_parameter, selection_groups,
                                value=None):
        '''Add a processes selection switch definition to the pipeline.

        Selectors are a "different" kind of switch: one pipeline node set in a
        group is enabled, the others are disabled.

        The selector has 2 levels:

        selection_parameter selects a group.

        A group contains a set of nodes which will be activated together.
        Groups are mutually exclusive.

        selection_parameter: string (mandatory)
            name of the selector parameter: the parameter is added in the
            pipeline, and its value is the name of the selected group.
        selection_groups: dict or OrderedDict
            nodes groups contained in the selector : {group_name: [Node names]}
        value: str (optional)
            initial state of the selector (default: 1st group)
        '''
        self.add_trait(selection_parameter, Enum(*selection_groups))
        self.nodes[''].plugs[selection_parameter].has_default_value = True
        self.user_traits_changed = True
        self.processes_selection = getattr(self, 'processes_selection', {})
        self.processes_selection[selection_parameter] = selection_groups
        self.on_trait_change(self._change_processes_selection,
                             selection_parameter)
        self._change_processes_selection(selection_parameter,
                                         getattr(self, selection_parameter))
            setattr(self, selection_parameter, value)

    def get_processes_selections(self):
        '''Get process_selection groups names (corresponding to selection
        parameters on the pipeline)
        '''
        if not hasattr(self, 'processes_selection'):
            return []
        return list(self.processes_selection.keys())

    def get_processes_selection_groups(self, selection_parameter):
        '''Get groups names involved in a processes selection switch
        '''
        return self.processes_selection[selection_parameter]

    def get_processes_selection_nodes(self, selection_parameter, group):
        '''Get nodes names involved in a processes selection switch with
        value group
        '''
        return self.processes_selection.get(selection_parameter, {}).get(group)

    def set_study_config(self, study_config):
        ''' Set a StudyConfig for the process.
        Note that it can only be done once: once a non-null StudyConfig has
        been assigned to the process, it should not change.
        '''
        super(Pipeline, self).set_study_config(study_config)
            if node_name != "":
                node.set_study_config(study_config)

    def define_groups_as_steps(self, exclusive=True):
        ''' Define parameters groups according to which steps they are
        connected to.

        exclusive: bool (optional)
            if True, a parameter is assigned to a single group, the first step
            it is connected to. If False, a parameter is assigned all steps
            groups it is connected to.
        '''
        steps = getattr(self, 'pipeline_steps', None)
        if not steps:
            return
        inv_steps = {}
        steps_priority = {}
        p = 0
            nodes = trait.nodes
            steps_priority[step] = p
            p += 1
            for node in nodes:
                inv_steps[node] = step

        if not self.trait('visible_groups'):
            # add a trait without a plug
            Controller.add_trait(self, 'visible_groups', Set())
            self.trait('visible_groups').optional = True
        plugs = self.pipeline_node.plugs
        for param, trait in six.iteritems(self.user_traits()):
            plug = plugs.get(param)
            if not plug:
                continue
            else:
            groups = []
                node_name = link[0]
                step = inv_steps.get(node_name)
                if step and step not in groups:
                    groups.append(step)
            if groups:
                groups = sorted(groups, key=lambda x: steps_priority[x])
                if exclusive:
                    groups = [groups[0]]
                trait.groups = groups

    def check_requirements(self, environment='global', message_list=None):
        '''
        Reimplementation for pipelines of
        :meth:`capsul.process.process.Process.check_requirements <Process.check_requirements>`

        A pipeline will return a list of unique configuration values.
        '''
        # start with pipeline-level requirements
        conf = super(Pipeline, self).check_requirements(
            environment, message_list=message_list)
        if conf is None:
        confs = []
        if conf:
            confs.append(conf)
        from capsul.pipeline import pipeline_tools
        success = True
        for key, node in six.iteritems(self.nodes):
                continue
            if pipeline_tools.is_node_enabled(self, key, node):
                conf = node.check_requirements(
                    message_list=message_list)
                if conf is None:
                    # requirement failed
                    if message_list is None:
                        # return immediately
                        return None
                    else:
                        success = False
                else:
                    if conf != {} and conf not in confs:
                        if isinstance(conf, list):
                            confs += [c for c in conf if c not in confs]
                        else:
                            confs.append(conf)
        if success:
            return confs
        else:

    def rename_node(self, old_node_name, new_node_name):
        '''
        Change the name of the selected node and updates the pipeline.

        old_node_name: str
            old node name
        new_node_name: str
            new node name
        '''
        if new_node_name in list(self.nodes.keys()):
            raise ValueError("Node name already in pipeline")

        else:

            node = self.nodes[old_node_name]

            # Removing links of the selected node and copy
            # the origin/destination
            links_to_copy = []
            for parameter, plug in six.iteritems(node.plugs):
                if plug.output:
                    for (dest_node_name, dest_parameter, dest_node, dest_plug,
                        weak_link) in plug.links_to:
                        slinks = dest_plug.links_from
                        slinks.remove((old_node_name, parameter, node, plug,
                                      weak_link))
                        slinks.add((new_node_name, parameter, node, plug,
                                    weak_link))

                else:
                        weak_link) in plug.links_from:
                        slinks = dest_plug.links_to

            # change the node entry with the new name and delete the former
            self.nodes[new_node_name] = node
            del self.nodes[old_node_name]

            # look for the node in the pipeline_steps, if any
            steps = getattr(self, 'pipeline_steps', None)
            if steps:
                for step, trait in six.iteritems(steps.user_traits()):
                    if old_node_name in trait.nodes:
                        trait.nodes = [n if n != old_node_name
                                          else new_node_name
                                       for n in trait.nodes]

            # nodes positions and dimensions
            if old_node_name in getattr(self, 'node_position', {}):
                self.node_position[new_node_name] \
                    = self.node_position[old_node_name]
                del self.node_position[old_node_name]
            if old_node_name in getattr(self, 'node_dimension', {}):
                self.node_dimension[new_node_name] \
                    = self.node_dimension[old_node_name]
                del self.node_dimension[old_node_name]

'''
This module is an internal machinery, the user needs not to know it and bother about it. In brief it provide helper functions to build a pipeline from an IO serialization.

Classes
=======
:class:`PipelineConstructor`
----------------------------
:class:`ConstructedPipeline`
'''

import xml.etree.cElementTree as ET

from capsul.process.xml import string_to_value
from capsul.process.process import ProcessMeta

from traits.api import Undefined, Directory

class PipelineConstructor(object):
    """
    A PipelineConstructor is used to build a new Pipeline class and dynamically
    constructs its contents, for instances by adding processes nodes and
    creating links between parameters. This class must be used whenever one
    wants to create a pipeline from an external source (i.e. a source that is
    not an installed Python module). For instance, the creation of a pipeline
    from an XML representation uses a PipelineConstructor.
    
    `pipeline`: ConstructedPipeline derived class
        the constructed Pipeline class
    """
    
    def __init__(self, module, name):
        """
        Initialize a new empty ConstructedPipeline class that is ready to be
        customized by calling methods of the PipelineConstructor. When pipeline
        creation is done, the pipeline class can be accessed with the
        `pipeline` attribute.
        
        module: str (mandatory)
            name of the module for the created Pipeline class.
            name of the created Pipeline class.
        """
        class_kwargs = {
            '__module__': module,
            '_pipeline_definition_calls': [],
            'do_autoexport_nodes_parameters': False,
            'node_position': {}
        }
        self.pipeline = type(name, (ConstructedPipeline,), class_kwargs)
        # self.__calls is simply a shortcut to 
        # self.pipeline._pipeline_definition_calls
        self._calls = self.pipeline._pipeline_definition_calls
    
    def add_process(self, *args, **kwargs):
        """Adds a new process to the pipeline.
        """
        self._calls.append(('add_process', args, kwargs))
    
    def add_iterative_process(self, *args, **kwargs):
        """Adds a new iterative process to the pipeline.
        """
        self._calls.append(('add_iterative_process', args, kwargs))
    
    def call_process_method(self, *args, **kwargs):
        with add_process or add_itConstructedPipelineerative_process.
        """
        self._calls.append(('call_process_method', args, kwargs))

    def add_link(self, *args, **kwargs):
        """Add a link between pipeline processes.
        """
        self._calls.append(('add_link', args, kwargs))

    def add_subpipeline_link(self, *args, **kwargs):
        """Add a link between subpipeline processes.
        """
        self._calls.append(('add_subpipeline_link', args, kwargs))

    def export_parameter(self, *args, **kwargs):
        """Export an internal parameter to the pipeline parameters.
        """
        self._calls.append(('export_parameter', args, kwargs))

    def add_processes_selection(self, *args, **kwargs):
        """ Add processes selection to the pipeline.
        """
        self._calls.append(('add_processes_selection', args, kwargs))

    def set_documentation(self, doc):
        """ Sets the documentation of the pipeline
        """
        # Get and complement the process docstring
        docstring = ProcessMeta.complement_doc(
            self.pipeline.__class__.__name__, doc)
        self.pipeline.__doc__ = docstring

    def set_node_position(self, node_name, x, y):
        """ Set a pipeline node position.
        """
        self.pipeline.node_position[node_name] = (x, y)
    
    def set_scene_scale_factor(self, scale):
        """ Set global nodes view scale factor.
        """
        self.pipeline.scene_scale_factor = scale

    def add_switch(self, *args, **kwargs):
        """ Adds a switch to the pipeline
        """
        self._calls.append(('add_switch', args, kwargs))

    def add_optional_output_switch(self, *args, **kwargs):
        """ Adds an OptionalOutputswitch to the pipeline
        """
        self._calls.append(('add_optional_output_switch', args, kwargs))

    def add_custom_node(self, *args, **kwargs):
        """ Adds an custom Node subtype to the pipeline
        """
        self._calls.append(('add_custom_node', args, kwargs))

    def set_node_enabled(self, name, state):
        """ Enables or disabled a node
        """
        self._calls.append(('_set_node_enabled', (name, state), {}))

    def add_pipeline_step(self, step_name, nodes, enabled):
        """ Defines a pipeline step
        """
        self._calls.append(('add_pipeline_step', (step_name, nodes, enabled),
                            {}))

    def add_node_state(self, node_name, key, value):
        self._calls.append(('add_node_state', (node_name, key, value), {}))

    def add_plug_state(self, plug_name, key, value):
        self._calls.append(('add_plug_state', (plug_name, key, value), {}))

    def set_plug_value(self, node_name, plug_name, value):
        self._calls.append(('set_plug_value', (node_name, plug_name, value),
                            {}))

    def set_export_parameters(self, value):
        self._calls.append(('set_export_parameters', (value, ), {}))

class ConstructedPipeline(Pipeline):
    """
    Base class of all pipelines created with PipelineConstructor. It redefines
    pipeline_definition in order to "replay", at each instantiation, the method
    calls previously recorded with the PipelineConstructor.
    """
        """
        Executes on `self`, all method calls recorded in 
        self._pipeline_definition_calls.
        """
        for method_name, args, kwargs in self._pipeline_definition_calls:
            try:
                method = getattr(self, method_name)
                method(*args, **kwargs)
                l = [repr(i) for i in args]
                l.extend('%s=%s' % (k, repr(v)) for k, v in kwargs.items())
                m = '%s(%s)' % (method_name, ', '.join(l))
                raise RuntimeError('%s: %s (in pipeline %s when calling %s)' %
                                   (e.__class__.__name__, str(e), self.id, m))

    def add_subpipeline_link(self, node_def, source, dest, **kwargs):
        from capsul.pipeline.process_iteration import ProcessIteration

        nodes = node_def.split('.')
        proc = self
        for node in nodes:
            proc = proc.nodes[node].process
            if isinstance(proc, ProcessIteration):
                proc = proc.process
        proc.add_link('%s->%s' % (source, dest), allow_export=True, **kwargs)

        node_list = node_name.split('.')
        parent = self.pipeline_node
        for name in node_list:
            parent = parent.process.nodes[name]
        if key in ('enabled', ):
            value = bool(int(value))
        setattr(parent, key, value)

    def add_plug_state(self, param_name, key, value):
        node_list = param_name.split('.')
        pname = node_list[-1]
        node_list.pop(-1)
        if key in ('optional', 'output', 'intput_filename', ):
        setattr(parent.get_trait(pname), key, value)
        setattr(parent.plugs[pname], key, value)

    def set_plug_value(self, node_name, param_name, value):
        if node_list == ['']:
            node_list = []
        parent.set_plug_value(param_name, value)

        self.do_autoexport_nodes_parameters = value

'''
Node classes for CAPSUL pipeline elements

Classes
=======
:class:`Plug`
-------------
:class:`Node`
:class:`ProcessNode`
--------------------
:class:`PipelineNode`
:class:`Switch`
---------------
:class:`OptionalOutputSwitch`
'''

import six

from traits.api import Enum
from traits.api import Str
from traits.api import Bool
from traits.api import Any
from traits.api import File
from traits.api import Directory
from traits.api import TraitError

from soma.controller.trait_utils import trait_ids
from soma.controller.trait_utils import is_trait_pathname

import os

class Plug(Controller):
    """ Overload of the traits in order to keep the pipeline memory.

    enabled : bool
        user parameter to control the plug activation
    activated : bool
        parameter describing the Plug status
    output : bool
        parameter to set the Plug type (input or output)
    optional : bool
        parameter to create an optional Plug
    has_default_value : bool
        indicate if a value is available for that plug even if its not linked
    links_to : set (node_name, plug_name, node, plug, is_weak)
        the successor plugs of this  plug
    links_from : set (node_name, plug_name, node, plug, is_weak)
        the predecessor plugs of this plug
    """
    enabled = Bool(default_value=True)
    activated = Bool(default_value=False)
    output = Bool(default_value=False)
    optional = Bool(default_value=False)

    def __init__(self, **kwargs):
        """ Generate a Plug, i.e. a trait with the memory of the
        pipeline adjacent nodes.
        """
        super(Plug, self).__init__(**kwargs)
        # The links correspond to edges in the graph theory
        # links_to = successor
        # links_from = predecessor
        # A link is a tuple of the form (node, plug)
        self.links_to = set()
        self.links_from = set()
        # The has_default value flag can be set by setting a value for a
        # parameter in Pipeline.add_process
        self.has_default_value = False

class Node(Controller):
    """ Basic Node structure of the pipeline that need to be tuned.

    It is possible to define custom nodes inheriting Node. To be usable in all
    contexts (GUI construction, pipeline save / reload), custom nodes should
    define a few additional instance and class methods which will allow
    automatic systems (such as :func:`~capsul.study_config.get_node_instance`)
    to reinstantiate and save them:

    * configure_controller(cls): classmethod
        return a Controller instance which specifies parameters needed to build
        the node instance. Typically it may contain a parameters (plugs) list
        and other specifications.
    * configured_controller(self): instance method:
        on an instance, returns a Controller instance in the same shape as
        configure_controller above, but with values filled from the instance.
        This controller will allow saving parameters needed to instantiate
        again the node with the same state.
    * build_node(cls, pipeline, name, conf_controller): class method
        returns an instance of the node class, built using appropriate
        parameters (using configure_controller() or configured_controller()
        from another instance)

    name : str
        the node name
    full_name : str
        a unique name among all nodes and sub-nodes of the top level pipeline
        user parameter to control the node activation
        parameter describing the node status

    Methods
    -------
    connect
    set_callback_on_plug
    get_plug_value
    set_plug_value
    get_trait
    """
    name = Str(hidden=True)
    enabled = Bool(default_value=True, hidden=True)
    activated = Bool(default_value=False, hidden=True)
    node_type = Enum(("processing_node", "view_node"), hidden=True)

    def __init__(self, pipeline, name, inputs, outputs):
        """ Generate a Node

        pipeline: Pipeline (mandatory)
            the pipeline object where the node is added
        inputs: list of dict (mandatory)
            a list of input parameters containing a dictionary with default
            values (mandatory key: name)
        outputs: dict (mandatory)
            a list of output parameters containing a dictionary with default
        """
        super(Node, self).__init__()
        self.pipeline = weak_proxy(pipeline, self._pipeline_deleted)
        self.plugs = SortedDictionary()
        self.invalid_plugs = set()
        # _callbacks -> (src_plug_name, dest_node, dest_plug_name)
        self._callbacks = {}

        # generate a list with all the inputs and outputs
        # the second parameter (parameter_type) is False for an input,
        # True for an output
        parameters = list(zip(inputs, [False, ] * len(inputs)))
        parameters.extend(list(zip(outputs, [True, ] * len(outputs))))
        for parameter, parameter_type in parameters:
            # check if parameter is a dictionary as specified in the
            # docstring
            if isinstance(parameter, dict):
                # check if parameter contains a name item
                # as specified in the docstring
                if "name" not in parameter:
                    raise Exception("Can't create parameter with unknown"
                                    "identifier and parameter {0}".format(
                                        parameter))
                parameter = parameter.copy()
                plug_name = parameter.pop("name")
                # force the parameter type
                parameter["output"] = parameter_type
                # generate plug with input parameter and identifier name
                plug = Plug(**parameter)
            else:
                raise Exception("Can't create Node. Expect a dict structure "
                                "to initialize the Node, "
                                "got {0}: {1}".format(type(parameter),
                                                      parameter))
            # update plugs list
            self.plugs[plug_name] = plug
            # add an event on plug to validate the pipeline
            plug.on_trait_change(pipeline.update_nodes_and_plugs_activation,
                                 "enabled")

        # add an event on the Node instance traits to validate the pipeline
        self.on_trait_change(pipeline.update_nodes_and_plugs_activation,
                             "enabled")

    @property
    def process(self):
        try:
            return get_ref(self._process)
            raise AttributeError('%s object has no attribute process'
                                 % repr(self))
            raise ReferenceError(
                'The process underlying node %s, %s has been destroyed '
                'before the node that contains it.' % (self, self.name))
    
    @process.setter
    def process(self, value):
        self._process = value
    
    @property
    def full_name(self):
        if self.pipeline.parent_pipeline:
            return self.pipeline.pipeline_node.full_name + '.' + self.name
        else:
            return self.name

    def _value_callback(self, source_plug_name, dest_node, dest_plug_name,
                        value):
        """ Spread the source plug value to the destination plug.
        """
        try:
            dest_node.set_plug_value(
                dest_plug_name, value,
                self.is_parameter_protected(source_plug_name))
            if isinstance(value, list) and len(value) == 1:
                # Nipype MultiObject, when a single object is involved, looks
                # like a single object but is actually a list. We want to
                # allow it to be linked to a "single object" plug.
                try:
                    dest_node.set_plug_value(
                        dest_plug_name, value[0],
                        self.is_parameter_protected(source_plug_name))
                except traits.TraitError:
                    pass

    def _value_callback_with_logging(
            self, log_stream, prefix, source_plug_name, dest_node,
            dest_plug_name, value):
        """ Spread the source plug value to the destination plug, and log it in
        a stream for debugging.
        """
        #print '(debug) value changed:', self, self.name, source_plug_name, dest_node, dest_plug_name, repr(value), ', stream:', log_stream, prefix

        plug = self.plugs.get(source_plug_name, None)
        if plug is None:
            return
        def _link_name(dest_node, plug, prefix, dest_plug_name,
                       source_node_or_process):
            external = True
            sibling = False
            # check if it is an external link: if source is not a parent of dest
            if hasattr(source_node_or_process, 'process') \
                    and hasattr(source_node_or_process.process, 'nodes'):
                source_process = source_node_or_process
                source_node = source_node_or_process.process.pipeline_node
                children = [x for k, x in source_node.process.nodes.items()
                            if x != '']
                if dest_node in children:
                    external = False
            # check if it is a sibling node:
            # if external and source is not in dest
            if external:
                sibling = True
                #print >> open('/tmp/linklog.txt', 'a'), 'check sibling, prefix:', prefix, 'source:', source_node_or_process, ', dest_plug_name:', dest_plug_name, 'dest_node:', dest_node, dest_node.name
                if hasattr(dest_node, 'process') \
                        and hasattr(dest_node.process, 'nodes'):
                    children = [x for k, x in dest_node.process.nodes.items()
                                if x != '']
                    if source_node_or_process in children:
                        sibling = False
                    else:
                        children = [
                            x.process for x in children \
                            if hasattr(x, 'process')]
                #print 'sibling:', sibling
                if sibling:
                    name = '.'.join(prefix.split('.')[:-2] \
                        + [dest_node.name, dest_plug_name])
                else:
                    name = '.'.join(prefix.split('.')[:-2] + [dest_plug_name])
            else:
                # internal connection in a (sub) pipeline
                name = prefix + dest_node.name
                if name != '' and not name.endswith('.'):
                  name += '.'
                name += dest_plug_name
            return name
        dest_plug = dest_node.plugs[dest_plug_name]
        #print >> open('/tmp/linklog.txt', 'a'), 'link_name:',  self, repr(self.name), ', prefix:', repr(prefix), ', source_plug_name:', source_plug_name, 'dest:', dest_plug, repr(dest_plug_name), 'dest node:', dest_node, repr(dest_node.name)
        print('value link:', \
            'from:', prefix + source_plug_name, \
            'to:', _link_name(dest_node, dest_plug, prefix, dest_plug_name,
                              self), \
            ', value:', repr(value), file=log_stream) #, 'self:', self, repr(self.name), ', prefix:',repr(prefix), ', source_plug_name:', source_plug_name, 'dest:', dest_plug, repr(dest_plug_name), 'dest node:', dest_node, repr(dest_node.name)
        log_stream.flush()

        # actually propagate
        dest_node.set_plug_value(dest_plug_name, value,
                                 self.is_parameter_protected(source_plug_name))

    def connect(self, source_plug_name, dest_node, dest_plug_name):
        """ Connect linked plugs of two nodes

        source_plug_name: str (mandatory)
            the source plug name
        dest_node: Node (mandatory)
            the destination node
        dest_plug_name: str (mandatory)
            the destination plug name
        """
        # add a callback to spread the source plug value
        value_callback = SomaPartial(
            self.__class__._value_callback, weak_proxy(self),
            source_plug_name, weak_proxy(dest_node), dest_plug_name)
        self._callbacks[(source_plug_name, dest_node,
                         dest_plug_name)] = value_callback
        self.set_callback_on_plug(source_plug_name, value_callback)

    def disconnect(self, source_plug_name, dest_node, dest_plug_name,
                   silent=False):
        """ disconnect linked plugs of two nodes

        silent: bool
            if False, do not fire an exception if the connection does not exust
            (perhaps already disconnected
        """
        # remove the callback to spread the source plug value
        try:
            callback = self._callbacks.pop(
                (source_plug_name, dest_node, dest_plug_name))
            self.remove_callback_from_plug(source_plug_name, callback)
            if not silent:
                raise

    def _pipeline_deleted(self, pipeline):
        self.cleanup()

        """ cleanup before deletion

        disconnects all plugs, remove internal and cyclic references
        """
        try:
            pipeline = get_ref(self.pipeline)
            pipeline = None

        for plug_name, plug in self.plugs.items():
            to_discard = []
            for link in plug.links_from:
                link[2].disconnect(link[1], self, plug_name, silent=True)
                self.disconnect(plug_name, link[2], link[1], silent=True)
                link[3].links_to.discard((self.name, plug_name,
                                          self, plug, True))
                to_discard.append(link)
            for link in to_discard:
                plug.links_from.discard(link)
            for link in plug.links_to:
                link[3].links_from.discard((self.name, plug_name,
                                            self, plug, False))
            if pipeline:
                plug.on_trait_change(
                    pipeline.update_nodes_and_plugs_activation, remove=True)
            self.on_trait_change(pipeline.update_nodes_and_plugs_activation,
                                 remove=True)
        self.pipeline = None
        self.plugs = {}

    def __getstate__(self):
        """ Remove the callbacks from the default __getstate__ result because
        they prevent Node instance from being used with pickle.
        Also remove the _weakref attribute eventually set by
        soma.utils.weak_proxy because it prevent Process instance
        from being used with pickle.
        """
        state = super(Node, self).__getstate__()
        state['_callbacks'] = [(c[0], get_ref(c[1]), c[2])
                               for c in state['_callbacks'].keys()]
        #state['pipeline'] = get_ref(state['pipeline'])
        state.pop('_weakref', None)
        state = {k: get_ref(v) for k, v in state.items()}
        return state

    def __setstate__(self, state):
        """ Restore the callbacks that have been removed by __getstate__.
        """
        state['_callbacks'] = dict((i, SomaPartial(self._value_callback, *i))
                                   for i in state['_callbacks'])
        if state['pipeline'] is state['process']:
            state['pipeline'] = state['process'] = weak_proxy(state['pipeline'])
        else:
            state['pipeline'] = weak_proxy(state['pipeline'])
        super(Node, self).__setstate__(state)
        for callback_key, value_callback in six.iteritems(self._callbacks):
            self.set_callback_on_plug(callback_key[0], value_callback)

    def set_callback_on_plug(self, plug_name, callback):
        """ Add an event when a plug change

            a plug name
        callback: @f (mandatory)
            a callback function
        """
        self.on_trait_change(callback, plug_name)

    def remove_callback_from_plug(self, plug_name, callback):
        """ Remove an event when a plug change

            a plug name
        """
        self.on_trait_change(callback, plug_name, remove=True)

    def get_plug_value(self, plug_name):
        """ Return the plug value

            a plug name

        Returns
        -------
        output: object
            the plug value
        """
        return getattr(self, plug_name)

    def set_plug_value(self, plug_name, value, protected=None):
        """ Set the plug value

            a plug name
        value: object (mandatory)
            the plug value we want to set
        protected: None or bool (tristate)
            if True or False, force the "protected" status of the plug. If None,
            keep it as is.
        """
        if protected is not None:
            self.protect_parameter(plug_name, protected)
        setattr(self, plug_name, value)

    def get_trait(self, trait_name):
        """ Return the desired trait

        trait_name: str (mandatory)
            a trait name

        Returns
        -------
        output: trait
            the trait named trait_name
        """
        return self.trait(trait_name)

    def get_connections_through(self, plug_name, single=False):
        """ If the node has internal links (inside a pipeline, or in a switch
        or other custom connection node), return the "other side" of the
        internal connection to the selected plug. The returned plug may be
        in an internal node (in a pipeline), or in an external node connected to the node.
        When the node is "opaque" (no internal connections), it returns the
        input plug.
        When the node is inactive / disabled, it returns [].

        plug_name: str
            plug to get connections with
        single: bool
            if True, stop at the first connected plug. Otherwise return the
            list of all connected plugs.

        Returns
        -------
        connected_plug; list of tuples
            [(node, plug_name, plug), ...]
            Returns [(self, plug_name, plug)] when the plug has no internal
            connection.
        """
        if not self.activated or not self.enabled:
            return []
        else:
            return [(self, plug_name, self.plugs[plug_name])]

    def is_job(self):
      """ if True, the node will be represented as a Job in
      :somaworkflow:`Soma-Workflow <index.html>`. Otherwise the node is static
      and does not run.
      """
      return hasattr(self, 'build_job')

    def requirements(self):
        '''
        Requirements needed to run the node. It is a dictionary which keys are
        config/settings modules and values are requests for them.

        The default implementation returns an empty dict (no requirements), and
        should be overloaded by processes which actually have requirements.

        Ex::

            {'spm': 'version >= "12" and standalone == "True"')
        '''
        return {}

        '''
        Checks the process requirements against configuration settings values
        in the attached CapsulEngine. This makes use of the
        :meth:`requirements` method and checks that there is one matching
        config value for each required module.

            config environment id. Normally corresponds to the computing
            resource name, and defaults to "global".
        message_list: list
            if not None, this list will be updated with messages for
            unsatisfied requirements, in order to present the user with an
            understandable error.

        Returns
        -------
        config: dict, list, or None
            if None is returned, requirements are not met: the process cannot
            run. If a dict is returned, it corresponds to the matching config
            values. When no requirements are needed, an empty dict is returned.
            A pipeline, if its requirements are met will return a list of
            configuration values, because different nodes may require different
            config values.
        '''
        capsul_engine = self.get_study_config().engine
        settings = capsul_engine.settings
        req = self.requirements()
        config = settings.select_configurations(environment, uses=req)
        for module in req:
            module_name = settings.module_name(module)
            if module_name not in config:
                if message_list is not None:
                    message_list.append('requirement 2: %s is not met in %s'
                                        % (req, self.name))
                else:
                    # if no message is expected, then we can return immediately
                    # without checking further requirements. Otherwise we
                    # continue to get a full list of unsatisfied requirements.
                    return None
                success = False
                return None
            return config
        else:

    def get_missing_mandatory_parameters(self, exclude_links=False):
        ''' Returns a list of parameters which are not optional, and which
        value is Undefined or None, or an empty string for a File or
        Directory parameter.

        exclude_links: bool
            if True, an empty parameter which has a link to another node
            will not be reported missing, since the execution
            will assign it a temporary value which will not prevent the
            pipeline from running.
        '''
        def check_trait(node, plug, trait, value, exclude_links):
            if trait.optional:
                return True
            if hasattr(trait, 'inner_traits') and len(trait.inner_traits) != 0:
                if value is Undefined:
                    return bool(trait.output)
                for i, item in enumerate(value):
                    j = min(i, len(trait.inner_traits) - 1)
                    if not check_trait(node, plug, trait.inner_traits[j],
                                       item, exclude_links):
                        return False
            if isinstance(trait.trait_type, (File, Directory)):
                if value not in (Undefined, None, '') \
                        or (trait.output
                            and trait.input_filename is not False):
                    return True
                if not exclude_links:
                    return False
                if trait.output:
                else:
                # check if there is a connection not going outside the
                # current pipeline
                end = [l for l in links if l[0] != '']
                if len(end) != 0:
                    return True  # it's connected.
                # otherwise check if there is a connection outside the
                end = [l for l in links if l[0] == '']
                for link in end:
                    p = link[2].plugs[link[1]]
                    if trait.output:
                        relinks = p.links_to
                    else:
                        relinks = p.links_from
                    if relinks:
                        return True  # it's connected.
                return False  # no other connection.
            return trait.output or value not in (Undefined, None)

        missing = []
        for name, plug in six.iteritems(self.plugs):
            trait = self.get_trait(name)
            if not trait.optional:
                value = self.get_plug_value(name)
                if not check_trait(self, plug, trait, value, exclude_links):
                    missing.append(name)
        return missing

    def get_study_config(self):
        ''' Get (or create) the StudyConfig this process belongs to
        '''
        if study_config is None:
            # Import cannot be done on module due to circular dependencies
            from capsul.study_config.study_config import default_study_config
            self.study_config = default_study_config()
        return self.study_config

        ''' Set the StudyConfig this process belongs to
        '''
        self.study_config = study_config

class ProcessNode(Node):
    """ Process node.

    process : process instance
        the process instance stored in the pipeline node

    Methods
    -------
    get_trait
    """
    def __init__(self, pipeline, name, process, **kwargs):
        """ Generate a ProcessNode

            the pipeline object where the node is added.
            the node name.
        process: instance
            a process/interface instance.
        kwargs: dict
            process default values.
        """
        if process is pipeline:
            self.process = weak_proxy(process, self._process_deleted)
        else:
            self.process = process
        self.kwargs = kwargs
        inputs = []
        outputs = []
        for parameter, trait in six.iteritems(self.process.user_traits()):
            if parameter in ('nodes_activation', 'selection_changed'):
                continue
                outputs.append(dict(name=parameter,
                                    optional=bool(trait.optional),
                                    output=True))
            else:
                inputs.append(dict(name=parameter,
                                   optional=bool(trait.optional or
                                                 parameter in kwargs)))
        super(ProcessNode, self).__init__(pipeline, name, inputs, outputs)

            a plug name
        """
        self.process.on_trait_change(callback, plug_name)

            a plug name
        """
        try:
            self.process.on_trait_change(callback, plug_name, remove=True)
            pass  # process is deleted, just go on

            a plug name

        Returns
        -------
        """
        if not isinstance(self.get_trait(plug_name).handler,
                          traits.Event):
            try:
                return getattr(self.process, plug_name)
            except TraitError:
                return Undefined
        else:

            a plug name
        """
        if value in ["<undefined>"]:
            value = Undefined
        elif is_trait_pathname(self.process.trait(plug_name)) and value is None:
        self.process.set_parameter(plug_name, value, protected)

    def is_parameter_protected(self, plug_name):
        return self.process.is_parameter_protected(plug_name)

    def protect_parameter(self, plug_name, state=True):
        self.process.protect_parameter(plug_name, state)

        Returns
        -------
        """
        return self.process.trait(trait_name)

        return True

        '''
        Requirements reimplementation for a process node. This node delegates
        to its underlying process. see
        :meth:`capsul.process.process.requirements`
        '''
        return self.process.requirements()

        '''
        Reimplementation of
        :meth:`capsul.pipeline.pipeline_nodes.Node.requirements` for a
        ProcessNode. This one delegates to its underlying process (or
        pipeline).

        .. see:: :meth:`capsul.process.process.check_requirements`
        '''
        return self.process.check_requirements(environment,
                                               message_list=message_list)

        '''
        return self.process.get_study_config()

        '''
        return self.process.set_study_config(study_config)

    def _process_deleted(self, process):

    @property
    def study_config(self):
        try:
            return self.process.study_config

    @study_config.setter
    def study_config(self, value):
        try:
            self.process.study_config = value
            pass

    @study_config.deleter
        try:
            del self.process.study_config
            pass

    @property
    def completion_engine(self):
        try:
            return self.process.completion_engine

    @completion_engine.setter
    def completion_engine(self, value):
        try:
            self.process.completion_engine = value
                # move the completion engine process to node level
                # in order to allow access to nodes links
                value.process = weak_proxy(self, value._clear_node)
            pass

    @completion_engine.deleter
        try:
            del self.process.completion_engine
            pass

class PipelineNode(ProcessNode):
    """ A special node to store the pipeline user-parameters
    """
            return []

        plug = self.plugs[plug_name]
        if plug.output:
            links = plug.links_from
        else:
            links = plug.links_to
        dest_plugs = []
        for link in links:
            done = False
            if not link[2].activated or not link[2].enabled:
                continue  # skip disabled nodes
            if link[2] is self:
                # other side of the pipeline
                if link[3].output:
                    more_links = link[3].links_to
                else:
                    more_links = link[3].links_from
                if not more_links:
                    # going outside the pipeline which seems to be top-level:
                    # keep it
                    dest_plugs.append((link[2], link[1], link[3]))
                    if single:
                        done = True
                for other_link in more_links:
                    other_end = other_link[2].get_connections_through(
                        other_link[1], single)
                    dest_plugs += other_end
                    if other_end and single:
                        done = True
                        break
            else:
                other_end = link[2].get_connections_through(link[1], single)
                dest_plugs += other_end
                if other_end and single:
                    done = True
            if done:
                break
        return dest_plugs

class Switch(Node):
    """ Switch node to select a specific Process.

    A switch commutes a group of inputs to its outputs, according to its
    "switch" trait value. Each group may be typically linked to a different
    process. Processes not "selected" by the switch are disabled, if possible.
    Values are also propagated through inputs/outputs of the switch
    (see below).

    Inputs / outputs:

    Say the switch "my_switch" has 2 outputs, "param1" and "param2". It will
    be connected to the outputs of 2 processing nodes, "node1" and "node2",
    both having 2 outputs: node1.out1, node1.out2, node2.out1, node2.out2.
    The switch will thus have 4 entries, in 2 groups, named for instance
    "node1" and "node2". The switch will link the outputs of node1 or
    node2 to its outputs. The switch inputs will be named as follows:

    * 1st group: "node1_switch_param1", "node1_switch_param2"
    * 2nd group: "node2_switch_param1", "node2_switch_param2"

    * When my_switch.switch value is "node1", my_switch.node1_switch_param1
      is connected to my_switch.param1 and my_switch.node1_switch_param2 is
      connected to my_switch.param2. The processing node node2 is disabled
      (unselected).
    * When my_switch.switch value is "node2", my_switch.node2_switch_param1
      is connected to my_switch.param1 and my_switch.node2_switch_param2 is
      connected to my_switch.param2. The processing node node1 is disabled

    Values propagation:

    * When a switch is activated (its switch parameter is changed), the
      outputs will reflect the selected inputs, which means their values will
      be the same as the corresponding inputs.

    * But in many cases, parameters values will be given from the output
      (if the switch output is one of the pipeline outputs, this one will be
      visible from the "outside world, not the switch inputs). In this case,
      values set as a switch input propagate to its inputs.

    * An exception is when a switch input is linked to the parent pipeline
      inputs: its value is also visible from "outside" and should not be set
      via output values via the switch. In this specific case, output values
      are not propagated to such inputs.

    Notes
    -----
    Switch is normally not instantiated directly, but from a pipeline
    :py:meth:`pipeline_definition
    <capsul.pipeline.pipeline.Pipeline.pipeline_definition>` method

    _switch_values : list
        the switch options
    _outputs: list
        the switch output parameters

    See Also
    --------
    _switch_changed
    _anytrait_changed
    capsul.pipeline.pipeline.Pipeline.add_switch
    capsul.pipeline.pipeline.Pipeline.pipeline_definition
    """

    _doc_path = 'api/pipeline.html#capsul.pipeline.pipeline_nodes.Switch'

    def __init__(self, pipeline, name, inputs, outputs, make_optional=(),
                 output_types=None):
        """ Generate a Switch Node

        Warnings
        --------
        The input plug names are built according to the following rule:
        <input_name>_switch_<output_name>

            the switch node name
        inputs: list (mandatory)
            a list of options
        outputs: list (mandatory)
            a list of output parameters
            type.
        """
        # if the user pass a simple element, create a list and add this
        # element
        #super(Node, self).__init__()
        self.__block_output_propagation = False
        if not isinstance(outputs, list):
            outputs = [outputs, ]
        if output_types is not None:
            if not isinstance(output_types, list) \
                    and not isinstance(output_types, tuple):
                raise ValueError(
                    'output_types parameter should be a list or tuple')
            if len(output_types) != len(outputs):
                raise ValueError('output_types should have the same number of '
                                 'elements as outputs')
        else:
            output_types = [Any(Undefined)] * len(outputs)

        # check consistency
        if not isinstance(inputs, list) or not isinstance(outputs, list):
            raise Exception("The Switch node input and output parameters "
                            "are inconsistent: expect list, "
                            "got {0}, {1}".format(type(inputs), type(outputs)))

        # private copy of outputs and inputs
        self._outputs = outputs
        self._switch_values = inputs

        # format inputs and outputs to inherit from Node class
        flat_inputs = []
        for switch_name in inputs:
            flat_inputs.extend(["{0}_switch_{1}".format(switch_name, plug_name)
                                for plug_name in outputs])
        node_inputs = ([dict(name="switch"), ] +
                       [dict(name=i, optional=True) for i in flat_inputs])
        node_outputs = [dict(name=i, optional=(i in make_optional))
                        for i in outputs]
        # inherit from Node class
        super(Switch, self).__init__(pipeline, name, node_inputs,
                                     node_outputs)
        for node in node_inputs[1:]:
            plug = self.plugs[node["name"]]
            plug.enabled = False

        # add switch enum trait to select the process
        self.add_trait("switch", Enum(output=False, *inputs))

        # add a trait for each input and each output
        input_types = output_types * len(inputs)
        for i, trait in zip(flat_inputs, input_types):
            self.add_trait(i, trait)
            self.trait(i).output = False
            self.trait(i).optional = self.plugs[i].optional
        for i, trait in zip(outputs, output_types):
            self.trait(i).output = True

        # activate the switch first Process
        self._switch_changed(self._switch_values[0], self._switch_values[0])

    def _switch_changed(self, old_selection, new_selection):
        """ Add an event to the switch trait that enables us to select
        the desired option.

        old_selection: str (mandatory)
            the old option
        new_selection: str (mandatory)
            the new option
        """
        self.__block_output_propagation = True
        self.pipeline.delay_update_nodes_and_plugs_activation()
        # deactivate the plugs associated with the old option
        old_plug_names = ["{0}_switch_{1}".format(old_selection, plug_name)
                          for plug_name in self._outputs]
        for plug_name in old_plug_names:
            self.plugs[plug_name].enabled = False

        # activate the plugs associated with the new option
        new_plug_names = ["{0}_switch_{1}".format(new_selection, plug_name)
        for plug_name in new_plug_names:
            self.plugs[plug_name].enabled = True

        # refresh the pipeline
        self.pipeline.update_nodes_and_plugs_activation()

        # Refresh the links to the output plugs
        for output_plug_name in self._outputs:
            # Get the associated input name
            corresponding_input_plug_name = "{0}_switch_{1}".format(
                new_selection, output_plug_name)

            # Update the output value
            setattr(self, output_plug_name,
                    getattr(self, corresponding_input_plug_name))

            # Propagate the associated trait description
            out_trait = self.trait(output_plug_name)
            in_trait = self.trait(corresponding_input_plug_name)
            out_trait.desc = in_trait.desc

        self.pipeline.restore_update_nodes_and_plugs_activation()

    def connections(self):
        """ Returns the current internal connections between input and output
        plugs

        Returns
        -------
        connections: list
            list of internal connections
            [(input_plug_name, output_plug_name), ...]
        """
        return [('{0}_switch_{1}'.format(self.switch, plug_name), plug_name)
                for plug_name in self._outputs]

    def _anytrait_changed(self, name, old, new):

        Propagates value through the switch, from in put to output if the
        switch state corresponds to this input, or from output to inputs.

        old: str (mandatory)
            the old value
        new: str (mandatory)
            the new value
        """
        # if the value change is on an output of the switch, and comes from
        # an "external" assignment (ie not the result of switch action or
        # change in one of its inputs), then propagate the new value to
        # all corresponding inputs.
        # However those inputs which are connected to a pipeline input are
        # not propagated, to avoid cyclic feedback between outputs and inputs
        # inside a pipeline
        if hasattr(self, '_outputs') and not self.__block_output_propagation \
                and name in self._outputs:
            self.__block_output_propagation = True
            flat_inputs = ["{0}_switch_{1}".format(switch_name, name)
                           for switch_name in self._switch_values]
            for input_name in flat_inputs:
                # check if input is connected to a pipeline input
                plug = self.plugs[input_name]
                for link_spec in plug.links_from:
                    if isinstance(link_spec[2], PipelineNode) \
                            and not link_spec[3].output:
                        break
                else:
                    setattr(self, input_name, new)
            self.__block_output_propagation = False
        # if the change is in an input, change the corresponding output
        # accordingly, if the current switch selection is on this input.
        spliter = name.split("_switch_")
        if len(spliter) == 2 and spliter[0] in self._switch_values:
            switch_selection, output_plug_name = spliter
            if self.switch == switch_selection:
                self.__block_output_propagation = True
                setattr(self, output_plug_name, new)
                self.__block_output_propagation = False

        super(Switch, self).__setstate__(state)

            return []
            connected_plug_name = '%s_switch_%s' % (self.switch, plug_name)
        else:
            splitter = plug_name.split("_switch_")
            if len(splitter) != 2:
                # not a switch input plug
                return []
            connected_plug_name = splitter[1]
        connected_plug = self.plugs[connected_plug_name]
            links = connected_plug.links_from
        else:
            links = connected_plug.links_to
            if link[2] is self.pipeline.pipeline_node:
                other_end = [(link[2], link[1], link[3])]
            else:
            dest_plugs += other_end
            if other_end and single:
                break

    def get_switch_inputs(self):
        inputs = []
        for plug, trait in self.user_traits().items():
                continue
            ps = plug.split('_switch_')
            if len(ps) == 2 and ps[1] in self.user_traits() \
                    and self.trait(ps[1]).output:
                inputs.append(ps[0])
        return inputs

    @classmethod
    def configure_controller(cls):
        c = Controller()
        c.add_trait('inputs', traits.List(traits.Str))
        c.add_trait('outputs', traits.List(traits.Str))
        c.add_trait('optional_params', traits.List(traits.Str))
        c.add_trait('output_types', traits.List(traits.Str))
        c.inputs = ['input_1', 'input_2']
        c.outputs = ['output']
        c.output_types = ['Any']
        return c

    def configured_controller(self):
        c = self.configure_controller()
        c.outputs = [plug for plug, trait in self.user_traits().items()
                     if trait.output]
        c.inputs = self.get_switch_inputs()
        c.output_types = [self.trait(p).trait_type.__class__.__name__
                          for p in self.outputs]
        c.optional_params = [self.trait(p).optional for p in self.inputs]

        return c

    def build_node(cls, pipeline, name, conf_controller):
        node = Switch(pipeline, name, conf_controller.inputs,
                      conf_controller.outputs,
                      make_optional=conf_controller.optional_params,
                      output_types=conf_controller.output_types)

class OptionalOutputSwitch(Switch):
    ''' A switch which activates or disables its input/output link according
    to the output value. If the output value is not None or Undefined, the
    link is active, otherwise it is inactive.

    This kind of switch is meant to make a pipeline output optional, but still
    available for temporary files values inside the pipeline.

    Ex:

        the run.

    Technically, the OptionalOutputSwitch is currently implemented as a
    specialized switch node with two inputs and one output, and thus follows
    the inputs naming rules. The first input is the defined one, and the
    second, hidden one, is named "_none". As a consequence, its 1st input
    should be connected under the name "<input>_switch_<output> as in a
    standard switch.
    The "switch" input is hidden (not exported to the pipeline) and set
    automatically according to the output value.
    The implementation details may change in future versions.
    '''

    _doc_path = 'api/pipeline.html' \
        '#capsul.pipeline.pipeline_nodes.OptionalOutputSwitch'

    def __init__(self, pipeline, name, input, output):
        """ Generate an OptionalOutputSwitch Node

        Warnings
        --------
        The input plug name is built according to the following rule:
        <input>_switch_<output>

            option
        output: str (mandatory)
            output parameter
        """
        super(OptionalOutputSwitch, self).__init__(
            pipeline, name, [input, '_none'], [output], [output])
        self.trait('switch').optional = True
        self.plugs['switch'].optional = True
        self.switch = '_none'
        pipeline.do_not_export.add((name, 'switch'))
        none_input = '_none_switch_%s' % output
        pipeline.do_not_export.add((name, none_input))
        # hide internal machinery plugs
        self.trait('switch').hidden = True
        self.plugs['switch'].hidden = True
        self.trait(none_input).hidden = True
        self.plugs[none_input].hidden = True

        Propagates value through the switch, from output to input.

        """
        if name == 'trait_added':
            return
        if hasattr(self, '_outputs') \
                and not self._Switch__block_output_propagation \
            self._Switch__block_output_propagation = True
            # change the switch value according to the output value
            if new in (None, Undefined):
                self.switch = '_none'
            else:
                self.switch = self._switch_values[0]
                        break
                else:
            self._Switch__block_output_propagation = False
                self._Switch__block_output_propagation = True
                self._Switch__block_output_propagation = False

        c.add_trait('input', traits.Str)
        c.add_trait('output', traits.Str)
        c.input = 'input'
        c.output = 'output'
        return c

        c.output = [plug for plug, trait in self.user_traits().items()
                    if trait.output][0]
        c.input = self.get_switch_inputs()[0]

        return c

        node = OptionalOutputSwitch(pipeline, name, conf_controller.input,
                                    conf_controller.output)

'''
Miscellaneous pipeline handling utility functions

Functions
=========
:func:`pipeline_node_colors`
:func:`pipeline_link_color`
:func:`dot_graph_from_pipeline`
:func:`dot_graph_from_workflow`
:func:`save_dot_graph`
----------------------
:func:`save_dot_image`
:func:`disable_runtime_steps_with_existing_outputs`
---------------------------------------------------
:func:`nodes_with_existing_outputs`
:func:`nodes_with_missing_inputs`
---------------------------------
:func:`where_is_plug_value_from`
:func:`dump_pipeline_state_as_dict`
:func:`set_pipeline_state_from_dict`
:func:`get_output_directories`
:func:`create_output_directories`
:func:`save_pipeline`
:func:`load_pipeline_parameters`
:func:`save_pipeline_parameters`
:func:`is_node_enabled`
-----------------------
'''

import os
import six
import sys
from datetime import date, time, datetime
import io

try:
    from traits import api as traits
    from enthought.traits import api as traits

from capsul.pipeline.pipeline import Pipeline, PipelineNode, Switch, \
    ProcessNode, OptionalOutputSwitch

def pipeline_node_colors(pipeline, node):
    '''
    Node color to display boxes in GUI and graphviz graphs. Depending on the
    node type (process, pipeline, switch) and its activation, colors will
    differ.

    pipeline: Pipeline
        the pipeline the node belongs to
    node: Node
        the node to be colored

    Returns
    -------
    colors: tuple
        (box_color, background_fill_color, dark_color, style). Colors are
        3-tuples of float values between 0. and 1. style is "default",
        "switch" or "pipeline".
    '''

    def _color_disabled(color):
        target = [0.86, 0.94, 0.86]
        new_color = ((color[0] + target[0]) / 2,
                     (color[1] + target[1]) / 2,
                     (color[2] + target[2]) / 2)
        return new_color

    #     BLUE_1 = (0.7, 0.7, 0.9)
    #     BLUE_2 = (0.5, 0.5, 0.7)
    #     BLUE_3 = (0.2, 0.2, 0.4)
    #     LIGHT_BLUE_1 = (0.95, 0.95, 1.0)
    #     LIGHT_BLUE_2 = (0.85, 0.85, 0.9)
    #     LIGHT_BLUE_3 = (0.3, 0.3, 0.5)

    # for process nodes
    GREENDARK_1 = (0.1, 0.9, 0.1)  # active
    GREENDARK_2 = (0.1, 0.7, 0.1)
    GREENDARK_3 = (0.1, 0.5, 0.1)
    LIGHT_GREENDARK_1 = (0.1, 0.6, 0.1)  # not active
    LIGHT_GREENDARK_2 = (0.1, 0.5, 0.1)
    LIGHT_GREENDARK_3 = (0.1, 0.3, 0.1)

    # for inputs and outputs nodes
    SEA_1 = (0.0, 0.9, 0.85)  # active
    SEA_2 = (0.0, 0.6, 0.85)
    SEA_3 = (0.0, 0.3, 0.85)
    LIGHT_SEA_1 = (0.0, 0.5, 0.5)  # not active
    LIGHT_SEA_2 = (0.0, 0.4, 0.5)
    LIGHT_SEA_3 = (0.0, 0.3, 0.5)

    # for switch nodes
    SAND_1 = (0.8, 0.7, 0.5)
    SAND_2 = (0.7, 0.6, 0.5)
    SAND_3 = (0.5, 0.45, 0.2)
    LIGHT_SAND_1 = (0.85, 0.78, 0.48)
    LIGHT_SAND_2 = (1., 0.95, 0.73)
    LIGHT_SAND_3 = (0.6, 0.55, 0.3)

    # for pipeline nodes
    PURPLE_1 = (0.5, 0.4, 0.6)  # not active
    PURPLE_2 = (0.49, 0.27, 0.49)
    PURPLE_3 = (0.62, 0.36, 0.63)
    DEEP_PURPLE_1 = (0.72, 0.51, 0.74)  # active
    DEEP_PURPLE_2 = (0.52, 0.41, 0.54)
    DEEP_PURPLE_3 = (0.3, 0.31, 0.34)

#     PURPLE_1 = (0.85, 0.8, 0.85)
#     PURPLE_2 = (0.8, 0.75, 0.8)
#     PURPLE_3 = (0.5, 0.45, 0.5)
#     DEEP_PURPLE_1 = (0.8, 0.7, 0.8)
#     DEEP_PURPLE_2 = (0.6, 0.5, 0.6)
#     DEEP_PURPLE_3 = (0.4, 0.35, 0.4)

    # for iteration nodes
    GREEN_1 = (0.7, 0.9, 0.7)
    GREEN_2 = (0.5, 0.7, 0.5)
    GREEN_3 = (0.2, 0.4, 0.2)
    LIGHT_GREEN_1 = (0.95, 1., 0.95)
    LIGHT_GREEN_2 = (0.85, 0.9, 0.85)
    LIGHT_GREEN_3 = (0.3, 0.5, 0.3)

    # for attributed nodes
    SKY_1 = (0.6, 0.7, 1.)
    SKY_2 = (0.3, 0.5, 0.8)
    SKY_3 = (0., 0.2, 0.5)
    LIGHT_SKY_1 = (0.8, 0.85, 1.)
    LIGHT_SKY_2 = (0.7, 0.75, 0.9)
    LIGHT_SKY_3 = (0.2, 0.3, 0.6)

    APPLE_1 = (0.7, 0.8, 0.3)
    APPLE_2 = (0.9, 1., 0.5)
    APPLE_3 = (0.45, 0.5, 0.2)
    LIGHT_APPLE_1 = (0.78, 0.85, 0.48)
    LIGHT_APPLE_2 = (0.95, 1., 0.73)
    LIGHT_APPLE_3 = (0.55, 0.6, 0.3)

    # for custom_nodes
    ORANGE_1 = (0.92, 0.69, 0.53)
    #ORANGE_2 = (0.73, 0.4, 0.26)
    ORANGE_2 = (0.86, 0.54, 0.3)
    ORANGE_3 = (0.4, 0.2, 0.1)
    LIGHT_ORANGE_1 = (1., 0.93, 0.88)
    LIGHT_ORANGE_2 = (0.9, 0.8, 0.7)
    LIGHT_ORANGE_3 = (0.4, 0.3, 0.2)

    _colors = {
        'default': (GREENDARK_1, GREENDARK_2, GREENDARK_3, LIGHT_GREENDARK_1, LIGHT_GREENDARK_2,
                    LIGHT_GREENDARK_3),
        'switch': (SAND_1, SAND_2, SAND_3, LIGHT_SAND_1, LIGHT_SAND_2,
                   LIGHT_SAND_3),
        'pipeline': (DEEP_PURPLE_1, DEEP_PURPLE_2, DEEP_PURPLE_3, PURPLE_1,
                     PURPLE_2, PURPLE_3),
        'pipeline_io': (SEA_1, SEA_2, SEA_3, LIGHT_SEA_1, LIGHT_SEA_2,
                        LIGHT_SEA_3),
        'iteration': (GREEN_1, GREEN_2, GREEN_3, LIGHT_GREEN_1, LIGHT_GREEN_2,
                      LIGHT_GREEN_3),
        'attributed': (SKY_1, SKY_2, SKY_3, LIGHT_SKY_1, LIGHT_SKY_2,
                       LIGHT_SKY_3),
        'optional_output_switch': (APPLE_1, APPLE_2, APPLE_3, LIGHT_APPLE_1,
                                   LIGHT_APPLE_2, LIGHT_APPLE_3),
        'custom_node': (ORANGE_1, ORANGE_2, ORANGE_3, LIGHT_ORANGE_1,
                        LIGHT_ORANGE_2, LIGHT_ORANGE_3),
    }
    if node is pipeline.pipeline_node:
        style = 'pipeline_io'
    elif isinstance(node, OptionalOutputSwitch):
        style = 'optional_output_switch'
    elif isinstance(node, Switch):
        style = 'switch'
    elif isinstance(node, PipelineNode):
        style = 'pipeline'
    elif isinstance(node, ProcessNode) \
            and isinstance(node.process, ProcessIteration):
        style = 'iteration'
    elif isinstance(node, ProcessNode):
        if hasattr(node.process, 'completion_engine'):
            style = 'attributed'
        else:
            style = 'default'
    else:
        style = 'custom_node'
    if node.activated and node.enabled:
        color_1, color_2, color_3 = _colors[style][0:3]
    else:
        color_1, color_2, color_3 = _colors[style][3:6]
    if node in pipeline.disabled_pipeline_steps_nodes():
        color_1 = _color_disabled(color_1)
        color_2 = _color_disabled(color_2)
        color_3 = _color_disabled(color_3)
    return color_1, color_2, color_3, style

def pipeline_link_color(plug, link):
    '''
    Link color and style for graphical display and graphviz graphs.

    plug: Plug
        the plug the link belong to
    link: link tuple (5 values)
        link to color

    Returns
    -------
    link_props: tuple
        (color, style, active, weak) where color is a RGB tuple of float values
        (between 0. and 1.), style is a string ("solid", "dotted"), active and
        weak are booleans.
    '''
    GRAY_1 = (0.7, 0.7, 0.8)
    RED_2 = (0.92, 0.51, 0.12)

    active = plug.activated and link[3].activated
    if active:
        color = RED_2
    else:
        color = GRAY_1
    weak = link[4]
    if weak:  # weak link
        style = 'dotted'
    else:
        style = 'solid'
    return color, style, active, weak

def dot_graph_from_pipeline(pipeline, nodes_sizes={}, use_nodes_pos=False,
                            include_io=True, enlarge_boxes=0.):
    '''
    Build a graphviz/dot-compatible representation of the pipeline.
    The pipeline representation uses one link between two given boxes:
    parameters are not represented.

    This is different from the workflow graph, as given by
    :py:meth:`capsul.pipeline.Pipeline.workflow_graph`, in that the full graph
    is represented here, including disabled nodes.

    To build a workflow graph, see :py:func:`dot_graph_from_workflow`.

        pipeline to convert to a dot graph
    nodes_sizes: dict (optional)
        nodes sizes may be specified here, keys are node names, and values are
        tuples (width, height). Special "inputs" and "outputs" keys represent
        the global inputs/outputs blocks of the pipeline.
    use_nodes_pos: bool (optional)
        if True, nodes positions in the pipeline.node_position variable will
        be used to force nodes positions in the graph.
    include_io: bool (optional)
        If True, build a node for the pipeline inputs and a node for the
        pipeline outputs. If False, these nodes will not be generated, and
        their links ignored.
    enlarge_boxes: float (optional)
        when nodes sizes are specified, enlarge them by this amount to produce
        bigger boxes

    Returns
    -------
    dot_graph: tuple
        a (nodes, edges) tuple, where nodes is a list of node tuples
        (id, node_name, props) and edges is a dict, where
        keys are (source_node_id, dest_node_id), and values are tuples
        (props, active, weak). In both cases props is a dictionary of
        properties.
        This representation is simple and is meant to feed
        :py:func:`save_dot_graph`
    '''

    def _link_color(plug, link):
        if link[4]:  # weak link
            style = 'dotted'
        else:
            style = 'solid'
        active = plug.activated and link[3].activated
        return (0, 0, 0), style, active, link[4]

    nodes = []
    edges = {}
    has_outputs = False
    nodes_pos = pipeline.node_position
    scale = 1. / 67.

    for node_name, node in six.iteritems(pipeline.nodes):
        if node_name == '':
            if not include_io:
                continue
            id = 'inputs'
        else:
            id = node_name
        color, bgcolor, darkcolor, style = pipeline_node_colors(pipeline, node)
        colorstr = '#%02x%02x%02x' % tuple([int(c * 255.9) for c in darkcolor])
        bgcolorstr = '#%02x%02x%02x' % tuple([int(c * 255.9) for c in bgcolor])
        node_props = {'color': colorstr, 'fillcolor': bgcolorstr}
        if style == 'switch':
            node_props.update({'shape': 'house', 'orientation': 270.})
        else:
            node_props.update({'shape': 'box'})
        if use_nodes_pos:
            pos = nodes_pos.get(id)
            if pos is not None:
                node_props.update({'pos': '%f,%f' % (pos[0] * scale,
                                                     -pos[1] * scale)})
        size = nodes_sizes.get(id)
        if size is not None:
            node_props.update({'width': (size[0] + enlarge_boxes) * scale,
                               'height': (size[1] + enlarge_boxes) * scale,
                               'fixedsize': 'true'})
        if node_name != '':
            nodes.append((id, node_name, node_props))
        has_inputs = False
            if (plug.output and node_name != '') \
                    or (not plug.output and node_name == ''):
                    has_inputs = True
                    color, style, active, weak = pipeline_link_color(
                        plug, link)
                    dest = link[0]
                    if dest == '':
                        if not include_io:
                            continue
                        dest = 'outputs'
                        has_outputs = True
                    edge = (id, dest)
                    old_edge = edges.get(edge)
                    if old_edge is not None:
                        # use strongest color/style
                        if not old_edge[2]:
                            weak = False
                            style = old_edge[0]['style']
                        if old_edge[1]:
                            active = True
                            color = old_edge[0]['color']
                    if isinstance(color, tuple):
                        color = '#%02x%02x%02x' \
                                % tuple([int(c * 255.9) for c in color])
                    props = {'color': color, 'style': style}
                    edges[edge] = (props, active, weak)
        if node_name == '' and include_io:
            main_node_props = dict(node_props)
            if has_inputs:
                nodes.append(('inputs', 'inputs', node_props))
    if has_outputs:
        size = nodes_sizes.get('outputs')
        for prop in ('width', 'height', 'fixedsize'):
            if prop in main_node_props:
                del main_node_props[prop]
            main_node_props.update(
                {'width': (size[0] + enlarge_boxes) * scale,
                 'height': (size[1] + enlarge_boxes) * scale,
                 'fixedsize': 'true'})
        nodes.append(('outputs', 'outputs', main_node_props))

    return (nodes, edges)

def dot_graph_from_workflow(pipeline, nodes_sizes={}, use_nodes_pos=False,
                            enlarge_boxes=0.):
    '''
    Build a graphviz/dot-compatible representation of the pipeline workflow.

    This is different from the pipeline graph, as obtained
    by:py:func:`dot_graph_from_pipeline`, since only used parts are visible
    here: switches and disabled branches are removed.

    Returns
    -------
    '''

    graph = pipeline.workflow_graph()
    nodes = []
    edges = {}

    for n in graph._nodes:
        node = pipeline.nodes[n]
        else:
            pos = pipeline.node_position.get(n)
        size = nodes_sizes.get(n)
        nodes.append((n, n, node_props))
    for n, v in graph._links:
        edge = (n, v)
        props = {'color': '#eb821e', 'style': 'solid'}
        edges[edge] = (props, True, False)

def save_dot_graph(dot_graph, filename, **kwargs):
    '''
    Write a graphviz/dot input file, which can be used to generate an
    image representation of the graph, or to make dot automatically
    position nodes.

    dot_graph: dot graph
        representation of the pipeline, obatained using
        :py:func:`dot_graph_from_pipeline`
    filename: string
        file name to save the dot definition in
    **kwargs: additional attributes for the dot graph
      like nodesep=0.1 or rankdir="TB"
    '''

    def _str_repr(item):
        if isinstance(item, six.string_types):
            return '"%s"' % item
        return repr(item)

    fileobj = open(filename, 'w')
    props = {'rankdir': 'LR'}
    props.update(kwargs)
    propsstr = ' '.join(['='.join([aname, _str_repr(val)])
                         for aname, val in six.iteritems(props)])
    rankdir = props['rankdir']

    fileobj.write('digraph {%s;\n' % propsstr)
    nodesep = 20.  # in qt scale space
    for id, node, props in dot_graph[0]:
        if rankdir == 'TB' and 'orientation' in props:
            props = dict(props)
            props['orientation'] -= 90
        attstr = ' '.join(['='.join([aname, _str_repr(val)])
                           for aname, val in six.iteritems(props)])
        if len(props) != 0:
            attstr = ' ' + attstr
        fileobj.write('  "%s" [label="%s" style="filled"%s];\n'
                      % (id, node, attstr))
    for edge, descr in six.iteritems(dot_graph[1]):
        props = descr[0]
        fileobj.write('  "%s" -> "%s" [%s];\n'
                      % (edge[0], edge[1], attstr))
    fileobj.write('}\n')

def save_dot_image(pipeline, filename, nodes_sizes={}, use_nodes_pos=False,
                   include_io=True, enlarge_boxes=0., workflow=False,
                   format=None, **kwargs):
    '''
    Save a dot/graphviz image of the pipeline in a file.

    It may use either the complete pipeline graph (with switches and disabled
    branches), or the workflow, hiding disabled parts (see the workflow
    parameter).

    Basically combines :py:func:`dot_graph_from_pipeline` or
    :py:func:`dot_graph_from_workflow`, and :py:func:`save_dot_graph`, then
    runs the `dot <http://www.graphviz.org>`_ command, which has to be
    installed and available on the system.

        file name to save the dot definition in.
    workflow: bool (optional)
        if True, the workflow corresponding to the current pipeline state will
        be used instead of the complete graph: disabled parts will be hidden.
    format: string (optional)
        dot output format (see `dot <http://www.graphviz.org>`_ command doc).
        If not specified, guessed from the file name extension.
    '''
    if workflow:
        dgraph = dot_graph_from_workflow(
            pipeline, nodes_sizes=nodes_sizes, use_nodes_pos=use_nodes_pos,
            enlarge_boxes=enlarge_boxes)
    else:
        dgraph = dot_graph_from_pipeline(
            include_io=include_io, enlarge_boxes=enlarge_boxes)
    tempf = tempfile.mkstemp()
    os.close(tempf[0])
    dot_filename = tempf[1]
    save_dot_graph(dgraph, dot_filename, **kwargs)
    if format is None:
        ext = filename.split('.')[-1]
        formats = {'txt': 'plain'}
        format = formats.get(ext, ext)
    cmd = ['dot', '-T%s' % ext, '-o', filename, dot_filename]
    soma.subprocess.check_call(cmd)
    os.unlink(dot_filename)

def disable_runtime_steps_with_existing_outputs(pipeline):
    '''
    Disable steps in a pipeline which outputs contain existing files. This
    disabling is the "runtime steps disabling" one (see
    :py:class:`capsul.pipeline.Pipeline`), not the node disabling with
    activation propagation, so it doesn't affect the actual pipeline state.
    The aim is to prevent overwriting files which have already been processed,
    and to allow downstream execution of the remaining steps of the pipeline.

    pipeline: Pipeline (mandatory)
        pipeline to disable nodes in.
    '''
    steps = getattr(pipeline, 'pipeline_steps', Controller())
    for step, trait in six.iteritems(steps.user_traits()):
        if not getattr(steps, step):
            continue  # already inactive
        for node_name in trait.nodes:
            node = pipeline.nodes[node_name]
            if not node.enabled or not node.activated:
                continue  # node not active anyway
            process = node.process
            for param in node.plugs:
                trait = process.trait(param)
                if trait.output and (isinstance(trait.trait_type, traits.File)
                                     or isinstance(trait.trait_type, traits.Directory)):
                    value = getattr(process, param)
                    if value is not None and value is not traits.Undefined \
                            and os.path.exists(value):
                        # check special case when the output is also an input
                        # (of the same node)
                        disable = True
                        for n, t in six.iteritems(process.user_traits()):
                            if not t.output and (isinstance(t.trait_type,
                                                            traits.File)
                                                 or isinstance(t.trait_type,
                                                               traits.Directory)):
                                v = getattr(process, n)
                                if v == value:
                                    disable = False
                                    break  # found in inputs
                        if disable:
                            # disable step
                            print('disable step', step, 'because of:',
                                  node_name, '.', param)
                            setattr(steps, step, False)
                            # no need to iterate other nodes in same step
                            break

def nodes_with_existing_outputs(pipeline, exclude_inactive=True,
                                recursive=False, exclude_inputs=True):
    '''
    Checks nodes in a pipeline which outputs contain existing files on the
    filesystem. Such nodes, maybe, should not run again. Only nodes which
    actually produce outputs are selected this way (switches are skipped).

    exclude_inactive: bool (optional)
        if this option is set, inactive nodes will not be checked
        nor returned in the list. Inactive means disabled, not active, or in a
        disabled runtime step.
        Default: True
    recursive: bool (optional)
        if this option is set, sub-pipelines will not be returned as a whole
        but will be parsed recursively to select individual leaf nodes.
        Default: False
    exclude_inputs: bool (optional, default: True)
        Some processes or pipelines have input/output files: files taken as
        inputs which are re-written as outputs, or may carry an input file to
        the outputs through a switch selection (in the case of preprocessing
        steps, for instance).
        If this option is set, such outputs which also appear in the same node
        inputs will not be listed in the existing outputs, so that they will
        not be erased by a cleaning operation, and will not prevent execution
        of these nodes.

    Returns
    -------
    selected_nodes: dict
        keys: node names
        values: list of pairs (param_name, file_name)
    '''
    selected_nodes = {}
    if exclude_inactive:
        steps = getattr(pipeline, 'pipeline_steps', Controller())
        disabled_nodes = set()
            if not getattr(steps, step):
                disabled_nodes.update(trait.nodes)

    # nodes = pipeline.nodes.items()
    nodes = list(pipeline.nodes.items())
    while nodes:
        node_name, node = nodes.pop(0)
        if node_name == '' or not hasattr(node, 'process'):
            # main pipeline node, switch...
            continue
        if not node.enabled or not node.activated \
                or (exclude_inactive and node_name in disabled_nodes):
            continue
        process = node.process
        if recursive and isinstance(process, Pipeline):
            nodes += [('%s.%s' % (node_name, new_name), new_node)
                      for new_name, new_node in six.iteritems(process.nodes)
                      if new_name != '']
            continue
        plug_list = []
        input_files_list = set()
            trait = process.trait(plug_name)
            if isinstance(trait.trait_type, traits.File) \
                    or isinstance(trait.trait_type, traits.Directory) \
                    or isinstance(trait.trait_type, traits.Any):
                if isinstance(value, six.string_types) \
                        and os.path.exists(value) \
                        and value not in input_files_list:
                        plug_list.append((plug_name, value))
                    elif exclude_inputs:
                        input_files_list.add(value)
        if exclude_inputs:
            new_plug_list = [item for item in plug_list
                             if item[1] not in input_files_list]
            plug_list = new_plug_list
        if plug_list:
            selected_nodes[node_name] = plug_list
    return selected_nodes

def nodes_with_missing_inputs(pipeline, recursive=True):
    '''
    Checks nodes in a pipeline which inputs contain invalid inputs.
    Inputs which are files non-existing on the filesystem (so, which cannot
    run), or have missing mandatory inputs, or take as input a temporary file
    which should be the output from another disabled node, are recorded.

        but will be parsed recursively to select individual leaf nodes. Note
        that if not set, a pipeline is regarded as a process, but pipelines may
        not use all their inputs/outputs so the result might be inaccurate.

    Returns
    -------
    '''
    disabled_nodes = set()
            disabled_nodes.update(
                [pipeline.nodes[node_name] for node_name in trait.nodes])

            continue
        if not node.enabled or not node.activated or node in disabled_nodes:
            continue
            continue
                if isinstance(trait.trait_type, traits.File) \
                        or isinstance(trait.trait_type, traits.Directory):
                    value = getattr(process, plug_name)
                    keep_me = False
                    if value is None or value is traits.Undefined \
                            or value == '' or not os.path.exists(value):
                        # check where this file comes from
                        origin_node, origin_param, origin_parent \
                            = where_is_plug_value_from(plug, recursive)
                        if origin_node is not None \
                                and (origin_node in disabled_nodes
                                     or origin_parent in disabled_nodes):
                            # file coming from another disabled node
                            # if not value or value is traits.Undefined:
                            ## temporary one
                            # print('TEMP: %s.%s' % (node_name, plug_name))
                            # value = None
                            keep_me = True
                        elif origin_node is None:
                            # unplugged: does not come from anywhere else
                            if (value is not traits.Undefined and value) \
                                    or not plug.optional:
                                # non-empty value, non-existing file
                                # or mandatory, empty value
                                keep_me = True
                        # the rest is a plugged input from another process,
                        # or an optional empty value
                    if keep_me:
                        plug_list = selected_nodes.setdefault(node_name, [])

def where_is_plug_value_from(plug, recursive=True):
    '''
    Find where the given (input) plug takes its value from.
    It has to be the output of an uphill process, or be unconnected.
    Looking for it may involve ascending through switches or pipeline walls.

    plug: Plug instance (mandatory)
        the plug to find source connection with

    Returns
    -------
    node: Node
        origin pipeline node. May be None if the plug was not connected to an
        active source.
    param_name: string
        origin plug name in the origin node. May be None if node is None
    parent: Node
        Top-level parent node of the origin node. Useful to determine if the
        origin node is in a runtime pipeline step, which only records top-level
        nodes.
    '''
    links = [link + (None,) for link in plug.links_from]
    while links:
        node_name, param_name, node, in_plug, weak, parent = links.pop(0)
        if not node.activated or not node.enabled:
            # disabled nodes are not influencing
            continue
        if isinstance(node, PipelineNode):
            if not recursive:
                return node, param_name, parent
            else:  # recursive
                # either output from a sibling sub_pipeline
                # or input from parent pipeline
                # but it is handled the same way.
                # check their inputs
                # just if sibling, keep them as parent
                if in_plug.output and parent is None:
                    new_parent = node
                else:
                    new_parent = parent
                links += [link + (new_parent,) for link in in_plug.links_from]
        else:
            other_end = node.get_connections_through(param_name, single=True)
            if other_end:
                return other_end[0][0], other_end[0][1], parent
    # not found
    return None, None, None

def find_plug_connection_sources(plug, pipeline=None):
    '''
    A bit like :func:`where_is_plug_value_from` but looks for all incoming
    connection sources

    Returns
    -------
    sources:  list
        [(node, param_name, parent_node), ...]
    '''
    sources = []
    if isinstance(pipeline, Pipeline):
        pipeline = pipeline.pipeline_node
    links = [link + (pipeline,) for link in plug.links_from]
            continue
            if not in_plug.links_from:
                # get out of the pipeline: keep it
                sources.append((node, param_name, parent))
                continue
            # either output from a sibling sub_pipeline
            # or input from parent pipeline
            # but it is handled the same way.
            # check their inputs
            # just if sibling, keep them as parent
            if in_plug.output and parent in (None, pipeline):
                new_parent = node
            else:
                new_parent = parent
            links += [link + (new_parent,) for link in in_plug.links_from]
        else:
            other_end = node.get_connections_through(param_name, single=False)
            for src in other_end:
                if not src[2].output and node is pipeline \
                        and not src[2].links_from:
                    # main pipeline input, keep it
                    sources.append((src[0], src[1], node))
                elif src[2] is in_plug:
                    # don't get through its node: keep the node as source
                elif src[2].output and not isinstance(src[0], PipelineNode):
                    # sub-pipeline output: inspect it
                    links.append((None, src[1], src[0], src[2], False, node))
                elif not src[2].output or isinstance(src[0], PipelineNode):
                    # input side of a non-opaque node: inspect its links
                    links += [link + (node,)
                              for link in src[2].links_from]
                else:
                    print('unhandle case in find_plug_connection_sources')
                    print('node:', src[0], ', param:', src[1])

    return sources

def find_plug_connection_destinations(plug, pipeline=None):
    '''
    A bit like :func:`find_plug_connection_sources` but the other way

    Returns
    -------
    dest:  list
    '''
    dest = []
    links = [link + (pipeline,) for link in plug.links_to]
            continue
            if not in_plug.links_to:
                dest.append((node, param_name, parent))
                continue
            # either input from a sibling sub_pipeline
            # or output from parent pipeline
            if not in_plug.output and parent in (None, pipeline):
            else:
            links += [link + (new_parent,) for link in in_plug.links_to]
        else:
            for dst in other_end:
                if dst[2].output and node is pipeline \
                        and not dst[2].links_to:
                    # main pipeline output, keep it
                    dest.append((dst[0], dst[1], node))
                elif dst[2] is in_plug:
                    # don't get through its node: keep the node as dest
                elif not dst[2].output \
                        and not isinstance(dst[0], PipelineNode):
                    # sub-pipeline input: inspect it
                    links.append((None, dst[1], dst[0], dst[2], False, node))
                elif dst[2].output or isinstance(dst[0], PipelineNode):
                    # output side of a non-opaque node: inspect its links
                              for link in dst[2].links_to]
                else:
                    print('node:', dst[0], ', param:', dst[1])

    return dest

def dump_pipeline_state_as_dict(pipeline):
    '''
    Get a pipeline state (parameters values, nodes activation, selected
    steps... in a dictionary.

    The returned dict may contain sub-pipelines state also.

    The dict may be saved, and used to restore a pipeline state, using
    :py:func:`set_pipeline_state_from_dict`.

    Note that :py:meth:`pipeline.export_to_dict <soma.controller.controller.export_to_dict>`
    would almost do the job, but does not include the recursive aspect.

    pipeline: Pipeline (or Process) instance
        pipeline (or process) to get state from

    Returns
    -------
    state_dict: dict
        pipeline state
    '''

    def should_keep_value(node, plug, components):
        '''
        Tells if a plug has already been taken into account in the plugs graph.

        Also filters out switches outputs, which should rather be set via their
        inputs.

        To do so, a connected components map has to be built for the plugs
        graph, which is done is a semi-lazy way in components.

        node: Node
            pipeline node the pluge belongs to
        plug: Plug
            the plug to test
        components: list of sets of plugs
            connected components will be added to this components list.

        Returns
        -------
        True if the plug value is a "new" one and should be recorded in the
        pipeline state. Otherwise it should be discarded (set from another
        connected plug).
        '''

        def _component(plug, components):
            for comp in components:
                if plug in comp:
                    return comp

        comp = _component(plug, components)
        if comp:
            # already done
            return False
        comp = set()
        todo = []
        todo.append(plug)
        allowed = True
        # propagate
            plug = todo.pop(0)
            comp.add(plug)
            # switches outputs should not be set (they will be through their
            # inputs)
            if plug.output and isinstance(node, Switch):
                allowed = False
            todo += [link[3] for link in plug.links_from
                     if link[3] not in comp]
            todo += [link[3] for link in plug.links_to
        components.append(comp)
        return allowed

    def prune_empty_dicts(state_dict):
        '''
        Remove empty dictionaries, and nodes containing empty dicts in pipeline
        state dictionary

        state_dict: dict
            the state_dict is parsed, and modified.
        '''
        if state_dict.get('nodes') is None:
            return
        todo = [(state_dict, None, None, True)]
            current_dict, parent, parent_key, recursive = todo.pop(0)
            nodes = current_dict.get('nodes')
            modified = False
            if nodes:
                if len(nodes) == 0:
                    del current_dict['nodes']
                    modified = True
                elif recursive:
                    todo = [(value, nodes, key, True)
                            for key, value in six.iteritems(nodes)] + todo
            if len(current_dict) == 0 and parent is not None:
                del parent[parent_key]
            elif modified:
                todo.append((current_dict, parent, parent_key, False))

    state_dict = {}
    nodes = [(None, pipeline.pipeline_node, state_dict)]
    components = []
        node_name, node, current_dict = nodes.pop(0)
        proc = node
            proc = node.process
        node_dict = proc.export_to_dict()
        # filter out forbidden and already used plugs
            if not should_keep_value(node, plug, components):
                del node_dict[plug_name]
        if node_name is None:
            if len(node_dict) != 0:
                current_dict['state'] = node_dict
            child_dict = current_dict
        else:
            child_dict = {}
            current_dict.setdefault('nodes', {})[node_name] = child_dict
                child_dict['state'] = node_dict
        if hasattr(proc, 'nodes'):
            nodes += [(child_node_name, child_node, child_dict)
                      for child_node_name, child_node
                      in six.iteritems(proc.nodes) if child_node_name != '']

    prune_empty_dicts(state_dict)
    return state_dict

def set_pipeline_state_from_dict(pipeline, state_dict):
    '''
    Set a pipeline (or process) state from a dict description.

    State includes parameters values, nodes activation, steps selection etc.
    The state is generally taken using :py:func:`dump_pipeline_state_as_dict`.

    pipeline: Pipeline or Process instance
        process to set state in
    state_dict: dict (mapping object)
    '''
    nodes = [(pipeline, state_dict)]
        node, current_dict = nodes.pop(0)
        else:
            proc = node
        proc.import_from_dict(current_dict.get('state', {}))
        sub_nodes = current_dict.get('nodes')
        if sub_nodes:
            nodes += [(proc.nodes[node_name], sub_dict)
                      for node_name, sub_dict in six.iteritems(sub_nodes)]

def get_output_directories(process):
    '''
    Get output directories for a process, pipeline, or node

    Returns
    -------
    dirs: dict
        organized directories list: a dict with recursive nodes mapping.
        In each element, the "directories" key holds a directories names set,
        and "nodes" is a dict with sub-nodes (node_name, dict mapping,
        organized the same way)
    flat_dirs: set
        set of all directories in the pipeline, as a flat set.
    '''
    all_dirs = set()
    root_dirs = {}
    nodes = [(process, '', root_dirs)]
    if isinstance(process, Pipeline):
        disabled_nodes = set(process.disabled_pipeline_steps_nodes())
    elif isinstance(process, PipelineNode):
        disabled_nodes = set(process.process.disabled_pipeline_steps_nodes())

        node, node_name, dirs = nodes.pop(0)
        plugs = getattr(node, 'plugs', None)
        if plugs is None:
            plugs = node.user_traits()
        else:
            process = node
        dirs_set = set()
        dirs['directories'] = dirs_set
        for param_name in plugs:
            trait = process.trait(param_name)
            if trait.output and isinstance(trait.trait_type, traits.File) \
                    or isinstance(trait.trait_type, traits.Directory):
                value = getattr(process, param_name)
                    directory = os.path.dirname(value)
                    if directory not in ('', '.'):
                        all_dirs.add(directory)
                        dirs_set.add(directory)
        sub_nodes = getattr(process, 'nodes', None)
            # TODO: handle disabled steps
            sub_dict = {}
            dirs['nodes'] = sub_dict
            for node_name, node in six.iteritems(sub_nodes):
                if node_name != '' and node.activated and node.enabled \
                        and not node in disabled_nodes:
                    sub_node_dict = {}
                    sub_dict[node_name] = sub_node_dict
                    nodes.append((node, node_name, sub_node_dict))
    return root_dirs, all_dirs

def create_output_directories(process):
    '''
    Create output directories for a process, pipeline or node.
    '''
    for directory in get_output_directories(process)[1]:
        if not os.path.exists(directory):
            os.makedirs(directory)

def save_pipeline(pipeline, file, format=None):
    '''
    Save the pipeline either in XML, JSON, or .py source file

    pipeline: Pipeline instance
    file: file object or str
        either a filename, or a file-like stream
    format: str
        'py', 'xml'... If not specified and file is a file name, it will be
        guessed from its extension. If file is not a string, then format will
        default to xml.
    '''
    from capsul.pipeline.xml import save_xml_pipeline
    from capsul.pipeline.python_export import save_py_pipeline
    from capsul.pipeline.json_io import save_json_pipeline

    if not isinstance(pipeline, Pipeline):
        # "pipeline" is actually a single process (or should, if it is not a
        # pipeline). Get it into a pipeline (with a single node) to make the
        # workflow.
        new_pipeline = Pipeline()
        new_pipeline.name = pipeline.name
        new_pipeline.set_study_config(pipeline.get_study_config())
        new_pipeline.add_process('main', pipeline)
        new_pipeline.autoexport_nodes_parameters(include_optional=True)
        pipeline = new_pipeline

    formats = {'.py': save_py_pipeline,
              '.xml': save_xml_pipeline,
              '.json': save_json_pipeline}

    if not format and isinstance(file, str):

        saved = False
        for ext, writer in six.iteritems(formats):
            if file.endswith(ext):
                format = ext[1:]

    if not format:
        format = 'xml'

    writer = formats['.%s' % format]
    writer(pipeline, file)

def load_pipeline_parameters(filename, pipeline):
    """
    Loading and setting pipeline parameters (inputs and outputs) from a Json file.
    """

    if filename:
        with io.open(filename, 'r', encoding='utf8') as file:
            dic = json.load(file)

        if "pipeline_parameters" not in dic:
            raise KeyError('No "pipeline_parameters" key found in {0}.'.format(filename))

        for trait_name, trait_value in dic["pipeline_parameters"].items():
            if trait_name not in pipeline.user_traits():
                # Should we raise an error or just "continue"?
                raise KeyError('No "{0}" parameter in pipeline.'.format(trait_name))

            try:
                setattr(pipeline, trait_name, trait_value)
            except traits.TraitError:
                # This case happen when the trait type is date, time or datetime
                # Couldn't find an other solution for now
                if trait_value not in (None, traits.Undefined, ''):
                    t = pipeline.trait(trait_name)
                    if isinstance(t.trait_type, traits.Date):
                        trait_value = date.fromisoformat(trait_value)
                    elif isinstance(t.trait_type, traits.Time):
                        trait_value = time.fromisoformat(trait_value)
                    elif isinstance(t.trait_type, traits.Datetime):
                        trait_value = datetime.fromisoformat(trait_value)
                    else:
                        trait_value = None
                try:
                    setattr(pipeline, trait_name, trait_value)
                    setattr(pipeline, trait_name, traits.Undefined)

        pipeline.update_nodes_and_plugs_activation()

def save_pipeline_parameters(filename, pipeline):
    """
    Saving pipeline parameters (inputs and outputs) to a Json file.
    """

    def check_value(val):
        """
        Checking if the value is a list, Undefined, a date or a time
        :param val: value
        :return: the serializable value
        """
        if type(val) in [list, traits.TraitListObject, traits.List]:
            for idx, element in enumerate(val):
                new_list_value = check_value(element)
                val[idx] = new_list_value

        if val is traits.Undefined:
            val = ""

        if type(val) in [date, time, datetime]:
            val = str(val)

        return val

        # Generating the dictionary
        param_dic = {}
        for trait_name, trait in pipeline.user_traits().items():
            if trait_name in ["nodes_activation"]:
                continue
            value = check_value(getattr(pipeline, trait_name))
            param_dic[trait_name] = value

        # In the future, more information may be added to this dictionary
        dic = {}
        dic["pipeline_parameters"] = param_dic

        # Saving the dictionary in the Json file
        if sys.version_info[0] >= 3:
            with open(filename, 'w', encoding='utf8') as file:
                json.dump(dic, file)
        else:
            with open(filename, 'w') as file:

def find_node(pipeline, node):
    ''' Find the given node in the pipeline or a sub-pipeline

    Returns
    -------
    node_chain: list
        list of node names in the pipeline going through sub-pipelines to the
        given node
    '''
    nodes = []
    pipelines = [(pipeline.pipeline_node, [])]
    while pipelines:
        n, names = pipelines.pop(0)
        for sk, sn in six.iteritems(n.process.nodes):
            if node is sn:
                return names + [sk]
            if sn is not n and isinstance(sn, PipelineNode):
                pipelines.append((sn, names + [sk]))

    raise KeyError('Node %s not found in the pipeline %s'
                   % (node.name, pipeline.name))

def is_node_enabled(pipeline, node_name=None, node=None, use_steps=True):
    ''' Checks if the given node is enabled in the pipeline.
    It may be disabled if it has its ``enabled`` or ``activated`` properties set to False, or if it is part of a disabled step (if used_steps is True).
    The node may be given as a Node instance, or its name in the pipeline.
    '''
    names = [node_name]
    if node is None:
        node = pipeline.nodes[node_name]

    if not node.enabled or not node.activated:

    elif node_name is None:

        if not node.enabled or not node.activated:

        # probably a node of a sub-pipeline
        names = find_node(pipeline, node)

    p = pipeline
    for name in names:
        if use_steps:
            steps = getattr(p, 'pipeline_steps', Controller())
                if not getattr(steps, step) and name in trait.nodes:
        p = p.nodes[name]
        p = getattr(p, 'process', p)

    # not disabled ? OK then it's enabled
    return True

def trait_str(trait, with_att=True):

    def str_from_trait_id(tid, with_att=True):
        vals = []
        if tid[0] == 'Enum':
            vals += [repr(x) for x in trait.handler.values]
        tname = tid[0]
        if tname.startswith('List_'):
            t = tname
            tname = 'List'
            while t.startswith('List_'):
                t = t[5:]
                vals = [str_from_trait_id([t], False)]
        if with_att:
            vals += ['output=%s' % repr(bool(trait.output)),
                     #'default_value=%s' % repr(trait.default),
                     'optional=%s' % repr(bool(trait.optional))]
            if trait.input_filename:
                vals.append('input_filename=True')
        t_str = 'traits.%s(%s)' % (tname, ', '.join(vals))

        return t_str

    from soma.controller.trait_utils import trait_ids

    tid = trait_ids(trait)
    return str_from_trait_id(tid, with_att)

def write_fake_process(process, filename, sleep_time=0):
    ''' Write a "fake process" with same class name and parameters as the input
    process, but with a fake execution function meant for tests.
    '''

    with open(filename, 'w') as f:
        f.write('''# -*- coding: utf-8 -*-

from capsul.api import Process
import os

class %s(Process):
        super(%s, self).__init__()
        self.name = '%s'

''' % (process.__class__.__name__, process.__class__.__name__, process.name))

        for name, trait in process.user_traits().items():
            t_str = trait_str(trait, False)[:-1]
            meta = {k: v for k, v in trait.__dict__.items()
                    if k not in {'type', 'trait'}}
            meta_str = ''
            if meta:
                meta_str = ', '.join('%s=%s' % (k, repr(v))
                                     for k, v in meta.items())
                if not t_str.endswith('('):
                    meta_str = ', ' + meta_str
            f.write('        self.add_trait("%s", %s%s))\n'
                    % (name, t_str, meta_str))
            value = getattr(process, name, traits.Undefined)
            if value is not traits.Undefined:
                f.write('        self.%s = %s\n' % (name, repr(value)))

        f.write('''
        for name, trait in self.user_traits().items():
            if isinstance(trait.trait_type, traits.File):
                    outputs.append(name)
                    continue
                filename = getattr(self, name)
                if filename not in (None, traits.Undefined, ''):
                    if not os.path.exists(filename):
                          'Input parameter: %s, file %s does not exist'
                          % (name, repr(filename)))

''')
        if sleep_time != 0:
            f.write('        import time\n')
            f.write('        time.sleep(%f)\n\n' % sleep_time)
        f.write('''        for name in outputs:
            trait = self.trait(name)
            filename = getattr(self, name)
            if filename not in (None, traits.Undefined, ''):
                with open(filename, 'w') as f:
                    f.write('class: %s\\n' % self.__class__.__name__)
                    f.write('name: %s\\n' % self.name)
                    f.write('parameter: %s\\n' % name)
''')

def write_fake_pipeline(pipeline, module_name, dirname, sleep_time=0):
    ''' Write a "fake pipeline" with same class name, structure, and parameters
    as the input pipeline, but replacing its processes with "fake" processes
    which do not actually do a real job while executing.

    This is meant for tests, to mimic a "real" pipeline structure without its
    dependencies.

    :warning:`This function actually modifies the input pipeline, which is
    transformed into a fake one.`
    '''

    meta_forbidden = {'type', '_metadata'}

    def replace_node(node, module_name, dirname, done):
        basename = node.process.__class__.__name__.lower()
        modname = '.'.join([module_name, basename])
        filename = os.path.join(dirname, '%s.py' % basename)
        if modname not in done:
            done.add(modname)
            write_fake_process(node.process, filename, sleep_time=sleep_time)
        try:
            new_proc \
                = pipeline.get_study_config().engine.get_process_instance(
                    filename)
            print('Failed to reload node:', filename)
            raise
        new_proc.__class__.__module__ = modname

        # fix fields state (if modified)
        for name, trait in node.process.user_traits().items():
            if name not in new_proc.user_traits():
                new_proc.add_trait(name, trait)
                continue
                    if k not in meta_forbidden}
            for k, v in meta.items():
                setattr(new_proc.trait(name), k, v)
            # set back old value
            setattr(new_proc, name, getattr(node.process, name))

        node.process = new_proc

    sys.path.insert(0, dirname)
    dirname = os.path.join(dirname, module_name.rsplit('.')[-1])
    if not os.path.exists(dirname):
        os.makedirs(dirname)
    with open(os.path.join(dirname, '__init__.py'), 'w'):
        pass

    nodes = [node[1] for node in pipeline.nodes.items() if node[0] != '']
    done = set()
    pipelines = [pipeline]
        node = nodes.pop(0)
        if not hasattr(node, 'process'):
            continue
        if isinstance(node.process, Pipeline):
            if node.process.__class__ not in done:
                nodes += [n[1] for n in node.process.nodes.items()
                          if n[0] != '']
                node.process.__class__.__module__ = '.'.join(
                    [module_name, node.process.__class__.__name__.lower()])
                done.add(node.process.__class__)
                pipelines.append(node.process)
        elif isinstance(node.process, ProcessIteration):
            proc = node.process.process
            if isinstance(proc, Pipeline):
            else:
                replace_node(node, module_name, dirname, done)
        else:
            replace_node(node, module_name, dirname, done)

    for pipeline in reversed(pipelines):
        filename = os.path.join(dirname, '%s.py' \
            % pipeline.__class__.__name__.lower())
        save_pipeline(pipeline, filename)

    del sys.path[0]

"""Capsul Pipeline conversion into soma-workflow workflow.

Standard use case::

    workflow = workflow_from_pipeline(pipeline)
    controller, wf_id = workflow_run(workflow_name, workflow, study_config)

Functions
=========
:func:`workflow_from_pipeline`
:func:`workflow_run`
"""

import os
import socket
import sys
import six

import soma_workflow.client as swclient
import soma_workflow.info as swinfo

from capsul.pipeline.pipeline import Pipeline, Switch, PipelineNode
from capsul.process.process import Process
from capsul.pipeline.topological_sort import Graph
from traits.api import Directory, Undefined, File, Str, Any, List
from soma.sorted_dictionary import OrderedDict, SortedDictionary
from .process_iteration import ProcessIteration
from capsul.attributes import completion_engine_iteration
from capsul.attributes.completion_engine import ProcessCompletionEngine
from soma_workflow.custom_jobs import MapJob, ReduceJob

class TempFile(str):
    # class needed temporary to identify temporary paths in the pipeline.
    # must inherit a string type since it is used as a trait value
    def __init__(self, string=''):
        # in python3 super(..).__init__() cannot take an argument
        # moreover the str value is assigned anyway.
        super(TempFile, self).__init__()
        if isinstance(string, TempFile):
            self.pattern = string.pattern
            self.value = string.value
            self.ref = string.ref if string.ref else string
        else:
            self.pattern = '%s'
            self.value = string
            self.ref = None

    def referent(self):
        return self.ref if self.ref else self

    def get_value(self):
        return self.referent().value

    def __add__(self, other):
        res = TempFile(str(self) + str(other))
        res.pattern = self.pattern + str(other)
        res.value = self.value
        res.ref = self.referent()
        return res

    def __radd__(self, other):
        res = TempFile(str(other) + str(self))
        res.pattern = str(other) + self.pattern
        return res

    def __iadd__(self, other):
        self.pattern += str(other)
        str(TempFile, self).__iadd__(str(other))

        return self.pattern % self.get_value()

    def __hash__(self):
        if self.ref:
            return self.referent().__hash__()
        return super(TempFile, self).__hash__()

    def __eq__(self, other):
        if not isinstance(other, TempFile):
        return self.referent() is other.referent()

    def __lt__(self, other):
        return str(self) < other

    def __gt__(self, other):
        return str(self) > other

    def __le__(self, other):
        return str(self) <= other

    def __ge__(self, other):
        return str(self) >= other

def workflow_from_pipeline(pipeline, study_config=None, disabled_nodes=None,
                           jobs_priority=0, create_directories=True,
                           environment='global', check_requirements=True,
                           complete_parameters=False):
    """ Create a soma-workflow workflow from a Capsul Pipeline

        a CAPSUL pipeline
    study_config: StudyConfig (optional), or dict
        holds information about file transfers and shared resource paths.
        If not specified, it will be accessed through the pipeline.
    disabled_nodes: sequence of pipeline nodes (Node instances) (optional)
        such nodes will be disabled on-the-fly in the pipeline, file transfers
        will be adapted accordingly (outputs may become inputs in the resulting
        workflow), and temporary files will be checked. If a disabled node was
        to produce a temporary files which is still used in an enabled node,
        then a ValueError exception will be raised.
        If disabled_nodes is not passed, they will possibly be taken from the
        pipeline (if available) using disabled steps:
        see Pipeline.define_steps()
    jobs_priority: int (optional, default: 0)
        set this priority on soma-workflow jobs.
    create_directories: bool (optional, default: True)
        if set, needed output directories (which will contain output files)
        will be created in a first job, which all other ones depend on.
    environment: str (default: "global")
        configuration environment name (default: "global"). See
        :class:`capsul.engine.CapsulEngine` and
        :class:`capsul.engine.settings.Settings`.
    check_requirements: bool (default: True)
        if True, check the pipeline nodes requirements in the capsul engine
        settings, and issue an exception if they are not met instead of
        proceeding with the workflow.
    complete_parameters: bool (default: False)
        Perform parameters completion on the input pipeline while building
        the workflow. The default is False because we should avoid to do things
        several times when it's already done, but in iteration nodes,
        completion needs to be done anyway for each iteration, so this option
        offers to do the rest of the "parent" pipeline completion.

    Returns
    -------
    workflow: Workflow
        a soma-workflow workflow
    """

    def _files_group(path, merged_formats):
        bname = os.path.basename(path)
        l0 = len(path) - len(bname)
        p0 = 0
        paths = [path]
        while True:
            p = bname.find('.', p0)
            if p < 0:
                break
            ext = bname[p:]
            p0 = p + 1
            format_def = merged_formats.get(ext)
            if format_def:
                path0 = path[:l0 + p]
                paths += [path0 + e[0] for e in format_def]
                break
        paths.append(path + '.minf')
        return paths

    def _translated_path(path, shared_map, shared_paths, trait=None):
        if path is None or path is Undefined \
                or not shared_paths \
                or (trait is not None
                    and not isinstance(trait.trait_type, File) \
                    and not isinstance(trait.trait_type, Directory)):
            return None # not a path
        item = shared_map.get(path)
        if item is not None:
            # already in map
            return item

        for base_dir, (namespace, uuid) in six.iteritems(shared_paths):
            if path.startswith(base_dir + os.sep):
                rel_path = path[len(base_dir)+1:]
                #uuid = path
                item = swclient.SharedResourcePath(
                    rel_path, namespace, uuid=uuid)
                shared_map[path] = item
                return item

    def _replace_in_list(rlist, temp_map):
        for i, item in enumerate(rlist):
            if isinstance(item, (list, tuple, set)):
                deeperlist = list(item)
                _replace_in_list(deeperlist, temp_map)
                if isinstance(item, tuple):
                    deeperlist = tuple(deeperlist)
                elif isinstance(item, set):
                    deeperlist = set(deeperlist)
                rlist[i] = deeperlist
            #elif item is Undefined:
                #rlist[i] = ''
            elif isinstance(item, (dict, OrderedDict, SortedDictionary)):
                _replace_in_dict(item, temp_map)
            elif item in temp_map:
                value = temp_map[item]
                value = value.__class__(value)
                if hasattr(item, 'pattern'):
                    # temp case (differs from shared case)
                    value.pattern = item.pattern
                rlist[i] = value

    def _replace_in_dict(rdict, temp_map):
        for name, item in six.iteritems(rdict):
                rdict[name] = deeperlist
                #rdict[name] = ''
                rdict[name] = value

    def _get_replaced(rlist, temp_map):
        if isinstance(rlist, (dict, OrderedDict, SortedDictionary)):
            return _get_replaced(list(rlist.values()), temp_map)
        if isinstance(rlist, (list, tuple)):
            l = [x for x in [_get_replaced(y, temp_map) for y in rlist]
                 if x is not None]
            out = []
            while l:
                item = l.pop(0)
                if isinstance(item, list):
                    l = item + l
                else:
                    out.append(item)
            return out
        if rlist not in temp_map:
            if isinstance(rlist, swclient.SpecialPath):
                return rlist
        value = temp_map[rlist]
        if hasattr(rlist, 'pattern'):
            # temp case (differs from shared case)
            value.pattern = rlist.pattern
        return [value]

    def _replace_transfers(rlist, process, itransfers, otransfers):
        param_name = None
        i = 3
        for item in rlist[3:]:
            if param_name is None:
                param_name = item
            else:
                transfer = itransfers.get(param_name)
                if transfer is None:
                    transfer = otransfers.get(param_name)
                if transfer is not None:
                    value = transfer[0]
                    if isinstance(item, (list, tuple)):
                        # TODO: handle lists of files [transfers]
                        #deeperlist = list(item)
                        #_replace_in_list(deeperlist, transfers)
                        #rlist[i] = deeperlist
                        print('*** LIST! ***')
                    else:
                        rlist[i] = value
                param_name = None
            i += 1

    def _replace_dict_transfers(rdict, process, itransfers, otransfers):
        for param_name, item in six.iteritems(rdict):
            transfer = itransfers.get(param_name)
            if transfer is None:
                transfer = otransfers.get(param_name)
            if transfer is not None:
                value = transfer[0]
                if isinstance(item, (list, tuple)):
                    # TODO: handle lists of files [transfers]
                    #deeperlist = list(item)
                    #_replace_in_list(deeperlist, transfers)
                    #rdict[param_name] = deeperlist
                    print('*** LIST! ***')
                else:
                    rdict[param_name] = value

    def build_job(process, temp_map={}, shared_map={}, transfers=[{}, {}],
                  shared_paths={}, forbidden_temp=set(), name='', priority=0,
                  step_name='', engine=None, environment='global'):
        """ Create a soma-workflow Job from a Capsul Process

        process: Process (mandatory) or custom Node
            a CAPSUL process instance or custom Node instance
        temp_map: dict (optional)
            temporary paths map.
        shared_map: dict (optional)
            file shared translated paths, global pipeline dict.
            This dict is updated when needed during the process.
        shared_paths: dict (optional)
            holds information about shared resource paths base dirs for
            soma-workflow.
            If not specified, no translation will be used.
        forbidden_temp: dict
            values forbidden for temporary files (?)
        name: string (optional)
            job name. If empty, use the process name.
        priority: int (optional)
            priority assigned to the job
        step_name: str (optional)
            the step name will be stored in the job user_storage variable
        engine: CapsulEngine (optional)
            used to check configuration requirements for non-process nodes
        environment: str (default: "global")
            configuration environment name (default: "global"). See
            :class:`capsul.engine.CapsulEngine` and
            :class:`capsul.engine.settings.Settings`.

        Returns
        -------
        job: Job
            a soma-workflow Job instance that will execute the CAPSUL process
        """
        job_name = name
        if not job_name:
            job_name = process.name

        # check for special modified paths in parameters
        input_replaced_paths = []
        output_replaced_paths = []
        param_dict = {}
        has_outputs = False
        forbidden_traits = ('nodes_activation', 'selection_changed',
                            'activated', 'enabled', 'name', 'node_type', )
        for param_name, parameter in six.iteritems(process.user_traits()):
            if param_name not in forbidden_traits:
                if parameter.output \
                        and (parameter.input_filename is False
                             or not (isinstance(parameter.trait_type,
                                                (File, Directory))
                                     or (isinstance(parameter.trait_type,
                                                    List)
                                         and isinstance(
                                              parameter.inner_traits[0].trait_type,
                                              (File, Directory))))):
                    has_outputs = True
                    values = value
                else:
                    values = [value]
                for value in values:
                    if isinstance(value, TempFile):
                        # duplicate swf temp and copy pattern into it
                        tval = temp_map[value]
                        tval = tval.__class__(tval)
                        tval.pattern = value.pattern
                        if parameter.output:
                            output_replaced_paths.append(tval)
                        else:
                            if value in forbidden_temp:
                                raise ValueError(
                                    'Temporary value used cannot be generated '
                                    'in the workflkow: %s.%s'
                                    % (job_name, param_name))
                            input_replaced_paths.append(tval)
                    else:
                        _translated_path(value, shared_map, shared_paths,
                                        parameter)

        # Get the process command line
        #process_cmdline = process.get_commandline()
        process_cmdline = process.params_to_command()
        # and replace in commandline
        iproc_transfers = transfers[0].get(process, {})
        oproc_transfers = transfers[1].get(process, {})
        #proc_transfers = dict(iproc_transfers)
        #proc_transfers.update(oproc_transfers)
        _replace_in_list(process_cmdline, temp_map)
        _replace_in_list(process_cmdline, shared_map)
        _replace_transfers(
            process_cmdline, process, iproc_transfers, oproc_transfers)

        config = {}
        config = process.check_requirements(environment)
            # here we bypass unmet requirements, it's not our job here.
            config = {}

        use_input_params_file = False
        if process_cmdline[0] == 'capsul_job':
            # use python executable from config, if any
            pconf = config.get('capsul.engine.module.python')
            if not pconf:
                pconf = process.get_study_config().engine.settings. \
                    select_configurations(environment, {'python': 'any'})
                if pconf:
                    if not config:
                        config = pconf
                    else:
                        if 'capsul.engine.module.python' in pconf:
                            config['capsul.engine.module.python'] \
                                = pconf['capsul.engine.module.python']
                        uses = pconf.get('capsul_engine', {}).get('uses', {})
                        if uses:
                            config.setdefault('capsul_engine', {}).setdefault(
                                'uses', {}).update(uses)
            python_command = pconf.get(
                'capsul.engine.module.python', {}).get('executable')
            if not python_command:
                python_command = os.path.basename(sys.executable)
            path_trick = ''
            # python path cannot be passed in a library since the access to
            # this library (capsul module typically) may be conditioned by
            # this path. We cannot use PYTHONPATH env either because it would
            # completely erase any user settings (.bashrc). So we add it here.
            ppath = pconf.get(
                'capsul.engine.module.python', {}).get('path')
            if ppath:
                path_trick = 'import sys; sys.path = %s + sys.path; ' \
                    % repr(ppath)
            process_cmdline = [
                'capsul_job', python_command, '-c',
                '%sfrom capsul.api import Process; '
                'Process.run_from_commandline("%s")'
                % (path_trick, process_cmdline[1])]
            use_input_params_file = True
            param_dict = process.export_to_dict(exclude_undefined=True)
        elif process_cmdline[0] in ('json_job', 'custom_job'):
        for name in forbidden_traits:
            if name in param_dict:
                del param_dict[name]

        _replace_in_dict(param_dict, temp_map)
        _replace_in_dict(param_dict, shared_map)
        _replace_dict_transfers(
            param_dict, process, iproc_transfers, oproc_transfers)

        # handle native specification (cluster-specific specs as in
        # soma-workflow)
        native_spec = getattr(process, 'native_specification', None)

        # Return the soma-workflow job
        if process_cmdline[0] == 'custom_job':
            job = build_custom_job(
                process, process_cmdline, name=job_name,
                referenced_input_files
                    =input_replaced_paths \
                        + [x[0] for x in iproc_transfers.values()],
                referenced_output_files
                    =output_replaced_paths \
                        + [x[0] for x in oproc_transfers.values()],
                param_dict=param_dict)
        else:
            job = swclient.Job(
                name=job_name,
                command=process_cmdline[1:],
                priority=priority,
                native_specification=native_spec,
                param_dict=param_dict,
                use_input_params_file=use_input_params_file,
                has_outputs=has_outputs)
            # print('job command:', job.command)
            # handle parallel job info (as in soma-workflow)
            parallel_job_info = getattr(process, 'parallel_job_info', None)
            if parallel_job_info:
                job.parallel_job_info = parallel_job_info
        if step_name:
            job.user_storage = step_name
        if hasattr(process, 'uuid'):
            # propagate the process uuid, if any, to maintain link with it
            job.uuid = process.uuid
        job.configuration = config
        # associate job with process
        job.process = weakref.ref(process)
        job._do_not_pickle = ['process']
        job.process_hash = id(process)
        return job

    def build_custom_job(node, process_cmdline, name,
                         referenced_input_files, referenced_output_files,
                         param_dict):
        ''' Build a custom job (generally running on engine side) from a
        custom pipeline node.
        '''
        if hasattr(node, 'build_job'):
            job = node.build_job(
                name=name, referenced_input_files=referenced_input_files,
                referenced_output_files=referenced_output_files,
            return job

    def build_group(name, jobs):
        """ Create a group of jobs

            the group name
        jobs: list of Job (mandatory)
            the jobs we want to insert in the group

        Returns
        -------
        group: Group
            the soma-workflow Group instance
        """
        return swclient.Group(jobs, name=name)

    def get_jobs(group, groups):
        gqueue = list(group.elements)
        jobs = []
        while gqueue:
            group_or_job = gqueue.pop(0)
            if group_or_job in groups:
                gqueue += group_or_job.elements
            else:
                jobs.append(group_or_job)
        return jobs

    def assign_temporary_filenames(pipeline, count_start=0):
        ''' Find and temporarily assign necessary temporary file names'''
        temp_filenames = pipeline.find_empty_parameters()
        temp_map = {}
        count = count_start
        for node, plug_name, optional in temp_filenames:
            else:
            if trait.input_filename is False:
                # filename is explicitly an output: not a temporary.
                continue
            is_list = isinstance(trait.trait_type, List)
            values = []
            if is_list:
                todo = getattr(process, plug_name)
                trait = trait.inner_traits[0]
                if trait.input_filename is False:
                    # filename is explicitly an output: not a temporary.
                    continue
            else:
                todo = [Undefined]
            for item in todo:
                if item not in (Undefined, '', None):
                    # non-empty list element
                    values.append(item)
                    continue
                is_directory = isinstance(trait.trait_type, Directory)
                if trait.allowed_extensions:
                    suffix = trait.allowed_extensions[0]
                else:
                    suffix = ''
                swf_tmp = swclient.TemporaryPath(is_directory=is_directory,
                    suffix=suffix, name='temporary_%d' % count)
                tmp_file = TempFile('%d' % count)
                count += 1
                temp_map[tmp_file] = (swf_tmp, node, plug_name, optional)
                values.append(tmp_file)
            # set a TempFile value to identify the params / value
                setattr(process, plug_name, values)
            else:
                setattr(process, plug_name, values[0])
        return temp_map

    def restore_empty_filenames(temporary_map):
      ''' Set back Undefined values to temporarily assigned file names (using
      assign_temporary_filenames()
      '''
      for tmp_file, item in six.iteritems(temporary_map):
          node, plug_name = item[1:3]
          if hasattr(node, 'process'):
              process = node.process
          else:
              process = node
          value = getattr(process, plug_name)
          if isinstance(value, list):
              # FIXME TODO: only restore values in list which correspond to
              # a temporary.
              # Problem: they are sometimes transformed into strings
              # FIXME: several temp items can be part of the same list,
              # so this assignment is likely to be done several times.
              # It could probably be optimized.
              # WARNING: we set "" values instead of Undefined because they may
              # be mandatory
              setattr(process, plug_name, [''] * len(value))
          else:
              setattr(process, plug_name, Undefined)

    def _get_swf_paths(engine, environment):
        resource_conf = engine.settings.select_configurations(
            environment, {'somaworkflow': 'config_id=="somaworkflow"'})
        if 'capsul.engine.module.somaworkflow' in resource_conf:
            resource_conf = resource_conf['capsul.engine.module.somaworkflow']
            return (resource_conf.get('transfer_paths', []),
                    resource_conf.get('path_translations', {}))

        # otherwise fallback to old StudyConfig config -- OBSOLETE...
        study_config = engine.study_config
        computing_resource = getattr(
            study_config, 'somaworkflow_computing_resource', None)
        if computing_resource in (None, Undefined):
            return [], {}
        resources_conf = getattr(
            study_config, 'somaworkflow_computing_resources_config', None)
        if resources_conf in (None, Undefined):
        resource_conf = getattr(resources_conf, computing_resource, None)
        if resource_conf in (None, Undefined):
        return (resource_conf.transfer_paths,
                resource_conf.path_translations.export_to_dict())

    def _propagate_transfer(node, param, path, output, transfers,
                            transfer_item):
        todo_plugs = [(node, param, output)]
        done_plugs = set()
        while todo_plugs:
            node, param, output = todo_plugs.pop()
            plug = node.plugs[param]
            is_pipeline = False
            if plug is None or not plug.enabled or not plug.activated \
                    or plug in done_plugs:
                continue
            done_plugs.add(plug)
            if isinstance(node, Switch):
                if output:
                    # propagate to active input
                    other_param = node.switch + '_switch_' + param
                    #plug = node.plugs[input_param]
                else:
                    other_param = param[len(node.switch + '_switch_'):]
                    #other_plug = node.plugs[other_param]
                todo_plugs.append((node, other_param, not output))
            else:
                if hasattr(process, 'nodes'):
                    is_pipeline = True
                    #plug = process.nodes[''].plugs.get(param)
                else:
                    # process: replace its param
                    # check trait type (must be File or Directory, not Any)
                    trait = process.user_traits()[param]
                    #plug = node.plugs[param]
                    if isinstance(trait.trait_type, File) \
                            or isinstance(trait.trait_type, Directory):
                        transfers[bool(trait.output)].setdefault(process, {})[
                            param] = (transfer_item, path)
                        if trait.output:
                            # it's an output of a "real" process: the transfer
                            # will be as output only
                            transfer_item.initial_status \
                                = swclient.constants.FILES_DO_NOT_EXIST
                    #output = not output # invert IO status

            #if plug is None or not plug.enabled or not plug.activated \
                    #or plug in done_plugs:
                #continue
            if output:
            else:
            for proc_name, param_name, node, other_plug, act in links:
                if not node.activated or not node.enabled \
                        or not other_plug.activated or not other_plug.enabled \
                        or other_plug in done_plugs:
                    continue
                todo_plugs.append((node, param_name, not output))
            if is_pipeline:
                # in a pipeline node, go both directions
                if output:
                else:
                for proc_name, param_name, node, plug, act in links:
                    if not node.activated or not node.enabled \
                            or not plug.activated or not plug.enabled \
                            or plug in done_plugs:
                        continue
                    todo_plugs.append((node, param_name, output))

    def _get_transfers(pipeline, transfer_paths, merged_formats):
        """ Create and list FileTransfer objects needed in the pipeline.

        pipeline: Pipeline
            pipeline to build workflow for
        transfer_paths: list
            paths basedirs for translations from soma-worflow config

        Returns
        -------
        [in_transfers, out_transfers]
        each of which is a dict: { Process: proc_dict }
            proc_dict is a dict: { file_path : FileTransfer object }
        FileTransfer objects are reused when referring to the same path
        used from different processes within the pipeline.
        """
        in_transfers = {}
        out_transfers = {}
        transfers = [in_transfers, out_transfers]
        todo_nodes = [pipeline.pipeline_node]
        while todo_nodes:
            node = todo_nodes.pop(0)
            else:
            for param, trait in six.iteritems(process.user_traits()):
                if isinstance(trait.trait_type, File) \
                        or isinstance(trait.trait_type, Directory) \
                        or type(trait.trait_type) is Any:
                    # is value in paths
                    path = getattr(process, param)
                    if path is None or path is Undefined \
                            or isinstance(path, TempFile) or path == '':
                        continue
                    output = bool(trait.output)
                    existing_transfers = transfers[output].get(process, {})
                    existing_transfer = existing_transfers.get(param)
                    if existing_transfer:
                        continue
                    for tpath in transfer_paths:
                        if path.startswith(os.path.join(tpath, '')):
                            transfer_item = swclient.FileTransfer(
                                is_input=not output,
                                client_path=path,
                                client_paths=_files_group(path,
                                                          merged_formats))
                            _propagate_transfer(node, param,
                                                path, not output, transfers,
                                                transfer_item)
                            break
            if hasattr(process, 'nodes'):
                todo_nodes += [sub_node
                               for name, sub_node
                                  in six.iteritems(process.nodes)
                               if name != ''
                                  and not isinstance(sub_node, Switch)]
        return transfers

    def _expand_nodes(nodes):
        '''Expands the nodes list or set to leaf nodes by replacing pipeline
        nodes by their children list.

        Returns
        -------
        set of leaf nodes.
        '''
        nodes_list = list(nodes)
        expanded_nodes = set()
        while nodes_list:
            node = nodes_list.pop(0)
            if not hasattr(node, 'process'):
                continue # switch or something
            if isinstance(node.process, Pipeline):
                nodes_list.extend([p for p
                                   in six.itervalues(node.process.nodes)
                                   if p is not node])
            else:
                expanded_nodes.add(node)
        return expanded_nodes

    def _handle_disable_nodes(pipeline, temp_map, transfers, disabled_nodes):
        '''Take into account disabled nodes by changing FileTransfer outputs
        for such nodes to inputs, and recording output temporary files, so as
        to ensure that missing temporary outputs will not be used later in
        the workflow.

        disabled_nodes should be a list, or preferably a set, of leaf process
        nodes. Use _expand_nodes() if needed before calling
        _handle_disable_nodes().
        '''
        move_to_input = {}
        remove_temp = set()
        for node in disabled_nodes:
                continue # switch or something else
            otrans = transfers[1].get(process, None)
                if trait.output and (isinstance(trait.trait_type, File) \
                        or type(trait.trait_type) is Any):
                    if otrans is not None:
                        transfer, path2 = otrans.get(param, (None, None))
                    else:
                        transfer = None
                    if transfer is not None:
                        print('transfered output path:', path,
                              'from: %s.%s changes to input.'
                              % (node.name, param))
                        move_to_input[path] = transfer
                        transfer.initial_status \
                            = swclient.constants.FILES_ON_CLIENT
                    elif path in temp_map:
                        print('temp path in: %s.%s will not be produced.'
                        remove_temp.add(path)
        return move_to_input, remove_temp

    def iter_to_workflow(process, node_name, step_name, temp_map,
                         shared_map, transfers, shared_paths,
                         disabled_nodes, remove_temp,
                         steps, study_config, iteration, map_job=None,
                         reduce_job=None, environment='global'):
        '''
        Build a workflow for a single iteration step of a process/sub-pipeline

        is called for each iteration by build_iteration()

        Jobs inserted are tuples (process, iteration) because each job will be
        converted into as many jobs as the number of iterations.

        Returns
        -------
        (jobs, dependencies, groups, root_jobs, links)
        '''
            temp_map2 = assign_temporary_filenames(process, len(temp_map))
            temp_subst_list = [(x1, x2[0]) for x1, x2
                                  in six.iteritems(temp_map2)]
            temp_subst_map = dict(temp_subst_list)
            temp_subst_map.update(temp_map)
            try:
                (jobs1, dependencies, groups, sub_root_jobs, plinks, nodes) = \
                    workflow_from_pipeline_structure(
                        process, temp_subst_map, shared_map, transfers,
                        shared_paths, disabled_nodes=disabled_nodes,
                        forbidden_temp=remove_temp, steps=steps,
                        study_config=study_config, environment=environment)
                jobs = {}
                for proc, job in six.iteritems(jobs1):
                    jobs[(proc, iteration)] = job
                groot = []
                for j in list(sub_root_jobs.values()):
                    if isinstance(j, list):
                        groot += j
                    else:
                        groot.append(j)
                links = {}
                for dproc, dplink in six.iteritems(plinks):
                    dlink = {}
                    links[(dproc, iteration)] = dlink
                    for param, linkl in six.iteritems(dplink):
                        dlink[param] = [((link[0], iteration), link[1])
                                        for link in linkl]
                group = build_group(node_name, groot)
                groups[(process, iteration)] = group
                root_jobs = {(process, iteration): group}
            finally:
                restore_empty_filenames(temp_map2)
        elif isinstance(process, ProcessIteration):
            # sub-iteration
            pipeline = Pipeline()
            pipeline.set_study_config(study_config)
            pipeline.add_process('iter', process.process)
            return build_iteration(pipeline, step_name, temp_map=temp_map,
                                   shared_map=shared_map,
                                   transfers=transfers,
                                   shared_paths=shared_paths,
                                   disabled_nodes=disabled_nodes,
                                   remove_temp=remove_temp,
                                   steps=steps,
                                   study_config=study_config,
                                   environment=environment)
        else:
            # single process
            job = build_job(process, temp_map, shared_map,
                            transfers, shared_paths,
                            forbidden_temp=remove_temp,
                            name=node_name,
                            priority=jobs_priority,
                            step_name=step_name,
                            engine=getattr(study_config, 'engine', None),
                            environment=environment)
            jobs = {(process, iteration): job}
            groups = {}
            dependencies = {}
            links = {}
            root_jobs = {(process, iteration): job}
            nodes = []

        return (jobs, dependencies, groups, root_jobs, links, []) # nodes)

    def build_iteration(it_node, step_name, temp_map,
                        shared_map, transfers, shared_paths, disabled_nodes,
                        remove_temp, steps, study_config={},
                        environment='global'):
        '''
        Build workflow for an iterative process: the process / sub-pipeline is
        filled with appropriate parameters for each iteration, and its
        workflow is generated.

        Returns
        -------
        (jobs, dependencies, groups, root_jobs, links, nodes)
        '''
        it_process = it_node.process
            it_process)
        # # check if it is an iterative completion engine
        # if hasattr(completion_engine, 'complete_iteration_step'):
        size = completion_engine.iteration_size()

        #no_output_value = None
        #size = None
        #size_error = False

        ## calculate the number of iterations
        #for parameter in it_process.iterative_parameters:
            #trait = it_process.trait(parameter)
            #psize = len(getattr(it_process, parameter))
            #if psize:
                #if size is None:
                    #size = psize
                #elif size != psize:
                    #size_error = True
                    #break
                #if trait.output:
                    #if no_output_value is None:
                        #no_output_value = False
                    #elif no_output_value:
                        #size_error = True
                        #break
            #else:
                        #no_output_value = True
                    #elif not no_output_value:
                        #break

        #if size_error:
            #raise ValueError('Iterative parameter values must be lists of the '
                #'same size: %s' % ','.join('%s=%d'
                    #% (n, len(getattr(it_process, n)))
                    #for n in it_process.iterative_parameters))

        jobs = {}
        workflows = []
        for parameter in it_process.regular_parameters:
            setattr(it_process.process, parameter,
                    getattr(it_process, parameter))

        jobs = {}
        groups = {}
        root_jobs = {}
        links = {}
        nodes = []

        if size == 0:
            return (jobs, dependencies, groups, root_jobs, links, nodes)

        #if no_output_value:
            ## this case is a "really" dynamic iteration, the number of
            ## iterations and parameters are determined in runtime, so we
            ## cannot handle it at the moment.
            #raise ValueError('Dynamic iteration is not handled in this '
                #'version of CAPSUL / Soma-Workflow')

            #for parameter in it_process.iterative_parameters:
                #trait = it_process.trait(parameter)
                    #setattr(it_process, parameter, [])
            #outputs = {}
            #for iteration in range(size):
                #for parameter in it_process.iterative_parameters:
                    #if not no_output_value \
                            #or not it_process.trait(parameter).output:
                        #values = getattr(it_process, parameter)
                        #if len(values) != 0:
                            #if len(values) > iteration:
                                #setattr(it_process.process, parameter,
                                        #values[iteration])
                            #else:
                                        #values[-1])

                ## operate completion
                #complete_iteration(it_process, iteration)

                ##workflow = workflow_from_pipeline(it_process.process,
                                                  ##study_config=study_config)
                ##workflows.append(workflow)

                    #trait = it_process.trait(parameter)
                    #if trait.output:
                        #outputs.setdefault(
                            #parameter,[]).append(getattr(it_process.process,
                                                         #parameter))
            #for parameter, value in six.iteritems(outputs):
                #setattr(it_process, parameter, value)
        #else:

        if True:

            # iterations are built using
            # * a map job to dispatch input lists
            # * the iterated process or pipeline jobs, duplicated for each
            #   iteration
            # * a reduce job to gather outputs into lists
            # dependencies and parameters links have to be built.
            # links to processes outside the iteration are made, in order to
            # connect the iteration node to its neighbors.

            # collect iterated inputs / outputs
            map_param_dict = {}
            forbidden_traits = ('nodes_activation', 'selection_changed',
                                'pipeline_steps', 'visible_groups',
                                'protected_parameters', )
            # copy non-iterative inputs
            for param, trait in six.iteritems(it_process.user_traits()):
                if param in forbidden_traits:
                    continue  # skip
                value = getattr(it_process, param)
                if not trait.output:
                    map_param_dict[param] = value

            in_params = [p for p in it_process.iterative_parameters
                         if not it_process.trait(p).output]
            out_params = [p for p in it_process.iterative_parameters
                          if it_process.trait(p).output]
            in_values = [getattr(it_process, param) for param in in_params]
            out_values = [getattr(it_process, param) for param in out_params]

            # build map and reduce nodes
            map_param_dict.update({
                'input_names': in_params,
                'output_names': ['%s' % p + '_%d' for p in in_params],
            })
            reduce_param_dict = {}
                if trait.output and param not in forbidden_traits:
                    value = getattr(it_process, param)
                    reduce_param_dict[param] = value
                    if param in out_params:
                        # set input params of reduce node for a non-dynamic
                        # reduce case (outputs are set from outside)
                        for i, p in enumerate(value):
                            reduce_param_dict['%s_%d' % (param, i)] = p

            reduce_param_dict.update({
                'input_names': ['%s' % p + '_%d' for p in out_params],
                'output_names': out_params,
                'lengths': [size] * len(out_params),
            })

            # replace special paths in map/reduce parameters

            _replace_in_list(in_values, temp_map)
            in_values = _get_replaced(in_values, shared_map)
            _replace_in_list(out_values, temp_map)
            out_values = _get_replaced(out_values, shared_map)
            _replace_in_dict(map_param_dict, temp_map)
            _replace_in_dict(map_param_dict, shared_map)
            _replace_dict_transfers(
                map_param_dict, it_process, transfers[0], transfers[1])
            _replace_in_dict(reduce_param_dict, temp_map)
            _replace_in_dict(reduce_param_dict, shared_map)
                reduce_param_dict, it_process, transfers[0], transfers[1])

            map_job = MapJob(
                referenced_input_files=in_values,
                referenced_output_files=in_values,
                name=it_process.process.name + '_map',
                param_dict=map_param_dict)
            reduce_job = ReduceJob(
                referenced_input_files=out_values,
                referenced_output_files=out_values,
                name=it_process.process.name + '_reduce',
                param_dict=reduce_param_dict)
            map_job.process_hash = id(it_process)
            reduce_job.process_hash = id(it_process)

            # connect inputs of the map node, outputs to reduce node,
            # and record connections to iterated jobs
            map_links = {}
            map_iter_links = {}
            reduce_iter_links = {}
            red_iter_links = {}
            for param, plug in six.iteritems(it_node.plugs):
                    continue
                    # connect inputs of the map node
                    sources = pipeline_tools.find_plug_connection_sources(
                        it_node.plugs[param], it_node)
                    for pnode, pparam, pparent in sources:
                        if pnode.enabled and pnode.activated:
                            pproc = pnode
                            if hasattr(pnode, 'process'):
                                pproc = pnode.process
                            map_links.setdefault(param, []).append((pproc,
                                                                    pparam))
                    # record dest of links in iterated nodes
                    if isinstance(it_process.process, Pipeline):
                        dest = \
                            pipeline_tools.find_plug_connection_destinations(
                                it_process.process.pipeline_node.plugs[param],
                                it_process.process.pipeline_node)
                        for pnode, pparam, pparent in dest:
                            if pipeline_tools.is_node_enabled(
                                    it_process.process, node=pnode):
                                pproc = pnode
                                if hasattr(pnode, 'process'):
                                    pproc = pnode.process
                                map_iter_links.setdefault(pproc, {}) \
                                    .setdefault(pparam, []).append(
                                        (map_job, param))
                    else:
                        map_iter_links.setdefault(it_process.process, {}) \
                            .setdefault(param, []).append(
                                (map_job, param))
                else:
                    # connect outputs of the reduce node
                    dest = pipeline_tools.find_plug_connection_destinations(
                    for pnode, pparam, pparent in dest:
                            links.setdefault(pproc, {}).setdefault(pparam, []) \
                                .append((reduce_job, param))
                    # record source of links in iterated nodes
                        #print('reduce from pipeline', param)
                        sources = \
                            pipeline_tools.find_plug_connection_sources(
                        #print('sources:', sources)
                        for pnode, pparam, pparent in sources:
                                red_iter_links.setdefault(param, []).append(
                                    (pproc, pparam))
                    else:
                        red_iter_links.setdefault(param, []).append(
                            (it_process.process, param))
            links[map_job] = map_links
            reduce_iter_links[reduce_job] = red_iter_links
            jobs[map_job] = map_job
            jobs[reduce_job] = reduce_job
            root_jobs[map_job] = map_job
            root_jobs[reduce_job] = reduce_job

            # iterate the iterates process / pipeline

            iter_values = {}
            for iteration in range(size):
                for parameter in it_process.iterative_parameters:
                    if it_process.process.trait(parameter).input_filename \
                            is False:
                        # dynamic output has no forced value
                        continue
                    values = getattr(it_process, parameter)
                    if len(values) != 0:
                        if len(values) > iteration:
                            setattr(it_process.process, parameter,
                                    values[iteration])
                        else:
                            setattr(it_process.process, parameter, values[-1])

                # operate completion
                complete_iteration(it_process, iteration)

                # get iteration values to set on the parent iter node
                            is False:
                        continue
                    value = getattr(it_process.process, parameter)
                    iter_values.setdefault(parameter, []).append(value)

                # build a workflow for the job / pipeline iteration
                process_name = it_process.process.name + '_%d' % iteration
                (sub_jobs, sub_dependencies, sub_groups, sub_root_jobs,
                 sub_links, sub_nodes) = \
                    iter_to_workflow(it_process.process, process_name,
                        step_name,
                        temp_map, shared_map, transfers,
                        shared_paths, disabled_nodes, remove_temp, steps,
                        study_config, iteration, map_job=map_job,
                        reduce_job=reduce_job, environment=environment)
                jobs.update(sub_jobs)
                dependencies.update(sub_dependencies)
                groups.update(sub_groups)
                root_jobs.update(sub_root_jobs)
                links.update(sub_links)

                # connect map / reduce nodes to iterated jobs
                for proc, dlink in six.iteritems(map_iter_links):
                    slink = links.setdefault((proc, iteration), {})
                    for dparam, linkl in six.iteritems(dlink):
                        l = slink.setdefault(dparam, [])
                        for link in linkl:
                            if link[1] in in_params:
                                # iterative param
                                l.append((link[0],
                                          '%s_%d' % (link[1], iteration)))
                            else:
                                l.append(link)
                for proc, dlink in six.iteritems(reduce_iter_links):
                        if dparam in out_params:
                            # iterative param
                            dparam = '%s_%d' % (dparam, iteration)
                            links.setdefault(proc, {}) \
                                .setdefault(dparam, []) \
                                .append(((link[0], iteration), link[1]))

        # set values on the iter node (what a full completion does, but we're
        # doing it only partially here)
        for param, value in iter_values.items():
            setattr(it_process, param, value)

        # the iteration process is not a single job, but can be reached
        # (for links) through the map and reduce nodes. So we record a tuple
        # (map, reduce) here for this special node.
        jobs[it_process] = (map_job, reduce_job)  # special job(s)
        return (jobs, dependencies, groups, root_jobs, links, nodes)

    def complete_iteration(it_process, iteration):
        # check if it is an iterative completion engine
        if hasattr(completion_engine, 'complete_iteration_step'):
            completion_engine.complete_iteration_step(iteration)

    def workflow_from_pipeline_structure(
            pipeline, temp_map={}, shared_map={},
            transfers=[{}, {}], shared_paths={},
            disabled_nodes=set(), forbidden_temp=set(),
            jobs_priority=0, steps={}, current_step='',
            study_config={}, with_links=True, environment='global'):
        """ Convert a CAPSUL pipeline into a soma-workflow workflow

            a CAPSUL pipeline
            temporary files to replace by soma_workflow TemporaryPath objects
            shared translated paths maps (global to pipeline).
        transfers: list of 2 dicts (optional)
            File transfers dicts (input / output), indexed by process, then by
            file path.
            holds information about shared resource paths from soma-worflow
            section in study config.
        jobs_priority: int (optional, default: 0)
            set this priority on soma-workflow jobs.
        steps: dict (optional)
            node name -> step name dict
        current_step: str (optional)
            the parent node step name
        study_config: StydyConfig instance (optional)
            used only for iterative nodes, to be passed to create sub-workflows
        with_links: bool (optional)
            follow links to include other dependencies

        Returns
        -------
        workflow: tuple (jobs, dependencies, groups, root_jobs, links, nodes)
            the corresponding soma-workflow workflow definition (to be passed
            to Workflow constructor)
        """
        def _update_links(links1, links2):
            for dest_node, slink in links2.items():
                nlink = links1.setdefault(dest_node, {})
                nlink.update(slink)

        jobs = {}
        groups = {}
        group_nodes = {}
        links = {}

        nodes = [(pipeline, node_name, node)
                 for node_name, node in pipeline.nodes.items()
                 if node.activated and node.enabled
                    and node is not pipeline.pipeline_node]
        all_nodes = []
        engine = None
            engine = getattr(study_config, 'engine', None)

        # Go through all graph nodes
        for node_desc in nodes:

            n_pipeline, node_name, node = node_desc

            if node in disabled_nodes \
                    or not pipeline_tools.is_node_enabled(n_pipeline,
                                                          node_name=node_name):
                continue

            if not node.is_job():
                continue

            all_nodes.append(node_desc)

                # sub-pipeline
                group_nodes[node_name] = node

                step_name = current_step or steps.get(node_name, '')
                (sub_jobs, sub_deps, sub_groups, sub_root_jobs, sub_links,
                 sub_nodes) \
                    = workflow_from_pipeline_structure(
                        node.process, temp_map, shared_map, transfers,
                        shared_paths, disabled_nodes,
                        jobs_priority=jobs_priority,
                        steps=steps, current_step=step_name, with_links=False,
                        environment=environment)
                group = build_group(node_name,
                                    sum(list(sub_root_jobs.values()), []))
                groups[node] = group
                root_jobs[node] = [group]
                dependencies.update(sub_deps)
                all_nodes += sub_nodes
                #_update_links(links, sub_links)
            # Otherwise convert all the processes to jobs
            else:
                sub_jobs = {}
                if isinstance(node, Process):
                    process = node
                elif hasattr(node, 'process'):
                    # process node
                    process = node.process
                else:
                    process = node  # custom node
                step_name = current_step or steps.get(node.name)
                if isinstance(process, ProcessIteration):
                    # iterative node
                    group_nodes.setdefault(
                        node_name, []).append(node)
                    sub_workflows = build_iteration(
                        node, step_name, temp_map,
                        shared_map, transfers, shared_paths,
                        disabled_nodes,
                        {}, steps, study_config={})
                    (sub_jobs, sub_deps, sub_groups, sub_root_jobs,
                      sub_links, sub_nodes) = sub_workflows
                    group = build_group(node_name, list(sub_root_jobs.values()))
                    groups.setdefault(process, []).append(group)
                    root_jobs.setdefault(process, []).append(group)
                    groups.update(sub_groups)
                    jobs.update(sub_jobs)
                    dependencies.update(sub_deps)
                    all_nodes += sub_nodes
                    _update_links(links, sub_links)
                else:
                    job = build_job(process, temp_map, shared_map,
                                    transfers, shared_paths,
                                    forbidden_temp=forbidden_temp,
                                    name=node_name,
                                    priority=jobs_priority,
                                    step_name=step_name,
                                    engine=engine, environment=environment)
                    if job:
                        sub_jobs[process] = job
                        root_jobs[process] = [job]
                        jobs.update(sub_jobs)

        # links / dependencies
        if with_links:
            for node_desc in all_nodes:
                sub_pipeline, node_name, node = node_desc
                dproc = getattr(node, 'process', node)
                if isinstance(dproc, Pipeline):
                    continue  # pipeline nodes are virtual
                for param, plug in six.iteritems(node.plugs):
                        plug, True)
                    for source in sources:
                        snode, param_name, parent = source
                        if node in disabled_nodes \
                                or not pipeline_tools.is_node_enabled(
                                    pipeline, node=snode):
                            continue
                        process = getattr(snode, 'process', snode)
                        if not isinstance(snode, ProcessNode) \
                                and snode not in jobs:
                            # the node is a phantom node (switch). Add deps to
                            # all upstream nodes
                            new_nodes = [snode]
                            while new_nodes:
                                mnode = new_nodes.pop(0)
                                moredep = [
                                    pipeline_tools.find_plug_connection_sources
                                    (mlink[3])
                                    for mplug in mnode.plugs.values()
                                    for mlink in mplug.links_from
                                    if not mplug.output
                                ]
                                dependencies.update(
                                    [(x[0].process, node.process)
                                     for x in moredep
                                     if isinstance(x[0], ProcessNode)])
                                new_nodes += [x[0] for x in moredep
                                              if x[0] is not None
                                                  and not isinstance(
                                                      x[0], ProcessNode)]
                        elif isinstance(snode, PipelineNode):
                            # either link from main input, or from an
                            # unconnected optional input in a sub-pipeline:
                            # no dependency link
                            continue
                        else:  # ProcessNode
                            sjob = jobs[process]
                            if isinstance(sjob, tuple):  # iteration
                                sjob = sjob[1]  # source
                            djob = jobs[dproc]
                            if isinstance(djob, tuple):  # iteration
                                djob = djob[0]  # destination
                            dependencies.add((sjob, djob))
                            trait = process.trait(param_name)
                            if trait.input_filename is False \
                                    or (not isinstance(trait.trait_type,
                                                      (File, Directory)) \
                                        and (not isinstance(trait.trait_type,
                                                            List)
                                            or not isinstance(
                                                trait.inner_traits[0],
                                                (File, Directory)))):
                                links.setdefault(dproc, {}).setdefault(
                                    param, []).append((process, param_name))

        return jobs, dependencies, groups, root_jobs, links, all_nodes

    def _create_directories_job(pipeline, shared_map={}, shared_paths={},
                                priority=0, transfer_paths=[]):
        def _is_transfer(d, transfer_paths):
            for path in transfer_paths:
                if d.startswith(os.path.join(path, '')):

        directories = [d
                       for d in pipeline_tools.get_output_directories(
                          pipeline)[1]
                       if not _is_transfer(d, transfer_paths)]
        if len(directories) == 0:
            return None # no dirs to create.
        paths = []
        # check for path translations
        for path in directories:
            new_path = _translated_path(path, shared_map, shared_paths)
            paths.append(new_path or path)
        # use a python command to avoid the shell command mkdir
        cmdline = ['python', '-c',
                   'import sys, os; [os.makedirs(p) if not os.path.exists(p) '
                   'else None '
                   'for p in sys.argv[1:]]'] \
                  + paths

        job = swclient.Job(
            name='output directories creation',
            command=cmdline,
            priority=priority)
        return job

    # TODO: handle formats in a separate, centralized place
    # formats: {name: ext_props}
    #     ext_props: {ext: [dependent_exts]}
    #     dependent_exts: (ext, mandatory)
    formats = {
        'NIFTI-1': {'.nii': [], '.img': [('.hdr', True)], '.nii.gz': []},
        'GIS': {'.ima': [('.dim', True)]},
        'GIFTI': {'.gii': []},
        'MESH': {'.mesh': []},
        'ARG': {'.arg': [('.data', False)]},
    }
    # transform it to an ext-based dict
    # merged_formats: {ext: [dependent_exts]}
    # (formats names are lost here)
    merged_formats = {}
    for format, values in six.iteritems(formats):
        merged_formats.update(values)

    if study_config is None:
        study_config = pipeline.get_study_config()
    engine = study_config.engine

    if check_requirements:
        ml = []
        if pipeline.check_requirements(environment, message_list=ml) is None:
                'The pipeline requirements are not met in the current '
                'configuration settings: %s' % str(ml))

    temp_pipeline = False
        new_pipeline.set_study_config(study_config)
        # "pipeline" is actually a single process, the name is ClassName_1
        new_pipeline.add_process(pipeline.name.lower() + "_1", pipeline)
        temp_pipeline = True

    if complete_parameters:
        completion = ProcessCompletionEngine.get_completion_engine(pipeline)
        if completion:
            attributes = completion.get_attribute_values()
            completion.complete_parameters(complete_iterations=False)

    temp_map = assign_temporary_filenames(pipeline)
    temp_subst_list = [(x1, x2[0]) for x1, x2 in six.iteritems(temp_map)]
    temp_subst_map = dict(temp_subst_list)
    shared_map = {}

    swf_paths = _get_swf_paths(engine, environment)
    transfers = _get_transfers(pipeline, swf_paths[0], merged_formats)
    # get complete list of disabled leaf nodes
    if disabled_nodes is None:
        disabled_nodes = pipeline.disabled_pipeline_steps_nodes()
    disabled_nodes = disabled_nodes \
        + [name for name, node in six.iteritems(pipeline.nodes)
           if node.node_type != 'processing_node'
              and name not in disabled_nodes]
    disabled_nodes = _expand_nodes(disabled_nodes)
    move_to_input, remove_temp = _handle_disable_nodes(
        pipeline, temp_subst_map, transfers, disabled_nodes)
    #print('changed transfers:', move_to_input)
    #print('removed temp:', remove_temp)
    #print('temp_map:', temp_map, '\n')
    #print('SWF transfers:', swf_paths[0])
    #print('shared paths:', swf_paths[1])

    if create_directories:
        # create job
        dirs_job = _create_directories_job(
            pipeline, shared_map=shared_map, shared_paths=swf_paths[1],
            transfer_paths=swf_paths[0])

    # build steps map
    steps = {}
    if hasattr(pipeline, 'pipeline_steps'):
        for step_name, step \
                in six.iteritems(pipeline.pipeline_steps.user_traits()):
            nodes = step.nodes
            steps.update(dict([(node, step_name) for node in nodes]))

    # actually build the workflow, recursively if needed.
    try:
        (jobs, dependencies, groups, root_jobs, links, nodes) \
            = workflow_from_pipeline_structure(
                pipeline, temp_subst_map, shared_map, transfers,
                swf_paths[1],
                disabled_nodes=disabled_nodes, forbidden_temp=remove_temp,
                steps=steps, study_config=study_config,
                environment=environment)
    finally:
        restore_empty_filenames(temp_map)

    # post-process links to replace nodes with jobs
    param_links = {}
    # expand jobs map because jobs keys may be tuples (for iterations)
    jobs_map = {}
    for process, job in six.iteritems(jobs):
        while isinstance(process, tuple):
            process = process[0]
        jobs_map.setdefault(process, []).append(job)
    for dnode, dlinks in six.iteritems(links):
        if isinstance(dnode, (Pipeline, ProcessIteration)) \
                or (isinstance(dnode, tuple)
                    and isinstance(dnode[0], (Pipeline, ProcessIteration))):
            continue  # FIXME handle this
        djlinks = {}
        for param, linkl in six.iteritems(dlinks):
            for link in linkl:
                if link[0] is not pipeline \
                        and not isinstance(link[0],
                                          (Pipeline, ProcessIteration)):
                    # FIXME handle ProcessIteration cases
                    if isinstance(link[0], tuple):
                        if not isinstance(link[0][0],
                            djlinks.setdefault(param, []) \
                                .append((jobs[link[0]], link[1]))
                    else:
                        for job in jobs_map[link[0]]:
                                .append((job, link[1]))
        if isinstance(dnode, tuple):
            param_links[jobs[dnode]] = djlinks
        else:
            for job in jobs_map[dnode]:
                param_links[job] = djlinks

    all_jobs = [job for job in list(jobs.values()) if not isinstance(job, tuple)]
    root_jobs = sum(list(root_jobs.values()), [])

    # if directories have to be created, all other primary jobs will depend
    # on this first one
    if create_directories and dirs_job is not None:
        dependend_jobs = set()
        for dependency in dependencies:
            dependend_jobs.add(dependency[1])
        new_deps = [(dirs_job, job) for job in all_jobs
                    if job not in dependend_jobs]
        dependencies.update(new_deps)
        all_jobs.insert(0, dirs_job)
        root_jobs.insert(0, dirs_job)

    workflow = swclient.Workflow(jobs=all_jobs,
        dependencies=dependencies,
        root_group=root_jobs,
        name=pipeline.name,
        param_links=param_links)

    # mark workflow with pipeline
    workflow.pipeline = weakref.ref(pipeline)
    if temp_pipeline:
        # the pipeline is temporary - it will be deleted if we don't save a ref
        # to it somewhere else in the workflow
        workflow._pipeline = pipeline
    workflow._do_not_pickle = ['pipeline', '_pipeline']
    if hasattr(pipeline, 'uuid'):
        workflow.uuid = pipeline.uuid

    return workflow

def workflow_run(workflow_name, workflow, study_config):
    """ Create a soma-workflow controller and submit a workflow

    workflow_name: str (mandatory)
        the name of the workflow
    workflow: Workflow (mandatory)
        the soma-workflow workflow
    study_config: StudyConfig (mandatory)
        contains needed configuration through the SomaWorkflowConfig module
    """
    swm = study_config.modules['SomaWorkflowConfig']
    swm.connect_resource()
    queue = None
    if hasattr(study_config.somaworkflow_computing_resources_config,
               resource_id):
        res_conf = getattr(
            study_config.somaworkflow_computing_resources_config, resource_id)
        queue = res_conf.queue
        if queue is Undefined:
            queue = None
    swclient.Helper.wait_workflow(wf_id, controller)

    pipeline = getattr(workflow, 'pipeline', None)
        pipeline = pipeline()  # dereference the weakref
            else:

                        continue
                    process.import_from_dict(out_params)

    return controller, wf_id

'''
Utility class for iterated nodes in a pipeline. This is mainly internal infrastructure, which a normal programmer should not have to bother about.
A pipeline programmer will not instantiate :class:`ProcessIteration` directly, but rather use the :class:`~capsul.pipeline.pipeline.Pipeline` method :meth:`~capsul.pipeline.pipeline.Pipeline.add_iterative_process`.

Classes
=======
:class:`ProcessIteration`
'''

import sys
import six
from traits.api import List, Undefined

from capsul.study_config.process_instance import get_process_instance
import capsul.study_config as study_cmod
from traits.api import File, Directory

class ProcessIteration(Process):

    _doc_path = 'api/pipeline.html#processiteration'

    def __init__(self, process, iterative_parameters, study_config=None,
                 context_name=None):
        super(ProcessIteration, self).__init__()

        if self.study_config is None and hasattr(Process, '_study_config'):
            study_config = study_cmod.default_study_config()
        if study_config is not None:
            self.study_config = study_config

        self.process = get_process_instance(process,
                                            study_config=study_config)

        if context_name is not None:
            self.process.context_name = context_name
        self.regular_parameters = set()
        self.iterative_parameters = set(iterative_parameters)

        # use the completion system (if any) to get induced (additional)
        # iterated parameters
            # don't import this at module level to avoid cyclic imports
            from capsul.attributes.completion_engine \
                import ProcessCompletionEngine

            completion_engine \
                = ProcessCompletionEngine.get_completion_engine(self)
            if hasattr(completion_engine, 'get_induced_iterative_parameters'):
                induced_iterative_parameters \
                    = completion_engine.get_induced_iterative_parameters()
                self.iterative_parameters.update(induced_iterative_parameters)
                iterative_parameters = self.iterative_parameters

        # Check that all iterative parameters are valid process parameters
        user_traits = self.process.user_traits()
        has_output = False
        inputs = []
        for parameter in self.iterative_parameters:
            if parameter not in user_traits:
                raise ValueError('Cannot iterate on parameter %s '
                  'that is not a parameter of process %s'
                  % (parameter, self.process.id))
            if user_traits[parameter].output:
                has_output = True
            else:
                inputs.append(parameter)

        # Create iterative process parameters by copying process parameter
        # and changing iterative parameters to list
        for name, trait in six.iteritems(user_traits):
            if name in iterative_parameters:
                kw = {}
                    kw['input_filename'] = False
                self.add_trait(name, List(trait, output=trait.output,
                                          optional=trait.optional, **kw))
                if trait.groups:
                    self.trait(name).groups = trait.groups
                if trait.forbid_completion is not None:
                    # we don't have access to the pipeline or even the
                    # node in self, we cannot propagate the forbid_completion
                    # value outside of self.
                    # However this will be done in Pipeline.add_process() when
                    # inserting self in a pipeline, so this is OK.
                    trait(name).forbid_completion \
                        = trait.forbid_completion
            else:
                self.regular_parameters.add(name)
                self.add_trait(name, trait)
                # copy initial value of the underlying process to self
                # Note: should be this be done via a links system ?
                setattr(self, name, getattr(self.process, name))

        # if the process has iterative outputs, the output lists have to be
        # resized according to inputs
        if has_output:
            self.on_trait_change(self._resize_outputs, inputs)

    def _resize_outputs(self):
        num = 0
        for param in self.iterative_parameters:
            if self.process.trait(param).output:
                if isinstance(self.process.trait(param).trait_type,
                              (File, Directory)):
                    outputs.append(param)
            else:
                num = max(num, len(getattr(self, param)))
        for param in outputs:
            value = getattr(self, param)
            mod = False
            if len(value) > num:
                new_value = value[:num]
                mod = True
            else:
                if len(value) < num:
                    new_value = value \
                        + [self.process.trait(param).default] \
                            * (num - len(value))
                    mod = True
            if mod:
                setattr(self, param, new_value)

    def change_iterative_plug(self, parameter, iterative=None):
        '''
        Change a parameter to be iterative (or non-iterative)

            parameter name
        iterative: bool or None
            if None, the iterative state will be toggled. If True or False, the
            parameter state will be set accordingly.
        '''
        if parameter not in self.process.user_traits():
            raise ValueError('Cannot iterate on parameter %s '
              'that is not a parameter of process %s'
              % (parameter, self.process.id))

        is_iterative = parameter in self.iterative_parameters
        if is_iterative == iterative:
            return # nothing to be done
        if iterative is None:
            iterative = not is_iterative

        trait = self.process.trait(parameter)
        if iterative:

            # switch to iterative
            self.regular_parameters.remove(parameter)
            self.iterative_parameters.add(parameter)
            self.remove_trait(parameter)
            # Create iterative process parameter by copying process parameter
            # and changing iterative parameter to list
            self.add_trait(parameter, List(trait, output=trait.output,
                                      optional=trait.optional))
            if trait.groups:
                self.trait(parameter).groups = trait.groups

            # if it is an output, the output list has to be
            # resized according to inputs
                inputs = []
                for param in self.iterative_parameters:
                    if not self.process.trait(param).output:
                        inputs.append(param)
                self.on_trait_change(self._resize_outputs, inputs)

        else:

            # switch to non-iterative
            self.iterative_parameters.remove(parameter)
            self.regular_parameters.add(parameter)
            self.add_trait(parameter, trait)
            # copy initial value of the underlying process to self
            setattr(self, parameter, getattr(self.process, parameter))

        # Check that all iterative parameter value have the same size
        no_output_value = None
        size = None
        size_error = False
        iterative_parameters = self.iterative_parameters
        if hasattr(self.process, 'pipeline_node'):
            iterative_parameters = [
                p for p in iterative_parameters
                if self.process.pipeline_node.plugs[p].activated]

        for parameter in iterative_parameters:
            trait = self.trait(parameter)
            value = getattr(self, parameter)
            psize = len(value)
            if psize and (not trait.output
                          or len([x for x in value
                                  if x in ('', Undefined, None)]) == 0):
                if size is None:
                    size = psize
                elif size != psize:
                    size_error = f'{parameter} (size {psize} != {size})'
                    break
                    if no_output_value is None:
                        no_output_value = False
                    elif no_output_value:
                        size_error = f'{parameter} (size = {psize} but other parameter has no value)'
                        break
            else:
                        no_output_value = True
                    elif not no_output_value:
                        size_error = f'{parameter} (size = {psize}, has no value)'
                        break
                else:
                    if size is None:
                        size = psize
                    elif size != psize:
                        size_error = f'{parameter} (size {psize} != {size})'
                        break

        if size_error:
            raise ValueError(f'Iterative parameter {size_error} values must be lists of the same size: %s' % ', '.join('%s=%d' % (n, len(getattr(self,n))) for n in iterative_parameters))
            return

        for parameter in self.regular_parameters:
            setattr(self.process, parameter, getattr(self, parameter))
        if no_output_value:
            for parameter in iterative_parameters:
                trait = self.trait(parameter)
                    setattr(self, parameter, [])
            outputs = {}
                for parameter in iterative_parameters:
                    #if not no_output_value or not self.trait(parameter).output:
                    value = getattr(self, parameter)
                    if len(value) > iteration:
                        setattr(self.process, parameter,
                                getattr(self, parameter)[iteration])
                self.complete_iteration(iteration)
                self.process()
                    trait = self.trait(parameter)
                        outputs.setdefault(parameter,[]).append(
                            getattr(self.process, parameter))
                        # reset empty value
                        setattr(self.process, parameter, Undefined)
            for parameter, value in six.iteritems(outputs):
                setattr(self, parameter, value)
        else:
                    setattr(self.process, parameter,
                            getattr(self, parameter)[iteration])

        super(ProcessIteration, self).set_study_config(study_config)
        self.process.set_study_config(study_config)

    def complete_iteration(self, iteration):
        # don't import this at module level to avoid cyclic imports
        from capsul.attributes.completion_engine import ProcessCompletionEngine

            self)

'''
Pipeline exportation function as a python source code file.

Functions
=========
:func:`save_py_pipeline`
'''

import os
import six
import sys

def save_py_pipeline(pipeline, py_file):
    '''
    Save a pipeline in an Python source file

        pipeline to save
    py_file: str or file-like object
        .py file to save the pipeline in
    '''
    # imports are done locally to avoid circular imports
    from capsul.api import Process, Pipeline
    from capsul.pipeline.pipeline_nodes import ProcessNode, Switch, \
        OptionalOutputSwitch, PipelineNode
    from capsul.pipeline.process_iteration import ProcessIteration
    from capsul.process.process import NipypeProcess
    from capsul.study_config.process_instance import get_process_instance
    from traits.api import Undefined

    def get_repr_value(value):
        # TODO: handle None/Undefined in lists/dicts etc
        if value is Undefined:
            repvalue = 'traits.Undefined'
        elif value is None:
            repvalue = 'None'
        elif isinstance(value, Controller):
            repvalue = repr(dict(value.export_to_dict()))
        else:
            repvalue = repr(value)
        return repvalue

    def _write_process(node, pyf, name, enabled, skip_invalid):
        if isinstance(process, NipypeProcess):
            mod = process._nipype_interface.__module__
            classname = process._nipype_interface.__class__.__name__
        else:
            mod = process.__module__
            # if process is a function with XML decorator, we need to
            # retrieve the original function name.
            func = getattr(process, '_function', None)
            if func:
                classname = func.__name__
            else:
                classname = process.__class__.__name__
        procname = '.'.join((mod, classname))
        proc_copy = get_process_instance(procname)
        make_opt = []
        for tname, trait in six.iteritems(proc_copy.user_traits()):
            ntrait = process.trait(tname)
            if ntrait.optional and not trait.optional:
                make_opt.append(tname)
        node_options = ''
        if len(make_opt) != 0:
            node_options += ', make_optional=%s' % repr(make_opt)
            node_options += ', skip_invalid=True'
        print('        self.add_process("%s", "%s"%s)' % (name, procname,
                                                          node_options),
              file=pyf)

        # check that sub-nodes enable and plugs optional states are the
        # expected ones
        todo = [('self.nodes["%s"]' % name, node,
                 ProcessNode(node.pipeline, node.name, proc_copy))]
            self_str, snode, cnode = todo.pop(0)
            if not snode.enabled:
                print('        %s.enabled = False' % self_str,
                      file=pyf)

            # if the node is a (sub)pipeline, and this pipeline has additional
            # exported traits compared to the its base module/class instance
            # (proc_copy),  then we must use explicit exports/links inside it
            if isinstance(snode, PipelineNode):
                sproc = snode.process
                cproc = cnode.process
                for param_name, trait in sproc.user_traits().items():
                    optional = None
                    if param_name not in cproc.traits():
                        # param added, not in the original process
                        is_input = not trait.output
                        if (is_input and sproc.pipeline_node.plugs[
                                    param_name].links_to) \
                                or (not is_input
                                    and sproc.pipeline_node.plugs[
                                        param_name].links_from):
                            if is_input:
                                for link in sproc.pipeline_node.plugs[
                                        param_name].links_to:
                                    print(f'        {self_str}.process.add_link("{param_name}->{link[0]}.{link[1]}", allow_export=True)\n',
                                          file=pyf)
                            else:
                                        param_name].links_from:
                                    print(f'        {self_str}.process.add_link("{link[0]}.{link[1]}->{param_name}", allow_export=True)\n',
                                          file=pyf)

            for param_name, plug in snode.plugs.items():
                trait = snode.get_trait(param_name)
                ctrait = cnode.get_trait(param_name)
                optional = None
                if param_name not in cnode.plugs \
                        or trait.optional != ctrait.optional:
                    optional = trait.optional
                if optional is not None:
                    if hasattr(cnode, 'process'):
                        print(f'        {self_str}.process.trait("{param_name}").optional = {optional}\n',
                              file=pyf)
                    print(f'        {self_str}.plugs["{param_name}"].optional = {optional}\n',
                          file=pyf)
                value = snode.get_plug_value(param_name)
                if value != cnode.get_plug_value(param_name):
                    #splug = snode.plugs[param_name]
                    #if splug.output and len(splug.links_to) == 0 \
                            #or not splug.output and len(splug.links_from) == 0:
                        ## unconnected with non-default value
                    print(
                        f'        {self_str}.set_plug_value("{param_name}", %s)'
                        % get_repr_value(value), file=pyf)

                sself_str = '%s.process.nodes["%s"]' % (self_str, '%s')
                for node_name, snode in snode.process.nodes.items():
                    scnode = cnode.process.nodes[node_name]

                    if node_name == '':
                        continue
                    todo.append((sself_str % node_name, snode, scnode))

        for pname in process.user_traits():
            value = getattr(process, pname)
            init_value = getattr(proc_copy, pname, Undefined)
            if value != init_value \
                    and not (value is Undefined and init_value == ''):
                repvalue = get_repr_value(value)
                print('        self.nodes["%s"].process.%s = %s'
                      % (name, pname, repvalue), file=pyf)
        #if isinstance(process, NipypeProcess):
            ## WARNING: not sure I'm doing the right things for nipype. To be
            ## fixed if needed.
            #for param in process.inputs_to_copy:
                #elem = ET.SubElement(procnode, 'nipype')
                #elem.set('name', param)
                #if param in process.inputs_to_clean:
                    #elem.set('copyfile', 'discard')
                #else:
                    #elem.set('copyfile', 'true')
                #np_input = getattr(process._nipype_interface.inputs, param)
                #if np_input:
                    #use_default = getattr(np_input, 'usedefault', False) # is it that?
                    #if use_default:
                        #elem.set('use_default', 'true')
            #for param, np_input in \
                    #six.iteritems(process._nipype_interface.inputs.__dict__):
                #use_default = getattr(np_input, 'usedefault', False) # is it that?
                #if use_default and param not in process.inputs_to_copy:
                    #elem = ET.SubElement(procnode, 'nipype')
                    #elem.set('name', param)
                    #elem.set('use_default', 'true')

    def _write_custom_node(node, pyf, name, enabled):
        mod = node.__module__
        classname = node.__class__.__name__
        nodename = '.'.join((mod, classname))
        if hasattr(node, 'configured_controller'):
            c = node.configured_controller()
            params = dict((p, v) for p, v in six.iteritems(c.export_to_dict())
                          if v not in (None, traits.Undefined))
            print(
                '        self.add_custom_node("%s", "%s", %s)'
                % (name, nodename, get_repr_value(params)), file=pyf)
        else:
            print('        self.add_custom_node("%s", "%s")'
                  % (name, nodename), file=pyf)
        # optional plugs
            if plug.optional:
                print('        self.nodes["%s"].plugs["%s"].optional = True'
                      % (name, plug_name), file=pyf)
        # non-default: values of unconnected plugs
            if len(plug.links_from) == 0 and len(plug.links_to) == 0 \
                    and plug_name in node.traits() \
                    and getattr(node, plug_name) \
                        != node.trait(plug_name).default:
                value = getattr(node, plug_name)
                print('        self.nodes["%s"].%s = %s'
                      % (name, plug_name, get_repr_value(value)), file=pyf)

    def _write_iteration(process_iter, pyf, name, enabled):
        process = process_iter.process
        else:
            if func:
            else:

        iteration_params = ', '.join(process_iter.iterative_parameters)
        # TODO: optional plugs, non-exported plugs...
        print('        self.add_iterative_process("%s", "%s", '
              'iterative_plugs=%s)'
              % (name, procname, process_iter.iterative_parameters),
              file=pyf)

    def _write_switch(switch, pyf, name, enabled):
        inputs_set = set()
        inputs = []
        optional = []
        opt_in = []
        options = ''
        for plug_name, plug in six.iteritems(switch.plugs):
            if plug.output:
                outputs.append(plug_name)
                if plug.optional:
                    optional.append(plug_name)
            else:
                name_parts = plug_name.split("_switch_")
                if len(name_parts) == 2 \
                        and name_parts[0] not in inputs_set:
                    inputs_set.add(name_parts[0])
                    inputs.append(name_parts[0])
                    if plug.optional:
                        opt_in.append(name_parts[0])
        optional_p = ''
        if len(optional) != 0:
            optional_p = ', make_optional=%s' % repr(optional)

        # output types
        from .pipeline_tools import trait_str
        out_types = []
        for p in switch._outputs:
            out_types.append(trait_str(switch.trait(p)))
        output_types = '[' + ', '.join(out_types) + ']'

        opt_inputs = getattr(switch, '_optional_input_nodes', None)
        if opt_inputs:
            opt_inputs = [i[1] for i in opt_inputs if i[0] in inputs]
            if opt_inputs == inputs:
                opt_inputs = True
            options += ', opt_nodes=%s' % repr(opt_inputs)
        value_p = ''
        if switch.switch != inputs[0]:
            value_p = ', switch_value=%s' % repr(switch.switch)
        print('        self.add_switch("%s", %s, %s, output_types=%s%s%s%s, export_switch=False)'
              % (name, repr(inputs), repr(outputs), output_types, optional_p,
                 value_p, options),
              file=pyf)

    def _write_optional_output_switch(switch, pyf, name, enabled):
        output = None
        input = None
                output = plug_name
            else:
                if len(name_parts) == 2 and name_parts[0] != '_none':
                    input = name_parts[0]
        if not name and output:
            name = output
        if not output or output == name:
            print('        self.add_optional_output_switch("%s", "%s")'
                  % (name, input), file=pyf)
        else:
            print('        self.add_optional_output_switch("%s", "%s", "%s")'
                  % (name, input, output), file=pyf)

    def _write_processes(pipeline, pyf):
        print('        # nodes', file=pyf)
        nodes = []
        # sort nodes, processes first
        for node_name, node in six.iteritems(pipeline.nodes):
                continue
            nodes.append((node_name, node))
        for node_name, node in nodes:
            if isinstance(node, OptionalOutputSwitch):
                _write_optional_output_switch(node, pyf, node_name,
                                              node.enabled)
            elif isinstance(node, Switch):
                _write_switch(node, pyf, node_name, node.enabled)
            elif isinstance(node, ProcessNode) \
                    and isinstance(node.process, ProcessIteration):
                _write_iteration(node.process, pyf, node_name, node.enabled)
            elif isinstance(node, ProcessNode):
                _write_process(node, pyf, node_name, node.enabled,
                               node_name in pipeline._skip_invalid_nodes)
            else:
                # custom node
                _write_custom_node(node, pyf, node_name, node.enabled)

    def _write_processes_selections(pipeline, pyf):
        selection_parameters = []
        if hasattr(pipeline, 'processes_selection'):
            print('\n        # processes selection', file=pyf)
            for selector_name, groups \
                    in six.iteritems(pipeline.processes_selection):
                print('        self.add_processes_selection("%s", %s)'
                      % (selector_name, repr(groups)), file=pyf)
        return selection_parameters

    def _write_export(pipeline, pyf, param_name):
        plug = pipeline.pipeline_node.plugs[param_name]
            link = list(plug.links_from)[0]
        else:
            link = list(plug.links_to)[0]
        trait = pipeline.trait(param_name)
        node_name, plug_name = link[:2]
        if param_name == plug_name:
            param_name = ''
        else:
            param_name = ', "%s"' % param_name
        weak_link = ''
        if link[-1]:
            weak_link = ', weak_link=True'
        if trait is None:
            print('missing tait', param_name, 'on:', pipeline)
        #if pipeline.trait(param_name).optional:
        is_optional = ', is_optional=%s' % repr(trait.optional)
        print('        self.export_parameter("%s", "%s"%s%s%s)'
              % (node_name, plug_name, param_name, weak_link, is_optional),
              file=pyf)
        return node_name, plug_name

    def _write_links(pipeline, pyf):
        exported = set()
        print('\n        # links', file=pyf)
                if (node_name == "" and not plug.output) \
                        or (node_name != "" and plug.output):
                    for link in links:
                        exported_plug = None
                        if node_name == "":
                            src = plug_name
                            if src not in exported:
                                exported_plug = _write_export(pipeline, pyf,
                                                              src)
                                exported.add(src)
                        else:
                            src = "%s.%s" % (node_name, plug_name)
                        if link[0] == "":
                            dst = link[1]
                            if dst not in exported:
                                                              dst)
                                exported.add(dst)
                        else:
                            dst = "%s.%s" % (link[0], link[1])
                        if not exported_plug \
                                or '.'.join(exported_plug) not in (src, dst):
                            weak_link = ''
                            if link[-1]:
                                weak_link = ', weak_link=True'
                            print('        self.add_link("%s->%s"%s)'
                                  % (src, dst, weak_link), file=pyf)

    def _write_param_order(pipeline, pyf):
        if len(pipeline.user_traits()) == 0:
            return
        print('\n        # parameters order', file=pyf)
        names = ['"%s"' % n for n in pipeline.user_traits().keys()
                 if n not in ('nodes_activation', 'pipeline_steps',
                              'visible_groups')]
        print('\n        self.reorder_traits((%s))'
              % ', '.join(names), file=pyf)

    def _write_steps(pipeline, pyf):
        steps = pipeline.trait('pipeline_steps')
        if steps and getattr(pipeline, 'pipeline_steps', None):
            print('\n        # pipeline steps', file=pyf)
            for step_name, step \
                    in six.iteritems(pipeline.pipeline_steps.user_traits()):
                enabled = getattr(pipeline.pipeline_steps, step_name)
                enabled_str = ''
                if not enabled:
                    enabled_str = ', enabled=false'
                nodes = step.nodes
                print('        self.add_pipeline_step("%s", %s%s)'
                      % (step_name, repr(nodes), enabled_str), file=pyf)

    def _write_nodes_positions(pipeline, pyf):
        node_position = getattr(pipeline, "node_position", None)
            print('\n        # nodes positions', file=pyf)
            print('        self.node_position = {', file=pyf)
            for node_name, pos in six.iteritems(pipeline.node_position):
                if not isinstance(pos, (list, tuple)):
                    # pos is probably a QPointF
                    pos = (pos.x(), pos.y())
                print('            "%s": %s,' % (node_name, repr(pos)),
                      file=pyf)
            print('        }', file=pyf)
        if hasattr(pipeline, "scene_scale_factor"):
            print('        self.scene_scale_factor = %s'
                  % repr(pipeline.scene_scale_factor), file=pyf)
            
    ########### add by Irmage OM #########################       
    def _write_nodes_dimensions(pipeline, pyf):
        node_dimension = getattr(pipeline, "node_dimension", None)
            print('\n        # nodes dimensions', file=pyf)
            print('        self.node_dimension = {', file=pyf)
            for node_name, dim in six.iteritems(pipeline.node_dimension):
                if not isinstance(dim, (list, tuple)):
                    dim = (dim.width(), dim.height())
                print('            "%s": %s,' % (node_name, repr(dim)),
                      file=pyf)
    ######################################################  

    def _write_doc(pipeline, pyf):
        if hasattr(pipeline, "__doc__"):
            docstr = pipeline.__doc__
            if docstr == Pipeline.__doc__:
                docstr = ""  # don't use the builtin Pipeline help
            else:
                # remove automatically added doc
                splitdoc = docstr.split('\n')
                notepos = [i for i, x in enumerate(splitdoc[:-2])
                              if x.endswith('.. note::')]
                autodocpos = None
                if notepos:
                    for i in notepos:
                        if splitdoc[i+2].find(
                                "* Type '{0}.help()'".format(
                                    pipeline.__class__.__name__)) != -1:
                            autodocpos = i
                if autodocpos is not None:
                    # strip empty trailing lines
                    while autodocpos >= 1 \
                            and splitdoc[autodocpos - 1].strip() == '':
                        autodocpos -= 1
                    docstr = '\n'.join(splitdoc[:autodocpos]) + '\n'
            if docstr.strip() == '':
                docstr = ''
            if docstr:
                doc = docstr.split('\n')
                docstr = '\n'.join([repr(x)[1:-1] for x in doc])
                print('    \"\"\"%s    \"\"\"' % docstr, file=pyf)

    def _write_values(pipeline, pyf):
        first = True
        for param_name, trait in six.iteritems(pipeline.user_traits()):
            if param_name not in pipeline.pipeline_node.plugs:
                continue
            # default cannot be set after trait creation
            #if trait.default:
                #if first:
                    #first = False
                    #print('\n        # default and initial values', file=pyf)
                #print('        self.trait("%s").default = %s'
                      #% (param_name, repr(trait.default)), file=pyf)
            value = getattr(pipeline, param_name)
            if value != trait.default and value not in (None, '', Undefined):
                if isinstance(value, Controller):
                    value_repr = repr(dict(value.export_to_dict()))
                else:
                    value_repr = repr(value)
                try:
                    eval(value_repr)
                    print('warning, value of parameter %s cannot be saved'
                          % param_name)
                    continue
                if first:
                    first = False
                    print('\n        # default and initial values', file=pyf)
                print('        self.%s = %s'
                      % (param_name, value_repr), file=pyf)

    class_name = type(pipeline).__name__
    if class_name == 'Pipeline':
        # don't accept the base Pipeline class
        if isinstance(py_file, str):
            class_name = os.path.basename(py_file)
        elif hasattr(py_file, 'name'):
            class_name = os.path.basename(py_file.name)
        else:
            class_name = 'CustomPipeline'
        if '.' in class_name:
            class_name = class_name[:class_name.index('.')]
        class_name = class_name[0].upper() + class_name[1:]

    if isinstance(py_file, str):
        pyf = open(py_file, 'w')
    else:
        pyf = py_file

    print('# -*- coding: utf-8 -*-\n', file=pyf)
    print('from capsul.api import Pipeline', file=pyf)
    print('import traits.api as traits', file=pyf)
    print(file=pyf)
    print('class %s(Pipeline):' % class_name, file=pyf)

    _write_doc(pipeline, pyf)

    print('    def pipeline_definition(self):', file=pyf)

    _write_processes(pipeline, pyf)
    _write_links(pipeline, pyf)
    _write_param_order(pipeline, pyf)
    _write_processes_selections(pipeline, pyf)
    _write_steps(pipeline, pyf)
    _write_values(pipeline, pyf)
    _write_nodes_positions(pipeline, pyf)
    _write_nodes_dimensions(pipeline, pyf) #add by Irmage OM

    print('\n        self.do_autoexport_nodes_parameters = False',
          file=pyf)

    if pyf is not py_file:
        pyf.close()

'''
Build graph with dependencies from a pipeline

Classes
=======
:class:`GraphNode`
'''

import six

class GraphNode(object):
    """ Simple Graph Node Structure

    name : str
    meta : object
        a python object stored in the node
    links_to : list
         object to store the graph edges: successor
    links_from : list
        object to store the graph edges: predecessor
    links_to_degree : int
        degree of the node regarding the successors
    links_from_degree : int
        degree of the node regarding the predecessors

    Methods
    --------
    add_link_to
    remove_link_to
    add_link_from
    remove_link_from
    """

    def __init__(self, name, meta):
        """ Create a Graph Node

        the name of the node

        meta: object
        an python object to store in the node
        """
        self.meta = meta
        # variables to store the graph edges
        self.links_to = []
        self.links_from = []
        # the degree of the node
        self.links_to_degree = 0
        self.links_from_degree = 0

    def add_link_to(self, node):
        """ Method to add a Successor

        node: GraphNode (mandatory)
        the successor node
        """
        if node not in self.links_to:
            self.links_to.append(node)
            self.links_to_degree += 1

    def remove_link_to(self, node):
        """ Method to remove a Successor

        """
        if node in self.links_to:
            self.links_to.remove(node)
            self.links_to_degree -= 1

    def add_link_from(self, node):
        """ Method to add a Predecessor

        the predecessor node
        """
        if node not in self.links_from:
            self.links_from.append(node)
            self.links_from_degree += 1

    def remove_link_from(self, node):
        """ Method to remove a Predecessor

        """
        if node in self.links_from:
            self.links_from.remove(node)
            self.links_from_degree -= 1

class Graph(object):
    """ Simple Graph Structure on which we want to perform a
    topological tree (no cycle).

    The algorithm is based on the R.E. Tarjanlinear linear
    optimization (O(N+A)).

    _nodes : dict
        the graph nodes {node.name: node}
    _links : list
        graph edges (from_node, to_node)

    Methods
    --------
    add_node
    find_node
    add_link
    topological_sort
    """

        """ Create a Graph
        """
        self._nodes = {}
        self._links = []

    def add_node(self, node):
        """ Method to add a GraphNode in the Graph

        the node to insert
        """
        logging.debug("node: {0}".format(node.name))
        if not isinstance(node, GraphNode):
            raise Exception("Expect a GraphNode, got {0}".format(node))
        if node.name in self._nodes:
            raise Exception("Expect a GraphNode with a unique name, "
                            "got {0}".format(node))
        self._nodes[node.name] = node

    def find_node(self, node_name):
        """ Method to find a GraphNode in the Graph

        the name of the desired node
        """
        if node_name in self._nodes:
            return self._nodes[node_name]

    def add_link(self, from_node, to_node):
        """ Method to add an edge between two GraphNodes of the Graph

        from_node: GraphNode (mandatory)
        a node in the graph
        to_node: GraphNode (mandatory)
        """
        logging.debug("link: {0}->{1}".format(from_node, to_node))
        if from_node not in self._nodes:
            raise Exception("Node {0} is not defined in the Graph."
                   "Use add_node() method".format(from_node))
        if to_node not in self._nodes:
                   "Use add_node() method".format(to_node))
        if (from_node, to_node) not in self._links:
            self._nodes[to_node].add_link_from(self._nodes[from_node])
            self._nodes[from_node].add_link_to(self._nodes[to_node])
            self._links.append((from_node, to_node))

    def topological_sort(self):
        """ Perform the topological sort: find an order in which all the
        nodes can be taken.
        Step 1: Identify nodes that have no incoming link (nnil).
        Step 2: Loop until there are nnil
        a) Delete the current nodes c_nnil of in-degree 0.
        b) Place it in the output.
        c) Remove all its outgoing links from the graph.
        d) If the node has in-degree 0, add the node to nnil.
        Step 3: Assert that there is no loop in the graph.

        Returns
        -------
        output: list of tuple
            a list of ordered nodes with a tuple element containing the node
            name and the node meta element.
        """
        ordered_nodes = []

        # Step 1
        nnil = []
        for name, node in six.iteritems(self._nodes):
            if node.links_from_degree == 0:
                nnil.append(node)

        # Step 2
        while len(nnil):
        #-- a
            c_nnil = nnil.pop()
        #-- b
            ordered_nodes.append(c_nnil)
        #-- c
            for node in c_nnil.links_to:
                node.remove_link_from(c_nnil)
        #-- d
                if node.links_from_degree == 0:
                    nnil.append(node)

        # Step 3
        if len(ordered_nodes) == len(self._nodes):
            return [(node.name, node.meta) for node in ordered_nodes]
        else:
            raise Exception("There is loop in the Graph."
                            "Please inverstigate")

    """ A toy example:
    slip -> chaussettes -> chemise -> veste -> pantalon -> ceinture ->
    chaussures -> cravate
    """

    objects = ["chaussures", "chaussettes", "slip", "pantalon", "ceinture",
        "chemise", "veste", "cravate"]

    dependencies = [
        ("slip", "pantalon"),
        ("chemise", "cravate"),
        ("chemise", "pantalon"),
        ("pantalon", "ceinture"),
        ("chaussettes", "chaussures"),
        ("pantalon", "chaussures"),
        ("ceinture", "chaussures"),
        ("chemise", "veste"),
        ]

    g = Graph()

    for o in objects:
        g.add_node(GraphNode(o, None))

    for d in dependencies:
        g.add_link(d[0], d[1])

    r = g.topological_sort()
    r = [x[0] for x in r]
    print(" -> ".join(r))

'''
XML IO for pipelines

Functions
=========
:func:`create_xml_pipeline`
:func:`save_xml_pipeline`
'''

import os
import six
import sys

from capsul.pipeline.pipeline_construction import PipelineConstructor
from capsul.pipeline.pipeline_nodes import PipelineNode

def create_xml_pipeline(module, name, xml_file):
    """
    Create a pipeline class given its Capsul XML 2.0 representation.
    
    module: str (mandatory)
        name of the module for the created Pipeline class (the Python module is
        not modified).
    name: str (mandatory)
        name of the new pipeline class
    xml_file: str (mandatory)
        name of file containing the XML description or XML string.
    
    """
    if hasattr(xml_file, 'read') or os.path.exists(xml_file):
        xml_pipeline = ET.parse(xml_file).getroot()
    elif isinstance(xml_file, bytes):
        xml_pipeline = ET.fromstring(xml_file)
    else:
        encoded_xml = xml_file.encode('utf-8')
        xml_pipeline = ET.fromstring(encoded_xml)

    version = xml_pipeline.get('capsul_xml')
    if version and version != '2.0':
        raise ValueError('Only Capsul XML 2.0 is supported, not %s' % version)

    class_name = xml_pipeline.get('name')
    if class_name:
        if name is None:
            name = class_name
        elif name != class_name:
            raise KeyError('pipeline name (%s) and requested object name '
                           '(%s) differ.' % (class_name, name))
    elif name is None:
        name = os.path.basename(xml_file).rsplit('.', 1)[0]

    builder = PipelineConstructor(module, name)

    for child in xml_pipeline:
        if child.tag == 'doc':
            if child.text is not None:
                builder.set_documentation(child.text.strip())
        elif child.tag == 'process':
            process_name = child.get('name')
            module = child.get('module')
            args = (process_name, module)
            kwargs = {}
            nipype_usedefault = []
            iterate = []
            iteration = child.get('iteration')
            links = []
            if iteration:
                iterate = [x.strip() for x in iteration.split(',')]
            for process_child in child:
                if process_child.tag == 'set':
                    name = process_child.get('name')
                    value = process_child.get('value')
                    value = string_to_value(value)
                    if value is not None:
                        kwargs[name] = value
                    kwargs.setdefault('make_optional', []).append(name)
                elif process_child.tag == 'nipype':
                    usedefault = process_child.get('usedefault')
                    if usedefault == 'true':
                        nipype_usedefault.append(name)
                    copyfile = process_child.get('copyfile')
                    if copyfile == 'true':
                        kwargs.setdefault('inputs_to_copy', []).append(name)
                    elif copyfile == 'discard':
                        kwargs.setdefault('inputs_to_clean', []).append(name)
                elif process_child.tag == 'link':
                    # internal export
                    source = process_child.get('source')
                    dest = process_child.get('dest')
                    links.append([source, dest, None])
                else:
                    raise ValueError('Invalid tag in <process>: %s' %
                                     process_child.tag)
            if links:
                todel = []
                    k = link[0]
                    if '.' in k:
                        k = link[1]
                    v = kwargs.get(k)
                    if v is not None:
                        link[2] = v
                        del kwargs[k]
            if iterate:
                kwargs['iterative_plugs'] = iterate
                builder.add_iterative_process(*args, **kwargs)
            else:
                builder.add_process(*args, **kwargs)
            for name in nipype_usedefault:
                builder.call_process_method(process_name, 'set_usedefault',
                                            name, True)
            if links:
                    builder.add_subpipeline_link(process_name, link[0],
                                                 link[1], value=link[2])
            enabled = child.get('enabled')
            if enabled == 'false':
                builder.set_node_enabled(process_name, False)
        elif child.tag == 'switch':
            switch_name = child.get('name')
            value = child.get('switch_value')
            kwargs = {'export_switch': False}
            if value:
                kwargs['switch_value'] = value
            inputs = []
            outputs = []
                if process_child.tag == 'input':
                    inputs.append(name)
                elif process_child.tag == 'output':
                    optional = process_child.get('optional')
                    if optional == 'true':
                        kwargs.setdefault('make_optional', []).append(name)
            builder.add_switch(switch_name, inputs, outputs, **kwargs)
                builder.set_node_enabled(switch_name, False)
        elif child.tag == 'optional_output_switch':
            kwargs = {}
            input = None
            output = None
                    if input is not None:
                            'Several inputs in optional_output_switch')
                    input = process_child.get('name')
                    if output is not None:
                            'Several outputs in optional_output_switch')
                    output = process_child.get('name')
            if input is None:
                raise ValueError('No input in optional_output_switch')
            builder.add_optional_output_switch(switch_name, input, output)

        elif child.tag == 'custom_node':
            node_name = child.get('name')
            params = {}
            kwargs = {}
            make_optional = []
                if process_child.tag == 'config':
                    for p, value in process_child.items():
                          params[p] = string_to_value(value)
                elif process_child.tag == 'set':
                    make_optional.append(name)
                else:
            # TODO optional plugs
            builder.add_custom_node(node_name, module, params, make_optional,
                                    **kwargs)

                builder.set_node_enabled(node_name, False)

        elif child.tag == 'link':
            source = child.get('source')
            dest = child.get('dest')
            weak_link = child.get('weak_link')
            if weak_link == 'true':
                weak_link = True
            else:
                weak_link = False
            builder.add_link('%s->%s' % (source, dest),
                              weak_link=weak_link, allow_export=True)
        elif child.tag == 'processes_selection':
            selection_parameter = child.get('name')
            selection_groups = OrderedDict()
            for select_child in child:
                if select_child.tag == 'processes_group':
                    group_name = select_child.get('name')
                    group = selection_groups[group_name] = []
                    for group_child in select_child:
                        if group_child.tag == 'process':
                            group.append(group_child.get('name'))
                        else:
                            raise ValueError('Invalid tag in <processes_group>'
                                             '<process>: %s' % group_child.tag)
                else:
                    raise ValueError('Invalid tag in <processes_selection>: %s'
                                     % select_child.tag)
            builder.add_processes_selection(selection_parameter,
                                            selection_groups)
        elif child.tag == 'pipeline_steps':
            for step_child in child:
                step_name = step_child.get('name')
                enabled = step_child.get('enabled')
                if enabled == 'false':
                    enabled = False
                else:
                    enabled = True
                nodes = []
                for step_node in step_child:
                    nodes.append(step_node.get('name'))
                builder.add_pipeline_step(step_name, nodes, enabled)
        elif child.tag == 'gui':
            for gui_child in child:
                if gui_child.tag == 'position':
                    name = gui_child.get('name')
                    x = float(gui_child.get('x'))
                    y = float(gui_child.get('y'))
                    builder.set_node_position(name, x, y)
                elif gui_child.tag == 'zoom':
                    builder.set_scene_scale_factor(
                        float(gui_child.get('level')))
                else:
                    raise ValueError('Invalid tag in <gui>: %s' %
                                     gui_child.tag)
        elif child.tag == 'plug_state':
            key = child.get('option')
            value = child.get('value')
            plug_name = child.get('param')
            builder.add_plug_state(plug_name, key, value)
        elif child.tag == 'state':
            node_name = child.get('node')
            builder.add_node_state(node_name, key, value)
        else:
            raise ValueError('Invalid tag in <pipeline>: %s' % child.tag)
    return builder.pipeline

def save_xml_pipeline(pipeline, xml_file):
    '''
    Save a pipeline in an XML file

    xml_file: str or file-like object
        XML file to save the pipeline in
    '''
        OptionalOutputSwitch

    def _write_process(node, parent, name, dont_write_plug_values=set(),
                       init_plug_values={}):
        procnode = ET.SubElement(parent, 'process')
        else:
            if func:
            else:
                if classname == 'Pipeline':
                    # don't accept the base Pipeline class
                    classname = name
                    if '.' in class_name:
                        classname = classname[:classname.index('.')]
                    classname = classname[0].upper() + class_name[1:]
        procnode.set('module', "%s.%s" % (mod, classname))
        procnode.set('name', name)
        try:
            proc_copy = get_process_instance("%s.%s" % (mod, classname))
            proc_copy = process  # don't duplicate, don't test differences
            # WARNING: not sure I'm doing the right things for nipype. To be
            # fixed if needed.
            if process.activate_copy:
                for param in process.inputs_to_copy:
                    elem = ET.SubElement(procnode, 'nipype')
                    elem.set('name', param)
                    if param in process.inputs_to_clean:
                        elem.set('copyfile', 'discard')
                    else:
                        elem.set('copyfile', 'true')
                    np_input = getattr(process._nipype_interface.inputs, param)
                    if np_input:
                        use_default = getattr(np_input, 'usedefault', False) # is it that?
                        if use_default:
                            elem.set('use_default', 'true')
            for param, np_input in \
                    six.iteritems(process._nipype_interface.inputs.__dict__):
                use_default = getattr(np_input, 'usedefault', False) # is it that?
                if use_default and param not in process.inputs_to_copy:
                    elem.set('use_default', 'true')
        # set initial values
        dont_write_plug_values = set(dont_write_plug_values)
        dont_write_plug_values.update(('nodes_activation',
                                       'selection_changed'))

        _write_states(parent, name, node, proc_copy)

        for param_name, trait in six.iteritems(process.user_traits()):
            if param_name not in proc_copy.traits():
                continue
            if param_name not in dont_write_plug_values:
                if param_name in init_plug_values:
                    value = init_plug_values[param_name]
                else:
                    value = getattr(process, param_name)
                if value not in (None, Undefined, '', []) \
                        or (trait.optional
                            and not proc_copy.trait(param_name).optional):
                    if isinstance(value, Controller):
                        value_repr = repr(dict(value.export_to_dict()))
                    else:
                        value_repr = repr(value)
                    try:
                        eval(value_repr)
                        print('warning, value of parameter %s cannot be saved'
                              % param_name)
                        continue
                    elem = ET.SubElement(procnode, 'set')
                    elem.set('name', param_name)
                    elem.set('value', value_repr)
        return procnode

    def _write_custom_node(node, parent, name):
        etnode = ET.SubElement(parent, 'custom_node')
        etnode.set('module', "%s.%s" % (mod, classname))
        etnode.set('name', name)
            if len(c.user_traits()) != 0:
                et = ET.SubElement(etnode, 'config')
                for param_name in c.user_traits():
                    value = getattr(c, param_name)
                    else:
                    try:
                        continue
                    et.set(param_name, value_repr)
        for param_name, plug in six.iteritems(node.plugs):
            trait = node.trait(param_name)
            value = getattr(node, param_name)
            if value not in (None, Undefined, '', []) or trait.optional:
                else:
                try:
                    continue
                elem = ET.SubElement(etnode, 'set')
                elem.set('name', param_name)
                elem.set('value', value_repr)

    def _write_iteration(node, parent, name):
        process_iter = node.process
        it_node = ProcessNode(node.pipeline, name, process_iter.process)
        iter_values = dict([(p, getattr(process_iter, p))
                            for p in process_iter.iterative_parameters])
        procnode = _write_process(
            it_node, parent, name, init_plug_values=iter_values)
        procnode.set('iteration', iteration_params)

    def _write_switch(switch, parent, name):
        swnode = ET.SubElement(parent, 'switch')
        swnode.set('name', name)
        inputs = set()
                elem = ET.SubElement(swnode, 'output')
                elem.set('name', plug_name)
                    elem.set('optional', 'true')
            else:
                        and name_parts[0] not in inputs:
                    inputs.add(name_parts[0])
                    elem = ET.SubElement(swnode, 'input')
                    elem.set('name', name_parts[0])
                        elem.set('optional', 'true')
        swnode.set('switch_value', six.text_type(switch.switch))
        return swnode

    def _write_optional_output_switch(switch, parent, name):
        swnode = ET.SubElement(parent, 'optional_output_switch')
            else:
                if len(name_parts) == 2:
                    if input != '_none':
                        elem = ET.SubElement(swnode, 'input')
                        elem.set('name', name_parts[0])
                        if plug.optional:
                            elem.set('optional', 'true')

    def _write_processes(pipeline, root):
                continue
                xmlnode = _write_optional_output_switch(node, root, node_name)
                xmlnode = _write_switch(node, root, node_name)
                xmlnode = _write_iteration(node, root, node_name)
                xmlnode = _write_process(node, root, node_name)
            else:
                xmlnode = _write_custom_node(node, root, node_name)
            if not node.enabled:
                xmlnode.set('enabled', 'false')

    def _write_processes_selections(pipeline, root):
                selection_parameters.append(selector_name)
                sel_node = ET.SubElement(root, 'processes_selection')
                sel_node.set('name', selector_name)
                for group_name, group in six.iteritems(groups):
                    grp_node = ET.SubElement(sel_node, 'processes_group')
                    grp_node.set('name', group_name)
                    for node in group:
                        proc_node = ET.SubElement(grp_node, 'process')
                        proc_node.set('name', node)

    def _write_links(pipeline, root):
                        else:
                        else:
                        linkelem = ET.SubElement(root, 'link')
                        linkelem.set('source', src)
                        linkelem.set('dest', dst)
                        if link[-1]:
                            linkelem.set('weak_link', "true")

    def _write_steps(pipeline, root):
        steps_node = None
            steps_node = ET.SubElement(root, 'pipeline_steps')
                step_node = ET.SubElement(steps_node, 'step')
                step_node.set('name', step_name)
                    step_node.set('enabled', 'false')
                for node in nodes:
                    node_item = ET.SubElement(step_node, 'node')
                    node_item.set('name', node)
        return steps_node

    def _write_nodes_positions(pipeline, root):
        gui = None
        if hasattr(pipeline, "node_position") and pipeline.node_position:
            gui = ET.SubElement(root, 'gui')
                node_pos = ET.SubElement(gui, 'position')
                node_pos.set('name', node_name)
                if hasattr(pos, 'x'):
                    # it's a QPointF
                    node_pos.set('x', six.text_type(pos.x()))
                    node_pos.set('y', six.text_type(pos.y()))
                else:
                    # it's a python iterable
                    node_pos.set('x', six.text_type(pos[0]))
                    node_pos.set('y', six.text_type(pos[1]))
        return gui

    def _write_doc(pipeline, root):
            else:
        else:
            docstr = ''
        if docstr.strip() == '':
            docstr = ''
        doc = ET.SubElement(root, 'doc')
        doc.text = docstr
        return doc

    def _write_states(root, name, node, proc_copy):
        todo = [(name, node, ProcessNode(node.pipeline, node.name, proc_copy))]
                item = ET.SubElement(root, 'state')
                item.set('option', 'enabled')
                item.set('value', '0')
                item.set('node', self_str)

                                    link_el = ET.SubElement(root, 'link')
                                    link_el.set('source',
                                                '%s.%s' % (self_str,
                                                           param_name))
                                    link_el.set('dest',
                                                '.'.join((self_str, link[0],
                                                          link[1])))
                            else:

                    item = ET.SubElement(root, 'plug_state')
                    item.set('option', 'optional')
                    item.set('value', str(int(optional)))
                    item.set('param', '%s.%s' % (self_str, param_name))

                        continue
                    todo.append(('%s.%s' % (self_str, node_name), snode,
                                 scnode))

    root = ET.Element('pipeline')
    root.set('capsul_xml', '2.0')
    class_name = pipeline.__class__.__name__
    if pipeline.__class__ is Pipeline:
        # if directly a Pipeline, then use a default new name
        class_name = 'CustomPipeline'
    root.set('name', class_name)

    _write_doc(pipeline, root)
    _write_processes(pipeline, root)
    _write_links(pipeline, root)
    _write_processes_selections(pipeline, root)
    _write_steps(pipeline, root)
    gui_node = _write_nodes_positions(pipeline, root)

    if hasattr(pipeline, "scene_scale_factor"):
        if gui_node is None:
            gui_node = ET.SubElement(root, 'gui')
        scale_node = ET.SubElement(gui_node, 'zoom')
        scale_node.set('level', six.text_type(pipeline.scene_scale_factor))

    tree = ET.ElementTree(root)
    try:
        tree.write(xml_file)
    except TypeError:
        # probably writing to a unicode stringIO
        # tree.write can only write encoded streeams
        import io
        b = io.BytesIO()
        tree.write(b)
        xml_file.write(b.getvalue().decode())

'''
:class:`StrCatNode`
-------------------
'''

from capsul.pipeline.pipeline_nodes import Node
import sys
import six

class StrCatNode(Node):
    '''
    This "inert" node concatenates its inputs (as strings) and generates the
    concatenation on one of its plugs. All plugs may be inputs or outputs.
    '''

    _doc_path = 'api/pipeline.html#strcatnode'

    def __init__(self, pipeline, name, params, concat_plug, outputs,
                 make_optional=(), param_types={}):
        '''
            pipeline which will hold the node
        name: str
            node name
        params: list
            names of parameters to be concatenated
        concat_plug: str
            name of the concatenated plug (should not be part of params)
        outputs: list
            list of parameters names which are outputs. May include elements
            from params, and/or concat_plug
            list of plug names which should be optional.
        param_types: dict
            parameters types dict: {param_name: trait_type_as_string}

        '''
        node_inputs = [dict(name=i, optional=(i in make_optional))
                       for i in params if i not in outputs]
                        for i in outputs if i in outputs]
        if concat_plug in outputs:
            node_outputs.append({'name': concat_plug,
                                 'optional': concat_plug in make_optional})
        else:
            node_inputs.append({'name': concat_plug,
                                'optional': concat_plug in make_optional})
        super(StrCatNode, self).__init__(pipeline, name, node_inputs,
                                      node_outputs)
        self._concat_sequence = params
        self._concat_plug = concat_plug
        self.add_parameters(param_types)
        self.cat_callback()
        self.set_callbacks()

    def add_parameters(self, param_types={}):
        added_traits = [self._concat_plug]
        for name in self._concat_sequence + added_traits:
            plug = self.plugs[name]
            ptype = param_types.get(name)
            if ptype is None:
                ptype = traits.Any(traits.Undefined)
            self.add_trait(name, ptype)
            self.trait(name).output = plug.output
            self.trait(name).optional = plug.optional

    def set_callbacks(self, update_callback=None):
        if update_callback is None:
            update_callback = self.cat_callback
        self.on_trait_change(update_callback, self._concat_sequence)

    def cat_callback(self):
        result = ''.join([six.text_type(getattr(self, name))
                          for name in self._concat_sequence])
        setattr(self, self._concat_plug, result)

        c.parameters = self._concat_sequence
        c.concat_plug = self._concat_plug
        param_types = [self.trait(x).trait_type.__class__.__name__
                       for x in c.parameters + [c.concat_plug]]
        c.outputs = [x for x in c.parameters + [c.concat_plug]
                     if self.trait(x).output]
        c.param_types = param_types
        return c

        c.add_trait('parameters', traits.List(traits.Str()))
        c.add_trait('concat_plug', traits.Str())
        c.add_trait('outputs', traits.List(traits.Str()))
        c.add_trait('param_types', traits.List(traits.Str('Str')))
        return c

        params = [(x, x in conf_controller.outputs) for x in conf_controller.parameters]
        t = {}
        if conf_controller.param_types:
            for pname, ptype in zip(conf_controller.parameters
                                    + [conf_controller.concat_plug],
                                    conf_controller.param_types):
                t[pname] = getattr(traits, ptype)()
        node = StrCatNode(pipeline, name, conf_controller.parameters,
                          conf_controller.concat_plug,
                          conf_controller.outputs,
                          param_types=t)

    def params_to_command(self):
        return ['custom_job']

    def build_job(self, name=None, referenced_input_files=[],
                  referenced_output_files=[], param_dict=None):
        from soma_workflow.custom_jobs import StrCatJob
        param_dict['input_names'] = self._concat_sequence
        param_dict['output_name'] = self._concat_plug
        # transmit values
        for param in self._concat_sequence:
            param_dict[param] = getattr(self, param)
        # [re] build the concatenated output
        param_dict[self._concat_plug] = getattr(self, self._concat_plug)
        job = StrCatJob(name=name,
                        referenced_input_files=referenced_input_files,
                        referenced_output_files=referenced_output_files,
                        param_dict=param_dict)
        return job

'''
:class:`StrConvNode`
'''

import six
import sys

class StrConvNode(Node):
    '''
    This "inert" node converts the input into a string.

    '''

    _doc_path = 'api/pipeline.html#strconvnode'

    def __init__(self, pipeline, name, input_type=None):
        in_traitsl = ['input']
        out_traitsl = ['output']
        in_traits = []
        out_traits = []
        for tr in in_traitsl:
            in_traits.append({'name': tr, 'optional': True})
        for tr in out_traitsl:
            out_traits.append({'name': tr, 'optional': True})
        super(StrConvNode, self).__init__(
            pipeline, name, in_traits, out_traits)
        if input_type:
            ptype = input_type
        else:
            ptype = traits.Any(traits.Undefined)

        self.add_trait('input', ptype)
        self.trait('input').output = False
        is_output = True  # not a choice for now.
        self.add_trait('output', traits.Str(output=is_output))
        self.input = 0
        self.filter_callback('input', 0)

        inputs = ['input']
            update_callback = self.filter_callback
        for name in inputs:
            self.on_trait_change(update_callback, name)

    def filter_callback(self, name, value):
        self.output = six.text_type(self.input)

        c.param_type = self.trait('input').trait_type.__class__.__name__
        return c

        c.add_trait('param_type', traits.Str('Any'))
        return c

        t = None
        if conf_controller.param_type == 'Str':
            t = traits.Str(traits.Undefined)
        elif conf_controller.param_type == 'File':
            t = traits.File(traits.Undefined)
        elif conf_controller.param_type == 'Any':
            t = traits.Any()
        elif conf_controller.param_type not in (None, traits.Undefined):
            t = getattr(traits, conf_controller.param_type)()
        node = StrConvNode(pipeline, name, input_type=t)

            return []
            connected_plug_name = 'input'
        else:
            connected_plug_name = 'output'
        else:
            else:
                break

'''
:class:`CrossValidationFoldNode`
'''

import sys

class CrossValidationFoldNode(Node):
    '''
    This "inert" node filters a list to separate it into (typically) learn and
    test sublists.

    The "outputs" are "train" and "test" output traits.
    '''

    _doc_path = 'api/pipeline.html#crossvalidationfoldnode'

        in_traitsl = ['inputs', 'fold', 'nfolds']
        out_traitsl = ['train', 'test']
        super(CrossValidationFoldNode, self).__init__(
        else:

        self.add_trait('inputs', traits.List(ptype, output=False))
        self.add_trait('fold', traits.Int())
        self.add_trait('nfolds', traits.Int(10))
        self.add_trait('train', traits.List(ptype, output=is_output))
        self.add_trait('test', traits.List(ptype, output=is_output))

        inputs = ['inputs', 'fold', 'nfolds']

    def filter_callback(self):
        n = len(self.inputs) // self.nfolds
        ninc = len(self.inputs) % self.nfolds
        begin = self.fold * n + min((ninc, self.fold))
        end = min((self.fold + 1) * n + min((ninc, self.fold + 1)),
                  len(self.inputs))
        self.train = self.inputs[:begin] + self.inputs[end:]
        self.test =  self.inputs[begin:end]

        c.param_type = self.trait('inputs').inner_traits[0].trait_type.__class__.__name__
        return c

        c.add_trait('param_type', traits.Str('Str'))
        return c

        t = None
        node = CrossValidationFoldNode(pipeline, name, input_type=t)

        from soma_workflow.custom_jobs \
            import CrossValidationFoldJob
        if param_dict is None:
            param_dict = {}
        else:
            param_dict = dict(param_dict)
        param_dict['inputs'] = self.inputs
        param_dict['train'] = self.train
        param_dict['test'] = self.test
        param_dict['nfolds'] = self.nfolds
        param_dict['fold'] = self.fold
        job = CrossValidationFoldJob(
            name=name,
            referenced_input_files=referenced_input_files,
            referenced_output_files=referenced_output_files,
            param_dict=param_dict)
        return job

'''
:class:`LeaveOneOutNode`
'''

import sys

class LeaveOneOutNode(Node):
    '''
    This "inert" node excludes one input from the list of inputs, to allow
    leave-one-out applications.
    The "outputs" may be either an output trait (to serve as inputs to
    other nodes), or an input trait (to assign output values to other nodes).
    '''

    _doc_path = 'api/pipeline.html#leaveoneoutnode'

    def __init__(self, pipeline, name, is_output=True, input_type=None,
                 test_is_output=True, has_index=True):
        self.has_index = has_index
        in_traitsl = ['inputs']
        if has_index:
            in_traitsl.append('index')
        if is_output:
            out_traitsl = ['train']
        else:
            out_traitsl = []
            in_traitsl.append('train')
        if test_is_output:
            out_traitsl.append('test')
        else:
            in_traitsl.append('test')
        super(LeaveOneOutNode, self).__init__(pipeline, name, in_traits,
                                              out_traits)
        else:

            self.add_trait('index', traits.Int(0))
        self.add_trait('test', ptype)
        self.trait('test').output = test_is_output
        self.trait('train').inner_traits[0].output = is_output

        inputs = ['inputs', 'test']
        if self.has_index:
            inputs.append('index')
            update_callback = self.exclude_callback
        self.on_trait_change(update_callback, inputs)

    def exclude_callback(self, name, value):
        if not self.has_index:
            try:
                index = self.inputs.index(self.test)
            except Exception:
                return
        else:
            index = self.index

        result = [x for i, x in enumerate(self.inputs) if i != index]
        self.train =  result
        if self.has_index and index < len(self.inputs):
            self.test = self.inputs[index]

        c.param_type = \
            self.trait('inputs').inner_traits[0].trait_type.__class__.__name__
        c.is_output = self.trait('train').output
        c.test_is_output = self.trait('test').output
        c.has_index = self.has_index
        return c

        c.add_trait('is_output', traits.Bool(True))
        c.add_trait('test_is_output', traits.Bool(True))
        c.add_trait('has_index', traits.Bool(True))
        return c

        t = None
        node = LeaveOneOutNode(pipeline, name, conf_controller.is_output,
                               input_type=t, test_is_output=conf_controller.test_is_output,
                               has_index=conf_controller.has_index)

        from soma_workflow.custom_jobs import LeaveOneOutJob
        index = 0
        else:
            try:
                pass
        param_dict = dict(param_dict)
        param_dict['index'] = index
        job = LeaveOneOutJob(name=name,
                             referenced_input_files=referenced_input_files,
                             referenced_output_files=referenced_output_files,
                             param_dict=param_dict)
        return job

'''
:class:`MapNode`
'''

from capsul.pipeline.pipeline_nodes import Node, Plug
import sys

class MapNode(Node):
    '''
    This node converts lists into series of single items. Typically an
    input named ``inputs`` is a list of items. The node will separate items and
    output each of them as an output parameter. The i-th item will be output as
    ``output_<i>`` by default.
    The inputs / outputs names can be customized using the constructor
    parameters ``input_names`` and ``output_names``. Several lists can be split
    in the same node.
    The node will also output a ``lengths`` parameter which will contain the
    input lists lengths. This lengths can typically be input in reduce nodes to
    perform the reverse operation (see
    :class:`~capsul.pipeline.custom_nodes.reduce_node.ReduceNode`).

    * ``input_names`` is a list of input parameters names, each being a list to
      be split. The default is ``['inputs']``
    * ``output_names`` is a list of patterns used to build output parameters
      names. Each item is a string containing a substitution pattern ``"%d"``
      which will be replaced with a number. The default is ``['output_%d']``.
      Each pattern will be used to replace items from the corresponding input
      in the same order. Thus ``input_names``  and ``output_names`` should be
      the same length.
    '''

    _doc_path = 'api/pipeline.html#mapnode'

    def __init__(self, pipeline, name, input_names=['inputs'],
                 output_names=['output_%d'], input_types=None):
        out_traits = [{'name': 'lengths', 'optional': True}]

        if input_types:
            ptypes = input_types
        else:
            ptypes = [traits.File(traits.Undefined, output=False)] \
                * len(input_names)
        self.input_types  = ptypes

        for tr in input_names:
            in_traits.append({'name': tr, 'optional': False})
        super(MapNode, self).__init__(pipeline, name, in_traits, out_traits)

        for tr, ptype in zip(input_names, ptypes):
            self.add_trait(tr, traits.List(ptype, output=False))
        self.add_trait('lengths',
                       traits.List(traits.Int(), output=True, optional=True,
                                   desc='lists lengths'))
        self.input_names = input_names
        self.output_names = output_names
        self.lengths = [0] * len(input_names)

    def set_callbacks(self):
        self.on_trait_change(self.map_callback, self.input_names)

    def map_callback(self, obj, name, old_value, value):
        index = self.input_names.index(name)
        output = self.output_names[index]
        ptype = self.input_types[index]
        if old_value in (None, traits.Undefined):
            old_value = []
        if value in (None, traits.Undefined):
            value = []
        if len(old_value) > len(value):
            for i in range(len(old_value) - 1, len(value) - 1, -1):
                pname = output % i
                self.remove_trait(pname)
                if pname in self.plugs:
                    # remove links to this plug
                    plug = self.plugs[pname]
                    to_del = []
                    for link in plug.links_to:
                        linkd = (self, pname, link[2], link[1])
                        to_del.append(linkd)
                    for linkd in to_del:
                        self.pipeline.remove_link(linkd)
                    del self.plugs[pname]
        for i in range(len(old_value), len(value)):
            pname = output % i
            ptype = self._clone_trait(ptype,
                                      {'output': True, 'optional': True})
            self.add_trait(pname, ptype)
            plug = Plug(name=output % i, optional=True, output=True)
            self.plugs[pname] = plug
            plug.on_trait_change(
                self.pipeline.update_nodes_and_plugs_activation, "enabled")
        for i, val in enumerate(value):
            setattr(self, output % i, val)
        # update lengths
        lengths = self.lengths
        if not isinstance(lengths, list):
            lengths = []
        while len(lengths) <= index:
            lengths.append(0)
        lengths[index] = len(value)
        self.lengths = lengths

        c.input_names = self.input_names
        c.output_names = self.output_names
        c.input_types = [p.trait_type.__class__.__name__
                         for p in self.input_types]
        return c

        c.add_trait('input_types', traits.List(traits.Str))
        c.add_trait('input_names', traits.List(traits.Str))
        c.add_trait('output_names', traits.List(traits.Str))
        c.input_names = ['inputs']
        c.output_names = ['output_%d']
        c.input_types = ['File']
        return c

        t = []
        for ptype in conf_controller.input_types:
            if ptype == 'Str':
                t.append(traits.Str(traits.Undefined))
            elif ptype == 'File':
                t.append(traits.File(traits.Undefined))
            elif ptype not in (None, traits.Undefined):
                t.append(getattr(traits, ptype)())
        node = MapNode(pipeline, name, conf_controller.input_names,
                       conf_controller.output_names, input_types=t)

        from soma_workflow.custom_jobs import MapJob
        param_dict['input_names'] = self.input_names
        param_dict['output_names'] = self.output_names
        for index, pname in enumerate(self.input_names):
            value = getattr(self, pname)
            param_dict[pname] = value
            output_name = self.output_names[index]
            if value not in (None, traits.Undefined):
                for i in range(len(value)):
                    opname = output_name % i
                    param_dict[opname] = getattr(self, opname)
        job = MapJob(name=name,
                     referenced_input_files=referenced_input_files,
                     referenced_output_files=referenced_output_files,
                     param_dict=param_dict)
        return job

'''
:class:`ReduceNode`
'''

import sys

class ReduceNode(Node):
    '''
    Reduce node: converts series of inputs into lists. Typically a series of
    inputs named ``input_0`` .. ``input_<n>`` will be output as a single list
    named ``outputs``.

    Several input series can be handled by the node, and input names can be
    customized.

    * The numbers of inputs for each series is given as the ``lengths`` input
      parameter. It is typically linked from the output of a
      :class:`~capsul.pipeline.custom_nodes.map_node.MapNode`.
    * Input parameters names patterns are given as the ``input_names``
      parameter. It is a list of patterns, each containing a ``"%d"`` pattern
      for the input number. The default value is ``['input_%d']``.
    * Output parameters names are given as the ``output_names`` parameter. The
      default is ``['outputs']``.

    '''

    _doc_path = 'api/pipeline.html#reducenode'

    def __init__(self, pipeline, name, input_names=['input_%d'],
                 output_names=['outputs'], input_types=None):
        in_traits = [{'name': 'lengths', 'optional': True},
                     {'name': 'skip_empty', 'optional': True}]

        else:
        self.input_types = ptypes

        for tr in output_names:
            out_traits.append({'name': tr, 'optional': False})
        super(ReduceNode, self).__init__(pipeline, name, in_traits, out_traits)

        for tr, ptype in zip(output_names, ptypes):
            self.add_trait(tr,
                           traits.List(traits.Either(ptype, traits.Undefined),
                                       output=True))
        self.add_trait('lengths', traits.List(traits.Int(), output=False,
                                              desc='lists lengths'))
        self.add_trait('skip_empty',
                       traits.Bool(
                          False, output=False,
                          desc='remove empty (Undefined, None, empty strings) '
                          'from the output lists'))

        self.lengths = [1] * len(input_names)

        self.on_trait_change(self.resize_callback, 'lengths')
        self.on_trait_change(self.reduce_callback, 'skip_empty')

    def resize_callback(self, obj, name, old_value, value):
            old_value = [0] * len(self.input_names)
            value = [0] * len(self.input_names)

        # remove former callback
        inputs = []
        for i, pname_p in enumerate(self.input_names):
            inputs += [pname_p % j for j in range(old_value[i])]
        self.on_trait_change(self.reduce_callback, inputs, remove=True)

        # adjust sizes
        for in_index in range(len(value)):
            ptype = self.input_types[in_index]
            pname_p = self.input_names[in_index]
            oval = old_value[in_index]
            val = value[in_index]
            if oval > val:
                for i in range(oval - 1, val - 1, -1):
                    pname = pname_p % i
                    self.remove_trait(pname)
                    if pname in self.plugs:
                        # remove links to this plug
                        plug = self.plugs[pname]
                        to_del = []
                        for link in plug.links_from:
                            linkd = (link[2], link[1], self, pname)
                            to_del.append(linkd)
                        for linkd in to_del:
                            self.pipeline.remove_link(linkd)
            for i in range(oval, val):
                pname =  pname_p % i
                ptype2 = self._clone_trait(
                    ptype, {'output': False, 'optional': False})
                self.add_trait(pname, ptype2)
                plug = Plug(name=pname, optional=False, output=False)
                self.plugs[pname] = plug
                    self.pipeline.update_nodes_and_plugs_activation, "enabled")
            if oval != val:
                ovalue = [getattr(self, pname_p % i) for i in range(val)]
                if isinstance(ptype,
                              (traits.Str, traits.File, traits.Directory)):
                    # List trait doesn't accept Undefined as items
                    ovalue = [v # if v not in (None, traits.Undefined) else ''
                              for v in ovalue]
                setattr(self, self.output_names[in_index], ovalue)

        # setup new callback
        inputs = []
            inputs += [pname_p % j for j in range(value[i])]
        self.on_trait_change(self.reduce_callback, inputs)

    def reduce_callback(self, obj, name, old_value, value):
        # find out which input pattern is used
        in_index = None
        for index, pname_p in enumerate(self.input_names):
            for i in range(self.lengths[index]):
                pname = pname_p % i
                if name == pname:
                    in_index = index
                    break
            if in_index is not None:
                break
        if in_index is None:
            in_indices = range(len(self.input_names))
        else:
            in_indices = [in_index]
        for in_index in in_indices:
            output = self.output_names[in_index]
            value = [getattr(self, pname_p % i)
                    for i in range(self.lengths[in_index])]
            if self.skip_empty:
                value = [v for v in value
                         if v not in (None, traits.Undefined, '')]
            elif isinstance(self.input_types[in_index],
                          (traits.Str, traits.File, traits.Directory)):
                # List trait doesn't accept Undefined as items
                value = [v # if v not in (None, traits.Undefined) else traits.Undefined
                         for v in value]
            setattr(self, output, value)

        c.input_types = [(p.trait_type.__class__.__name__ if p.trait_type
                            else p.__class__.__name__)
        return c

        c.input_names = ['input_%d']
        c.output_names = ['outputs']
        return c

        t = []
            elif ptype not in (None, traits.Undefined, 'None', 'NoneType',
                               '<undefined>'):
        node = ReduceNode(pipeline, name, conf_controller.input_names,
                          conf_controller.output_names, input_types=t)

        param_dict['lengths'] = self.lengths
                param_dict[pname] = getattr(self, pname)
            value = getattr(self, output_name)
                param_dict[output_name] = value
        return job

''' Process main class and infrastructure

Classes
-------

.. currentmodule:: capsul.process.process

:class:`Process`
++++++++++++++++
:class:`FileCopyProcess`
++++++++++++++++++++++++
:class:`InteractiveProcess`
+++++++++++++++++++++++++++
:class:`NipypeProcess`
++++++++++++++++++++++
:class:`ProcessMeta`
++++++++++++++++++++
:class:`ProcessResult`

'''

import os
import operator
from socket import getfqdn
from datetime import datetime as datetime
import six
import sys
import functools
import traceback

from traits.api import Directory, Undefined, Int, List, Bool, File
from traits.api import BaseTraitHandler

from soma.controller import trait_ids
from soma.controller.trait_utils import is_trait_value_defined
from soma.controller.trait_utils import get_trait_desc
from soma.utils import json_utils

from capsul.utils.version_utils import get_tool_version

class ProcessMeta(Controller.__class__):
    """ Class used to complete a process docstring

    Use a class and not a function for inheritance.
    """
    def complement_doc(name, docstr):
        """ complement the process docstring
        """
        docstring = docstr.split("\n")

        # we have to indent the note properly so that the docstring is
        # properly displayed, and correctly processed by sphinx
        indent = -1
        for line in docstring[1:]:
            lstrip = line.strip()
            if not lstrip:  # empty lines do not influence indent
                continue
            lindent = line.index(line.strip())
            if indent == -1 or lindent < indent:
                indent = lindent
        if indent < 0:
            indent = 0

        # Complete the docstring
        docstring += [' ' * indent + line for line in [
            "",
            ".. note::",
            "",
            "    * Type '{0}.help()' for a full description of "
            "this process parameters.".format(name),
            "    * Type '<{0}>.get_input_spec()' for a full description of "
            "this process input trait types.".format(name),
            "    * Type '<{0}>.get_output_spec()' for a full description of "
            "this process output trait types.".format(name),
            ""
        ]]

        return "\n".join(docstring)

    def __new__(mcls, name, bases, attrs):
        """ Method to print the full help.

        mcls: meta class (mandatory)
            a meta class.
            the process class name.
        bases: tuple (mandatory)
            the direct base classes.
        attrs: dict (mandatory)
            a dictionary with the class attributes.
        """
        # Update the class docstring with the full process help
            name, attrs.get("__doc__", ""))
        attrs["__doc__"] = docstring

        # Find all traits definitions in the process class and ensure that
        # it has a boolean value for attributes "output" and "optional".
        # If no value is given at construction, False will be used.
        for n, possible_trait_definition in six.iteritems(attrs):
            if isinstance(possible_trait_definition, BaseTraitHandler):
                possible_trait_definition._metadata['output'] \
                    = bool(possible_trait_definition.output)
                possible_trait_definition._metadata['optional'] \
                    = bool(possible_trait_definition.optional)

        return super(ProcessMeta, mcls).__new__(
            mcls, name, bases, attrs)

class Process(six.with_metaclass(ProcessMeta, Controller)):
    """ A process is an atomic component that contains a processing.

    A process is typically an object with typed parameters, and an execution
    function. Parameters are described using Enthought
    `traits <http://docs.enthought.com/traits/>`_ through Soma-Base
    :somabase:`Controller <api.html#soma.controller.controller.Controller>`
    base class.

    In addition to describing its parameters, a Process must implement its
    execution function, either through a python method, by overloading
    :meth:`_run_process`, or through a commandline execution, by overloading
    :meth:`get_commandline`. The second way allows to run on a remote
    processing machine which has not necessary capsul, nor python, installed.

    Parameters are declared or queried using the traits API, and their values
    are in the process instance variables:

    ::

        from __future__ import print_function
        from capsul.api import Process
        import traits.api as traits

        class MyProcess(Process):

            # a class trait
            param1 = traits.Str('def_param1')

            def __init__(self):
                super(MyProcess, self).__init__()
                # declare an input param
                self.add_trait('param2', traits.Int())
                # declare an output param
                self.add_trait('out_param', traits.File(output=True))

            def _run_process(self):
                with open(self.out_param, 'w') as f:
                    print('param1:', self.param1, file=f)
                    print('param2:', self.param2, file=f)

        # run it with parameters
        MyProcess()(param2=12, out_param='/tmp/log.txt')

    **Note about the File and Directory traits**

    The :class:`~traits.trait_types.File` trait type represents a file
    parameter. A file is actually two things: a filename (string), and the file itself (on the filesystem). For an input it is OK not to distinguish them, but for an output, there are two different cases:

    * the file (on the filesystem) is an output, but the filename (string) is
      given as an input: this is the classical "commandline" behavior, when we
      tell the program where it should write its output file.
    * the file is an output, and the filename is also an output: this is rather a
      "function return value" behavior: the process determines internally where
      it should write the file, and tells as an output where it did.

    To distinguish these two cases, in Capsul we normally add in the
    :class:`~traits.trait_types.File` or :class:`~traits.trait_types.Directory`
    trait a property ``input_filename`` which is True when the filename is an
    input, and False when the filename is an output::

        self.add_trait('out_file',
                       traits.File(output=True, input_filename=False))

    However, as most of our processes are based on the "commandline behavior"
    (the filename is an input) and we often forget to specify the
    ``input_filename`` parameter, the default is the "filename is an input"
    behavior, when not specified.

    **Attributes**

    name: str
        the class name.
    id: str
        the string description of the class location (ie., module.class).
    log_file: str (default None)
        if None, the log will be generated in the current directory
        otherwise it will be written in log_file path.
    """

        """ Initialize the Process class.
        """
        super(Process, self).__init__()

        # Initialize the process identifiers
        self.name = self.__class__.__name__
        self.id = self.__class__.__module__ + "." + self.name

        # Parameter to store which tools will be used dusring the processing
        self.versions = {
            "capsul": get_tool_version("capsul")
        }

        # Initialize the log file name
        self.log_file = None
        self.study_config = None

        default_values = getattr(self, 'default_values', None)
        if default_values:
            self.default_values = default_values.copy()
        else:
            self.default_values = {}
        for k, v in six.iteritems(kwargs):
            self.default_values[k] = v

        """ Remove the _weakref attribute eventually set by 
        soma.utils.weak_proxy because it prevents Process instance
        """
        state = super(Process, self).__getstate__()
        state.pop('_user_traits', None)
        state.pop('__doc__', None)
        state.pop('study_config', None)
    
        """Ensure that trait.output and trait.optional are set to a
        boolean value before calling parent class add_trait.
        """
        if not "_metadata" in trait.__dict__:
            trait._metadata = {}
        if trait._metadata is not None:
            trait._metadata['output'] = bool(trait.output)
            trait._metadata['optional'] = bool(trait.optional)
        else:
            trait.output = bool(trait.output)
            trait.optional = bool(trait.optional)
        super(Process, self).add_trait(name, trait)
        
    def __call__(self, **kwargs):
        """ Method to execute the Process.

        Keyword arguments may be passed to set process parameters.
        This in turn will allow calling the process like a standard
        python function.
        In such case keyword arguments are set in the process in
        addition to those already set before the call.

        Raise a TypeError if a keyword argument do not match with a
        process trait name.

        .. note:

            This method must not modify the class attributes in order
            to be able to perform smart caching.

        .. note:

            This method should **not** be overloaded by Process subclasses to
            perform actual processing. Instead, either the
            :meth:`_run_process` method or the :meth:`get_commandline` method
            should be overloaded.

        kwargs: dict (optional)
            should correspond to the declared parameter traits.

        Returns
        -------
        results:  ProcessResult object
            contains all execution information.
        """
        # Execute the process
        returncode = self.get_study_config().run(self, **kwargs)
        return returncode

    def run(self, **kwargs):
        '''
        Obsolete: use self.__call__ instead
        '''
        return self.__call__(**kwargs)

    ####################################################################
    # Private methods

        """Runs the processings when the instance is called.

        Either this _run_process() or :meth:`get_commandline` must be
        defined in derived classes.

        Note that _run_process() is called as a python function, on a Process
        instance. When using remote processing (cluster for instance), this
        means that the commandline which will run needs to be able to re-
        instantiate the same process: the process thus has to be stored in a
        file or python module which can be accessed from the remote machine,
        and python / capsul correctly installed and available on it.

        :meth:`get_commandline` at the contrary, can implement commandlines
        which are completely independent from Capsul, and from python.

        .. note::

            If both methods are not defined in the derived class a
            NotImplementedError error is raised.

            On the other side, if both methods are overloaded, the process
            behavior in local sequential computing mode and in Soma-Workflow
            modes may be different.
        """
        # Check if get_commandline() method is specialized
        # If yes, we can make use of it to execute the process
        if self.__class__.get_commandline != Process.get_commandline:
            commandline = self.get_commandline()
            soma.subprocess.check_call(commandline)

        # Otherwise raise an error
        else:
            raise NotImplementedError(
                "Either get_commandline() or _run_process() should be "
                "redefined in process ({0})".format(
                    self.__class__.__name__))
    
    def _before_run_process(self):
        """This method is called by StudyConfig.run() before calling
        _run_process(). By default, it does nothing but can be overridden
        in derived classes.
        """
        pass

    def _after_run_process(self, run_process_result):
        """This method is called by StudyConfig.run() after calling
        _run_process(). It is expected to return the final result of the
        process. By default, it does nothing but can be overridden
        """
        return run_process_result
        
    def _get_log(self, exec_info):
        """ Method that generate the logging structure from the execution
        information and class attributes.

        exec_info: dict (mandatory)
            the execution information,
            the dictionary is supposed to contain a runtime attribute.

        Returns
        -------
        log: dict
            the logging information.
        """
        # Set all the execution runtime information in the log
        log = exec_info.runtime

        # Add the process identifiaction class attribute
        log["process"] = self.id

        # Add the process inputs and outputs
        log["inputs"] = exec_info.inputs.copy()
        log["outputs"] = exec_info.outputs.copy()

        # Need to take the representation of undefined input or outputs
        # traits
        for parameter_type in ["inputs", "outputs"]:
            for key, value in six.iteritems(log[parameter_type]):
                    log[parameter_type][key] = repr(value)

        return log

    def _rst_table(self, data):
        """ Create a rst formatted table.

        data: list of lists of str (mandatory)
            the table line-cell centent.

        Returns
        -------
        rsttable: list of str
            the rst formatted table containing the input data.
        """
        # Output rst table
        rsttable = []

        for table_row in data:
            for index, cell_row in enumerate(table_row):
                # > set the parameter name in bold
                if index == 0 and ":" in cell_row:
                    delimiter_index = cell_row.index(":")
                    cell_row = ("**" + cell_row[:delimiter_index] + "**" +
                                cell_row[delimiter_index:])
                rsttable.append(cell_row)

        ## Get the size of the largest row in order to
        ## format properly the rst table (do not forget the '+' and '*')
        #row_widths = [len(item)
                      #for item in functools.reduce(operator.add, data)]
        #width = max(row_widths) + 11

        ## Generate the rst table

        ## > table synthax
        #rsttable.append("+" + "-" * width + "+")
        ## > go through the table lines
        #for table_row in data:
            ## > go through the cell lines
            #for index, cell_row in enumerate(table_row):
                ## > set the parameter name in bold
                #if index == 0 and ":" in cell_row:
                    #delimiter_index = cell_row.index(":")
                    #cell_row = ("**" + cell_row[:delimiter_index] + "**" +
                                #cell_row[delimiter_index:])
                ## >  add table rst content
                #rsttable.append(
                    #"| | {0}".format(cell_row) +
                    #" " * (width - len(cell_row) - 3) +
                    #"|")
            ## > close cell
            #rsttable.append("+" + "-" * width + "+")

        return rsttable

    # Public methods

    def save_log(self, returncode):
        """ Method to save process execution information in json format.

        If the class attribute `log_file` is not set, a log.json output
        file is generated in the process call current working directory.

        returncode: ProcessResult
            the process result return code.
        """
        # Build the logging information
        exec_info = self._get_log(returncode)

        # Generate an output log file name if necessary
        if not self.log_file:
            self.log_file = os.path.join(exec_info["cwd"], "log.json")

        # Dump the log
        json_struct = json.dumps(exec_info, sort_keys=True,
                                 check_circular=True, indent=4)

        # Save the json structure
        with open(self.log_file, "w") as f:
            f.write(six.text_type(json_struct))

    def help(cls, returnhelp=False):

        cls: process class (mandatory)
            a process class
        returnhelp: bool (optional, default False)
            if True return the help string message,
            otherwise display it on the console.
        """
        cls_instance = cls()
        return cls_instance.get_help(returnhelp)

    # Accessors

    def get_commandline(self):
        """ Method to generate a commandline representation of the process.

        If not implemented, it will generate a commandline running python,
        instantiating the current process, and calling its
        :meth:`_run_process` method.

        Returns
        -------
        commandline: list of strings
            Arguments are in separate elements of the list.
        """
        # Get command line arguments (ie., the process user traits)
        # Build the python call expression, keeping apart file names.
        # File names are given separately since they might be modified
        # externally afterwards, typically to handle temporary files, or
        # file transfers with Soma-Workflow.

        class ArgPicker(object):
            """ This small object is only here to have a __repr__() representation which will print sys.argv[n] in a list when writing the commandline code.
            """
            def __init__(self, num):
                self.num = num
            def __repr__(self):
                return 'sys.argv[%d]' % self.num

        reserved_params = ("nodes_activation", "selection_changed")
        # pathslist is for files referenced from lists: a list of files will
        # look like [sys.argv[5], sys.argv[6]...], then the corresponding
        # path args will be in additional arguments, here stored in pathslist
        pathslist = []
        # argsdict is the dict of non-path arguments, and will be printed
        # using repr()
        argsdict = {}
        # pathsdict is the dict of path arguments, and will be printed as a
        # series of arg_name, path_value, all in separate commandline arguments
        pathsdict = {}

        for trait_name, trait in six.iteritems(self.user_traits()):
            value = getattr(self, trait_name)
            if trait_name in reserved_params \
                    or not is_trait_value_defined(value):
                continue
            if is_trait_pathname(trait):
                pathsdict[trait_name] = value
            elif isinstance(trait.trait_type, List) \
                    and is_trait_pathname(trait.inner_traits[0]):
                plist = []
                for pathname in value:
                    if is_trait_value_defined(pathname):
                        plist.append(ArgPicker(len(pathslist) + 1))
                        pathslist.append(pathname)
                    else:
                        plist.append(pathname)
                argsdict[trait_name] = plist
            else:
                argsdict[trait_name] = value

        # Get the module and class names
        if hasattr(self, '_function'):
            # function with xml decorator
            module_name = self._function.__module__
            class_name = self._function.__name__
            call_name = class_name
        else:
            module_name = self.__class__.__module__
            class_name = self.name
            call_name = '%s()' % class_name

        # Construct the command line
        python_command = os.path.basename(sys.executable)
        commandline = [
            python_command,
            "-c",
            ("import sys; from {0} import {1}; kwargs={2}; "
             "kwargs.update(dict((sys.argv[i * 2 + {3}], "
             "sys.argv[i * 2 + {4}]) "
             "for i in range(int((len(sys.argv) - {3}) / 2)))); "
             "{5}(**kwargs)").format(module_name, class_name,
                                       repr(argsdict), len(pathslist) + 1,
                                       len(pathslist) + 2,
                                       call_name).replace("'", '"')
        ] + pathslist + sum([list(x) for x in pathsdict.items()], [])

        return commandline

        '''
        Generates a commandline representation of the process.

        This method is new in Capsul v3 and is a replacement for
        :meth:`get_commandline`.

        It can be overwritten by custom Process subclasses. Actually each
        process should overwrite either :meth:`params_to_command` or
        :meth:`_run_process`.

        The returned commandline is a list, which first element is a "method",
        and others are the actual commandline with arguments. There are several
        methods, the process is free to use either of the supported ones,
        depending on how the execution is implemented.

        **Methods:**

        `capsul_job`: Capsul process run in python
            The command will run the :meth:`_run_process` execution method of
            the process, after loading input parameters from a JSON dictionary
            file. The only second element in the commandline list is the
            process identifier (module/class as in
            :meth:`~capsul.engine.CapsulEngine.get_process_instance`). The
            location of the JSON file will be passed to the job execution
            through an environment variable `SOMAWF_INPUT_PARAMS`::

                return ['capsul_job', 'morphologist.capsul.morphologist']

        `format_string`: free commandline with replacements for parameters
            Command arguments can be, or contain, format strings in the shape
            `'%(param)s'`, where `param` is a parameter of the process. This
            way we can map values correctly, and call a foreign command::

                return ['format_string', 'ls', '%(input_dir)s']

        `json_job`: free commandline with JSON file for input parameters
            A bit like `capsul_job` but without the automatic wrapper::

                return ['json_job', 'python', '-m', 'my_module']

        Returns
        -------
        '''
            # get_commandline is overridden the old way: use it.
            return ['format_string'] + self.get_commandline()
        return ['capsul_job', self.id]

    def make_commandline_argument(self, *args):
        """This helper function may be used to build non-trivial commandline
        arguments in get_commandline implementations.
        Basically it concatenates arguments, but it also takes care of keeping
        track of temporary file objects (if any), and converts non-string
        arguments to strings (using repr()).

        Ex:

        >>> process.make_commandline_argument('param=', self.param)

        will return the same as:

        >>> 'param=' + self.param

        if self.param is a string (file name) or a temporary path.
        """
        built_arg = ""
        temp = None
        for arg in args:
            if hasattr(arg, 'pattern'): # tempfile
                built_arg = built_arg + arg
            elif isinstance(arg, six.string_types):
                built_arg += arg
            else:
                built_arg = built_arg + repr(arg)
        return built_arg

    def run_from_commandline(process_definition):
        '''
        Run a process from a commandline call. The process name (with module)
        are given in argument, input parameters should be passed through a JSON
        file which location is in the ``SOMAWF_INPUT_PARAMS`` environment
        variable.

        If the process has outputs, the ``SOMAWF_OUTUT_PARAMS`` environment
        variable should contain the location of an output file which will be
        written with a dict containing output parameters values.
        '''

        param_file = os.environ.get('SOMAWF_INPUT_PARAMS')

        # fix expandvars problem when the env var SOMAWF_OUTPUT_PARAMS is
        # defined from a script and passed into a container (like bv set
        # through soma-workflow config using "$SOMAWF_OUTPUT_PARAMS"): when the
        # "source" variable is not set, os.expandvars() leaves the value
        # "$SOMAWF_OUTPUT_PARAMS" untouched, but here we would expect an empty
        # variable
        if param_file in ('$SOMAWF_INPUT_PARAMS', '${SOMAWF_INPUT_PARAMS}'):
            param_file = None

        if not param_file:
            print('Warning: no input parameters, the env variable '
                  'SOMAWF_INPUT_PARAMS is not set.', file=sys.stderr)
            params_conf = {}
        else:
            with open(param_file) as f:
                params_conf = json_utils.from_json(json.load(f))

        configuration = params_conf.get('configuration_dict')
        if configuration:
            # activation will be re-done during run() but some global configs
            # (nipype SPM/Matlab settings) need to be done before any process
            # is instantiated, so we must do it earlier, right now.

            # clear activations for now.
            from capsul import engine
            engine.activated_modules = set()
            engine.activate_configuration(configuration)

        params = params_conf.get('parameters', {})
        ## filter out undefined values -- maybe this is not OK in all cases:
        ## we may want to manually reset a parameter, but in normal cases,
        ## Undefined values are just not set, which means that the values are
        ## left to defaults depending on the global config: nipype works like
        ## this for matlab parameters.
        #params = dict([(k, v) for k, v in params.items()
                       #if v is not Undefined])

        process = ce.get_process_instance(process_definition)
        try:
            process.import_from_dict(params)
        except Exception as e:
            print('error in setting parameters of process %s, with dict:'
                  % process.name, params, file=sys.stderr)
            raise
        # actually run the process
        ce.study_config.use_soma_workflow = False
        cwd = os.getcwd()
        with tempfile.TemporaryDirectory() as temp_work_dir:
            os.chdir(temp_work_dir)
            result = ce.study_config.run(process,
                                         configuration_dict=configuration)
            os.chdir(cwd)
        # collect output parameters
        out_param_file = os.environ.get('SOMAWF_OUTPUT_PARAMS')

        # variable
        if out_param_file in ('$SOMAWF_OUTPUT_PARAMS',
                              '${SOMAWF_OUTPUT_PARAMS}'):
            out_param_file = None

        output_params = {}
        if out_param_file:
            if result is None:
                result = {}
            reserved_params = ("nodes_activation", "selection_changed")
                if param in reserved_params or not trait.output:
                    continue
                if isinstance(trait.trait_type, (File, Directory)) \
                        and trait.input_filename is not False:
                    continue
                elif isinstance(trait.trait_type, List) \
                        and isinstance(trait.inner_traits[0].trait_type,
                                       (File, Directory)) \
                        and trait.inner_traits[0].trait_type.input_filename \
                            is not False \
                    continue
                output_params[param] = getattr(process, param)
            output_params.update(result)
            with open(out_param_file, 'w') as f:
                json.dump(json_utils.to_json(output_params), f)

        # if there was no exception, we assume the process has succeeded.
        # sys.exit(0)
        # no error, do a dirty exit, but avoid cleanup crashes after the
        # process has succeeded...
        os._exit(0)

    def get_log(self):
        """ Load the logging file.

        .. note:

            If no log file found, return None

        Returns
        -------
        log: dict
            the content of the log file.
        """
        if os.path.isfile(self.log_file):
            with open(self.log_file) as json_file:
                return json.load(json_file)
        else:

    def get_input_spec(self):
        """ Method to access the process input specifications.

        Returns
        -------
        outputs: str
            a string representation of all the input trait specifications.
        """
        output = "\nINPUT SPECIFICATIONS\n\n"
        # self.traits(output=False) skips params with no output property
                output += "{0}: {1}\n".format(
                    trait_name, trait_ids(self.trait(trait_name)))
        return output

    def get_output_spec(self):
        """ Method to access the process output specifications.

        Returns
        -------
            a string representation of all the output trait specifications.
        """
        output = "\nOUTPUT SPECIFICATIONS\n\n"
        for trait_name, trait in six.iteritems(self.traits(output=True)):
            output += "{0}: {1}\n".format(
                trait_name, trait_ids(self.trait(trait_name)))

    def get_inputs(self):
        """ Method to access the process inputs.

        Returns
        -------
        outputs: dict
            a dictionary with all the input trait names and values.
        """
        output = {}
                output[trait_name] = getattr(self, trait_name)

    def get_outputs(self):
        """ Method to access the process outputs.

        Returns
        -------
            a dictionary with all the output trait names and values.
        """
        output = {}
            output[trait_name] = getattr(self, trait_name)

    def get_help(self, returnhelp=False, use_labels=False):
        """ Generate description of a process parameters.

            if True return the help string message formatted in rst,
            otherwise display the raw help string message on the console.
        use_labels: bool
            if True, input and output sections will get a RestructuredText
            label to avoid ambiguities.
        """
        # Create the help content variable
        doctring = [""]

        # Update the documentation with a description of the pipeline
        # when the xml to pipeline wrapper has been used
        if returnhelp and hasattr(self, "_pipeline_desc"):
            str_desc = "".join(["    {0}".format(line)
                                for line in self._pipeline_desc])
            doctring += [
                ".. hidden-code-block:: python",
                "    :starthidden: True",
                "",
                str_desc,
                ""
            ]

        # Get the process docstring
        if self.__doc__:
            doctring += self.__doc__.split("\n") + [""]

        # Update the documentation with a reference on the source function
        # when the function to process wrapper has been used
        if hasattr(self, "_func_name") and hasattr(self, "_func_module"):
                "This process has been wrapped from {0}.{1}.".format(
                    self._func_module, self._func_name),
                ""
            ]
            if returnhelp:
                doctring += [
                    ".. currentmodule:: {0}".format(self._func_module),
                    "",
                    ".. autosummary::",
                    "    :toctree: ./",
                    "",
                    "    {0}".format(self._func_name),
                    ""
                ]

        # Append the input and output traits help
        if use_labels:
            in_label = ['.. _%s.%s_inputs:\n\n' % (self.__module__, self.name)]
            out_label = ['.. _%s.%s_outputs:\n\n'
                         % (self.__module__, self.name)]
        else:
            in_label = []
            out_label = []
        full_help = (doctring + in_label + self.get_input_help(returnhelp)
                     + [""] + out_label
                     + self.get_output_help(returnhelp) + [""])
        full_help = "\n".join(full_help)

        # Return the full process help
        if returnhelp:
            return full_help
        # Print the full process help
        else:
            print(full_help)

    def get_input_help(self, rst_formating=False):
        """ Generate description for process input parameters.

        rst_formating: bool (optional, default False)
            if True generate a rst table with the input descriptions.

        Returns
        -------
        helpstr: str
            the class input traits help
        """
        # Generate an input section
        helpstr = ["Inputs", "~" * 6, ""]

        # Markup to separate mandatory inputs
        manhelpstr = ["[Mandatory]", ""]

        # Get all the mandatory input traits
        mandatory_items = [x for x in six.iteritems(self.user_traits())
                           if not x[1].output and not x[1].optional]
        #mandatory_items.update(self.traits(output=None, optional=False))

        # If we have mandatory inputs, get the corresponding string
        # descriptions
        data = []
        if mandatory_items:
            for trait_name, trait in mandatory_items:
                trait_desc = get_trait_desc(trait_name, trait,
                                            use_wrap=not rst_formating)
                data.append(trait_desc)

        # If we want to format the output nicely (rst)
        if data != []:
            if rst_formating:
                manhelpstr += self._rst_table(data)
            # Otherwise
            else:
                manhelpstr += functools.reduce(operator.add, data)

        # Markup to separate optional inputs
        opthelpstr = ["", "[Optional]", ""]

        # Get all optional input traits
        optional_items = [x for x in six.iteritems(self.user_traits())
                          if not x[1].output and x[1].optional]
        #optional_items = self.traits(output=False, optional=True)
        #optional_items.update(self.traits(output=None, optional=True))

        # If we have optional inputs, get the corresponding string
        data = []
        if optional_items:
            for trait_name, trait in optional_items:
                data.append(
                    get_trait_desc(trait_name, trait,
                                   use_wrap=not rst_formating))
                if trait_name == 'pipeline_steps':
                    steps = ', '.join(
                        getattr(self, 'pipeline_steps').user_traits())
                    data[-1] += ['', '    Steps for this pipeline:',
                                 f'    {steps}', '']

                opthelpstr += self._rst_table(data)
            else:
                opthelpstr += functools.reduce(operator.add, data)

        # Add the mandatory and optional input string description if necessary
            helpstr += manhelpstr
            helpstr += opthelpstr

        return helpstr

    def get_output_help(self, rst_formating=False):
        """ Generate description for process output parameters.

        Returns
        -------
            the trait output help descriptions
        """
        # Generate an output section
        helpstr = ["Outputs", "~" * 7, ""]

        # Get all the process output traits, keep their order
        items = [(name, trait)
                 for name, trait in six.iteritems(self.user_traits())
                 if trait.output]

        # If we have no output trait, return no string description
        if not items:
            return [""]

        # If we have some outputs, get the corresponding string
        data = []
        for trait_name, trait in items:
            data.append(
                get_trait_desc(trait_name, trait, use_wrap=not rst_formating))

                helpstr += self._rst_table(data)
            else:
                helpstr += functools.reduce(operator.add, data)

    def set_parameter(self, name, value, protected=None):
        """ Method to set a process instance trait value.

        For File and Directory traits the None value is replaced by the
        special Undefined trait value.

            the trait name we want to modify
            the trait value we want to set
        """
        # The None trait value is Undefined, do the replacement

            self.protect_parameter(name, protected)
        # Set the new trait value
        setattr(self, name, value)

    def get_parameter(self, name):
        """ Method to access the value of a process instance.

        Returns
        -------
        value: object
            the trait value we want to access
        """
        return getattr(self, name)

        '''
        if self.study_config is None:
            self.set_study_config(default_study_config())

        '''
        if self.study_config is not None \
                and self.study_config is not study_config:
            raise ValueError("A StudyConfig had already been set in the "
                             "process %s. It cannot be changed afterwards."
                             % self.name)

    def get_missing_mandatory_parameters(self):
        '''
        def check_trait(trait, value):
                    if not check_trait(trait.inner_traits[j], item):
                if trait.output and trait.input_filename is False:
                    # filename is an output
                return value not in (Undefined, None, '')

        for name, trait in six.iteritems(self.user_traits()):
            if not trait.optional and trait.xor is None:
                value = self.get_parameter(name)
                if not check_trait(trait, value):
            elif not trait.optional:
                xors = [i for i in trait.xor if not self.traits()[i].optional]
                # Mutually exclusive means xors must have only one element?
                # value_xor = self.get_parameter(xors[0])
                # value = self.get_parameter(name)
                # if not check_trait(self.traits()[xors[0]], value_xor) and not check_trait(trait, value):
                #     missing.append(name)
                result_check = [check_trait(self.traits()[i],
                                            self.get_parameter(i)) for i in xors]
                result_check.append(check_trait(trait, self.get_parameter(name)))
                if not any(result_check):

        '''
        Requirements needed to run the process. It is a dictionary which keys are config/settings modules and values are requests for them.

        Ex::

        '''
        return {}

        '''

        Returns
        -------
        '''
        settings = self.get_study_config().engine.settings
        config = settings.select_configurations(environment, uses=req,
                                                check_invalid_mods=True)
        new_req = config.get('capsul_engine', {}).get('uses', {})
        for module in new_req:
            if module_name not in config and message_list is not None:
                message_list.append('requirement: %s: %s is not met in %s'
                                    % (module_name, new_req[module_name],
                                       self.name))
            elif module_name not in config:
                # if no message is expected, then we can return immediately
                # without checking further requirements. Otherwise we
                # continue to get a full list of unsatisfied requirements.
                print('requirement:', module_name, ':', new_req[module_name],
                      'not met in', self.name)
                print('config:', settings.select_configurations(environment),
                      end=2*'\n')
        else:

class FileCopyProcess(Process):
    """ A specific process that copies all the input files.

    copied_inputs : dict
        the list of copied file parameters {param: dst_value}
    copied_files: dict
        copied files {param: [dst_value1, ...]}

    Methods
    -------
    __call__
    _update_input_traits
    _get_process_arguments
    _copy_input_files
    """
    def __init__(self, activate_copy=True, inputs_to_copy=None,
                 inputs_to_clean=None, destination=None,
                 inputs_to_symlink=None, use_temp_output_dir=False):
        """ Initialize the FileCopyProcess class.

        activate_copy: bool (default True)
            if False this class is transparent and behaves as a Process class.
        inputs_to_copy: list of str (optional, default None)
            the list of inputs to copy.
            If None, all the input files are copied.
        inputs_to_clean: list of str (optional, default None)
            some copied inputs that can be deleted at the end of the
            processing. If None, all copied files will be cleaned.
        destination: str (optional default None)
            where the files are copied.
            If None, the output directory will be used, unless
            use_temp_output_dir is set.
        inputs_to_symlink: list of str (optional, default None)
            as inputs_to_copy, but for files which should be symlinked
        use_temp_output_dir: bool
            if True, the output_directory parameter is set to a temp one during
            execution, then outputs are copied / moved / hardlinked to the
            final location. This is useful when several parallel jobs are
            working in the same directory and may write the same intermediate
            files (SPM does this a lot).
        """
        super(FileCopyProcess, self).__init__()

        # Class parameters
        self.activate_copy = activate_copy
        self.destination = destination
        if self.activate_copy:
            self.inputs_to_clean = inputs_to_clean
            if inputs_to_symlink is None:
                self.inputs_to_symlink = list(self.user_traits().keys())
            else:
                self.inputs_to_symlink = inputs_to_symlink
            if inputs_to_copy is None:
                self.inputs_to_copy = [k for k in self.user_traits().keys()
                                       if k not in self.inputs_to_symlink]
            else:
                self.inputs_to_copy = inputs_to_copy
                self.inputs_to_symlink = [k for k in self.inputs_to_symlink
                                          if k not in self.inputs_to_copy]
            self.copied_inputs = None
            self.copied_files = None
        self.use_temp_output_dir = use_temp_output_dir

        """ Method to copy files before executing the process.
        """
        super(FileCopyProcess, self)._before_run_process()

        if self.destination is None:
            output_directory = getattr(self, 'output_directory', None)
            if output_directory in (None, Undefined, ''):
                output_directory = None
            if self.use_temp_output_dir:
                workspace = tempfile.mkdtemp(dir=output_directory,
                                             prefix=self.name)
                destdir = workspace
            else:
                destdir = output_directory
        else:
            destdir = self.destination
        if not destdir:
            raise ValueError('FileCopyProcess cannot be used without a '
                             'destination directory')
        self._destination = destdir
        output_directory = self.destination
        if output_directory is None:
        if output_directory not in (None, Undefined, ''):
            self._former_output_directory = output_directory
            self.output_directory = destdir

        # The copy option is activated

            # Copy the desired items
            self._update_input_traits()

            self._recorded_params = {}
            # Set the process inputs
            for name, value in six.iteritems(self.copied_inputs):
                self._recorded_params[name] = getattr(self, name)
                self.set_parameter(name, value)

        """ Method to clean-up temporary workspace after process
        """
        run_process_result = super(FileCopyProcess, self)._after_run_process(
            run_process_result)
        if self.use_temp_output_dir:
            self._move_outputs()
            # Clean the workspace
            self._clean_workspace()

        # restore initial values, keeping outputs
        # The situation here is that:
        # * output_directory should drive "final" output values
        # * we may have been using a temporary output directory, thus output
        #   values are already set to this temp dir, not the final one.
        #   (at least when use_temp_output_dir is set).
        #   -> _move_outputs() already changes these output values
        # * when we reset inputs, outputs are reset to values pointing to
        #   the input directory (via nipype callbacks).
        # So we must:
        # 1. record output values
        # 2. set again inputs to their initial values (pointing to the input
        #    directories). Outputs will be reset accordingly to input dirs.
        # 3. force output values using the recorded ones.

        outputs = {}
                outputs[name] = getattr(self, name)
        # 2. set again inputs to their initial values
        if hasattr(self, '_recorded_params'):
            for name, value in six.iteritems(self._recorded_params):
        # 3. force output values using the recorded ones
        for name, value in six.iteritems(outputs):
            self.set_parameter(name, value)
            del self._recorded_params

    def _clean_workspace(self):
        """ Removed some copied inputs that can be deleted at the end of the
        processing.
        """
        inputs_to_clean = self.inputs_to_clean
        if inputs_to_clean is None:
            # clean all copied inputs
            inputs_to_clean = list(self.copied_files.keys())
        for to_rm_name in inputs_to_clean:
            rm_files = self.copied_files.get(to_rm_name, [])
            if rm_files:
                self._rm_files(rm_files)
                del self.copied_files[to_rm_name]
                del self.copied_inputs[to_rm_name]

    def _move_outputs(self):
        tmp_output = self._destination
        dst_output = self._former_output_directory
        output_values = {}
        moved_dict = {}
                new_value = self._move_files(tmp_output, dst_output,
                                             getattr(self, param),
                                             moved_dict=moved_dict)
                output_values[param] = new_value
                self.set_parameter(param, new_value)

        shutil.rmtree(tmp_output)
        del self._destination
        self.destination = self._former_output_directory
        if hasattr(self, 'output_directory'):
            self.output_directory = self._former_output_directory
        del self._former_output_directory
        return output_values

    def _move_files(self, src_directory, dst_directory, value, moved_dict={}):
        if isinstance(value, (list, tuple)):
            new_value = [self._move_files(src_directory, dst_directory, item,
                                          moved_dict)
                         for item in value]
            if isinstance(value, tuple):
                return tuple(new_value)
            return new_value
        elif isinstance(value, dict):
            new_value = {}
            for name, item in six.iteritems(value):
                new_value[name] = self._move_files(
                    src_directory, dst_directory, item, moved_dict)
        elif isinstance(value, six.string_types):
            if value in moved_dict:
                return moved_dict[value]
            if os.path.dirname(value) == src_directory \
                    and os.path.exists(value):
                name = os.path.basename(value).split('.')[0]
                matfnames = glob.glob(os.path.join(
                    os.path.dirname(value), name + ".*"))
                todo = [x for x in matfnames if x != value]
                dst = os.path.join(dst_directory, os.path.basename(value))
                if os.path.exists(dst) or os.path.islink(dst):
                    print('warning: file or directory %s exists' % dst)
                    if os.path.isdir(dst):
                        shutil.rmtree(dst)
                    else:
                        os.unlink(dst)
                try:
                    # print('moving:', value, 'to:', dst)
                    shutil.move(value, dst)
                except Exception as e:
                    print(e, file=sys.stderr)
                moved_dict[value] = dst
                for item in todo:
                    self._move_files(src_directory, dst_directory, item,
                                     moved_dict)
                return dst

    def _rm_files(self, python_object):
        """ Remove a set of copied files from the filesystem.

        python_object: object
            a generic python object.
        """
        # Deal with dictionary
        if isinstance(python_object, dict):
            for val in python_object.values():
                self._rm_files(val)

        # Deal with tuple and list
        elif isinstance(python_object, (list, tuple)):
            for val in python_object:

        # Otherwise start the deletion if the object is a file
        else:
            if (isinstance(python_object, six.string_types) and
                    os.path.isfile(python_object)):
                os.remove(python_object)

    def _update_input_traits(self, copy=True):
        """ Update the process input traits: input files are copied.
        """
        # Get the new trait values
        input_parameters, input_symlinks = self._get_process_arguments()
        self.copied_files = {}
        self.copied_inputs \
            = self._copy_input_files(input_parameters, False,
                                     self.copied_files, copy=copy)
        self.copied_inputs.update(
            self._copy_input_files(input_symlinks, True, self.copied_files,
                                   copy=copy))

    def _copy_input_files(self, python_object, use_symlink=True,
                          files_list=None, copy=True):
        """ Recursive method that copy the input process files.

        use_symlink: bool
            if True, symbolic links will be make instead of full copies, on
            systems which support symlinks. Much faster and less disk consuming
            than full copies, but might have side effects in programs which
            detect and handle symlinks in a different way.
        files_list: list or dict
            if provided, this *output* parameter will be updated with the list
            of files actually created.
        copy: bool
            if False, files are not actually copied/symlinked but filenames are
            generated

        Returns
        -------
        out: object
            the copied-file input object.
        """
        if sys.platform.startswith('win') and sys.version_info[0] < 3:
            # on windows, no symlinks (in python2 at least).
            use_symlink = False

        # Create an output dict that will contain the copied file locations
        # and the other values
            out = {}
            for key, val in python_object.items():
                if val is not Undefined:
                    if isinstance(files_list, dict):
                        sub_files_list = files_list.setdefault(key, [])
                    else:
                        sub_files_list = files_list
                    out[key] = self._copy_input_files(val, use_symlink,
                                                      sub_files_list,
                                                      copy=copy)

        # Create an output list or tuple that will contain the copied file
        # locations and the other values
            out = []
                    out.append(self._copy_input_files(val, use_symlink,
                                                      files_list, copy=copy))
            if isinstance(python_object, tuple):
                out = tuple(out)

        # Otherwise start the copy (with metadata cp -p) if the object is
        # a file
        else:
            out = python_object
            if (python_object is not Undefined and
                    isinstance(python_object, six.string_types) and
                destdir = self._destination
                if not os.path.exists(destdir):
                    os.makedirs(destdir)
                fname = os.path.basename(python_object)
                out = os.path.join(destdir, fname)
                if out == python_object:
                    return out  # input=output, nothing to do
                if copy:
                    if os.path.exists(out) or os.path.islink(out):
                        if os.path.isdir(out):
                            shutil.rmtree(out)
                        else:
                            os.unlink(out)
                    if use_symlink:
                        os.symlink(python_object, out)
                    else:
                        shutil.copy2(python_object, out)
                    if files_list is not None:
                        files_list.append(out)

                    # Copy associated .mat/.json/.minf files
                    name = fname.rsplit(".", 1)[0]
                    matfnames = glob.glob(os.path.join(
                        os.path.dirname(python_object), name + ".*"))
                    for matfname in matfnames:
                        extrafname = os.path.basename(matfname)
                        extraout = os.path.join(destdir, extrafname)
                        if extraout != out:
                            if os.path.exists(extraout) \
                                    or os.path.islink(extraout):
                                if os.path.isdir(extraout):
                                    shutil.rmtree(extraout)
                                else:
                                    os.unlink(extraout)
                            if use_symlink:
                                os.symlink(matfname, extraout)
                            else:
                                shutil.copy2(matfname, extraout)
                            if files_list is not None:
                                files_list.append(extraout)

        return out

    def _get_process_arguments(self):
        """ Get the process arguments.

        The user process traits are accessed through the user_traits()
        method that returns a sorted dictionary.

        Returns
        -------
        input_parameters: dict
            the process input parameters that should be copied.
        input_symlinks: dict
            the process input parameters that should be symlinked.
        """
        # Store for input parameters
        input_parameters = {}
        input_symlinks = {}

        # Go through all the user traits
                continue
            # Check if the target parameter is in the check list
            c = name in self.inputs_to_copy
            s = name in self.inputs_to_symlink
            if c or s:
                # Get the trait value
                # Skip undefined trait attributes and outputs
                if value is not Undefined:
                    # Store the input parameter
                    if c:
                        input_parameters[name] = value
                    else:
                        input_symlinks[name] = value

        return input_parameters, input_symlinks

class NipypeProcess(FileCopyProcess):
    """ Base class used to wrap nipype interfaces.
    """

    def __new__(cls, *args, **kwargs):

        def init_with_skip(self, *args, **kwargs):

            cls = self.__init__.cls
            init_att = '__%s_np_init_done__' % cls.__name__
            if hasattr(self, init_att) and getattr(self, init_att):
                # may be called twice, from within __new__ or from python
                # internals
                return

            setattr(self, init_att, True)
            super(cls, self).__init__(*args, **kwargs)

        if cls.__init__ is cls.__base__.__init__ and cls is not NipypeProcess:
            # we must setup a conditional __init__ for each specialized class
            cls.__init__ = init_with_skip
            init_with_skip.cls = cls

        # determine if we were called from within nipype_factory()
        stack = traceback.extract_stack()
        # stack[-1] should be here
        # stack[-2] may be nipype_factory
        if len(stack) >= 2:
            s2 = stack[-2]
            if s2[2] == 'nipype_factory':
                instance = super(NipypeProcess, cls).__new__(cls, *args,
                                                            **kwargs)
                setattr(instance, '__%s_np_init_done__' % cls.__name__, False)
                return instance
        nipype_class = getattr(cls, '_nipype_class_type', None)
        nargs = args
        nkwargs = kwargs
        arg0 = None
        if nipype_class is not None:
            arg0 = nipype_class()
        else:
            if 'nipype_class' in kwargs:
                arg0 = kwargs['nipype_class']()
                nkwargs = {k: v for k, v in kwargs if k != 'nipype_class'}
            elif 'nipype_instance' in kwargs:
                pass
            elif len(args) != 0:
                import nipype.interfaces.base
                if isinstance(args[0], nipype.interfaces.base.BaseInterface):
                    arg0 = args[0]
                    nargs = nargs[1:]
                elif issubclass(args[0], nipype.interfaces.base.BaseInterface):
                    arg0 = args[0]()
                    nargs = args[1:]
        if arg0 is not None:
            from .nipype_process import nipype_factory
            instance = nipype_factory(arg0, base_class=cls, *nargs, **nkwargs)
            if cls != NipypeProcess:
                # override direct nipype reference
                instance.id = instance.__class__.__module__ + "." \
                    + instance.name
            instance.__postinit__(*nargs, **nkwargs)
        else:
            instance = super(NipypeProcess, cls).__new__(cls, *args, **kwargs)
            setattr(instance, '__%s_np_init_done__' % cls.__name__, False)

    def __init__(self, nipype_instance=None, use_temp_output_dir=None,
                 *args, **kwargs):
        """ Initialize the NipypeProcess class.

        NipypeProcess instance gets automatically an additional user trait
        'output_directory'.

        This class also fix some lacks of the nipype version '0.10.0'.

        NipypeProcess is normally not instantiated directly, but through the
        CapsulEngine factory, using a nipype interface name::

            ce = capsul_engine()
            npproc = ce.get_process_instance('nipype.interfaces.spm.Smooth')

        However, it is now still possible to instantiate it directly, using a
        nipype interface class or instance::

            npproc = NipypeProcess(nipype.interfaces.spm.Smooth)

        NipypeProcess may be subclassed for specialized interfaces. In such a
        case, the subclass may provide:

        * (optionally) a class attribute `_nipype_class_type` to specify the
        nipype interface class. If present the nipype interface class or
        instance will not be specified in the constructor call.
        * (optionally) a :meth:`__postinit__` method which will be called in
        addition to the constructor, but later once the instance is correctly
        setup. This `__postinit__` method allows to customize the new class
        instance.
        * (optionally) a class attribute `_nipype_trait_mapping`: a dict
        specifying a translation table between nipype traits names and the
        names they will get in the Process instance. By default, inputs get the
        same name as in their nipype interface, and outputs are prefixed with
        an underscore ('_') to avoid names collisions when a trait exists both
        in inputs and outputs in nipype. A special trait name
        `_spm_script_file` is also used in SPM interfaces to write the matlab
        script. It can also be translated to a different name in this dict.

        Subclasses should preferably *not* define an __init__ method, because
        it may be called twice if no precaution is taken to avoid it (a
        `__np_init_done__` instance attribute is set once init is done the
        first time).

        Ex::

            class Smooth(NipypeProcess):
                _nipype_class_type = spm.Smooth
                _nipype_trait_mapping = {
                    'smoothed_files': 'smoothed_files',
                    '_spm_script_file': 'spm_script_file'}

            smooth = Smooth()

        nipype_instance: nipype interface (mandatory, except from internals)
            the nipype interface we want to wrap in capsul.
        use_temp_output_dir: bool or None
            use a temp working directory during processing

        Attributes
        _nipype_interface : Interface
            private attribute to store the nipye interface
        _nipype_module : str
            private attribute to store the nipye module name
        _nipype_class : str
            private attribute to store the nipye class name
        _nipype_interface_name : str
            private attribute to store the nipye interface name
        """
        if hasattr(self, '__NipypeProcess_np_init_done__') \
                and self.__NipypeProcess_np_init_done__:
            # may be called twice, from within __new__ or from python internals
            return

        self.__NipypeProcess_np_init_done__ = True
        #super(NipypeProcess, self).__init__(*args, **kwargs)

        # Set some class attributes that characterize the nipype interface
        if nipype_instance is None:
            # probably called from a specialized subclass
            np_class = getattr(self, '_nipype_class_type', None)
            if np_class:
                nipype_instance = np_class()
            else:
                    'NipypeProcess.__init__ must either be called with a '
                    'nipype interface instance as 1st argument, or from a '
                    'specialized subclass providing the _nipype_class_type '
                    'class attribute')
        self._nipype_interface = nipype_instance
        self._nipype_module = nipype_instance.__class__.__module__
        self._nipype_class = nipype_instance.__class__.__name__
        msplit = self._nipype_module.split(".")
        if len(msplit) > 2:
            self._nipype_interface_name = msplit[2]
        else:
            self._nipype_interface_name = 'custom'

        # Inheritance: activate input files copy for spm interfaces.
        if self._nipype_interface_name == "spm":
            # Copy only 'copyfile' nipype traits
            inputs_to_copy = list(self._nipype_interface.inputs.traits(
                copyfile=True).keys())
            inputs_to_symlink = list(self._nipype_interface.inputs.traits(
                copyfile=False).keys())
            out_traits = self._nipype_interface.output_spec().traits()
            inputs_to_clean = [x for x in inputs_to_copy
                               if 'modified_%s' % x not in out_traits]
            if use_temp_output_dir is None:
                use_temp_output_dir = True
            super(NipypeProcess, self).__init__(
                activate_copy=True, inputs_to_copy=inputs_to_copy,
                inputs_to_symlink=inputs_to_symlink,
                inputs_to_clean=inputs_to_clean,
                use_temp_output_dir=use_temp_output_dir,
                *args, **kwargs)
        else:
                use_temp_output_dir = False
                  activate_copy=False, use_temp_output_dir=use_temp_output_dir,
                  *args, **kwargs)

        # Replace the process name and identification attributes
        self.id = ".".join([self._nipype_module, self._nipype_class])
        self.name = self._nipype_interface.__class__.__name__

        # Add a new trait to store the processing output directory
        super(Process, self).add_trait(
            "output_directory", Directory(Undefined, exists=True,
                                          optional=True))

        # Add a 'synchronize' nipype input trait that will be used to trigger
        # manually the output nipype/capsul traits sync.
        super(Process, self).add_trait("synchronize", Int(0, optional=True))

        # use the nipype doc for help
        doc = getattr(nipype_instance, '__doc__')
        if doc:
            self.__doc__ = doc

    def __postinit__(self, *args, **kwargs):
        '''
        `__postinit__` allows to customize subclasses. the base `NipypeProcess`
        implementation does nothing, it is empty.
        '''
        pass

        req = {'nipype': 'any'}
            req['spm'] = 'any'
        elif self._nipype_interface_name == "fsl":
            req['fsl'] = 'any'
        elif self._nipype_interface_name == "freesurfer":
            req['freesurfer'] = 'any'
        return req

    def set_output_directory(self, out_dir):
        """ Set the process output directory.

        out_dir: str (mandatory)
            the output directory
        """
        self.output_directory = out_dir

    def set_usedefault(self, parameter, value):
        """ Set the value of the usedefault attribute on a given parameter.

        parameter: str (mandatory)
            name of the parameter to modify.
        value: bool (mandatory)
            value set to the usedefault attribute
        """
        setattr(self._nipype_interface.inputs, parameter, value)

            # Set the spm working
            self.destination = None
        super(NipypeProcess, self)._before_run_process()
        # configure nipype from config env variables (which should have been set
        # before now)
        from capsul.in_context import nipype as inp_npp
        inp_npp.configure_all()

        """ Method that do the processing when the instance is called.

        Returns
        -------
        runtime: InterfaceResult
            object containing the running results
        """
        try:
            cwd = os.getcwd()
            cwd = None
        if self.output_directory is None or self.output_directory is Undefined:
            raise ValueError('output_directory is not set but is mandatory '
                             'to run a NipypeProcess')
        os.chdir(self.output_directory)
        self.synchronize += 1

        trait_map = getattr(self, '_nipype_trait_mapping', {})
        # Force nipype update
        for trait_name in self._nipype_interface.inputs.traits().keys():
            capsul_name = trait_map.get(trait_name, trait_name)
            if capsul_name in self.user_traits():
                old = getattr(self._nipype_interface.inputs, trait_name)
                new = getattr(self, capsul_name)
                if old is Undefined and old != new:
                    setattr(self._nipype_interface.inputs, trait_name, new)

        results = self._nipype_interface.run()

        # update outputs from nipype
        for trait_name \
                in self._nipype_interface.output_spec().traits().keys():
                old = getattr(self, capsul_name)
                new = getattr(self._nipype_interface.output_spec(),
                              trait_name)
                if old != new:
                    setattr(self, capsul_name, new)

        # For spm, need to move the batch
        # (create in cwd: cf nipype.interfaces.matlab.matlab l.181)
            mfile = os.path.join(
                os.getcwd(),
                self._nipype_interface.mlab.inputs.script_file)
            destmfile = os.path.join(
                self.output_directory,
            if os.path.isfile(mfile):
                shutil.move(mfile, destmfile)

        # Restore cwd
        if cwd is not None:

        #return results.__dict__

        script_tname = trait_map.get('_spm_script_file', '_spm_script_file')
        if getattr(self, script_tname, None) not in (None, Undefined, ''):
            script_file = os.path.join(
            if os.path.exists(script_file):
                shutil.move(script_file, getattr(self, script_tname))
        return super(NipypeProcess,
                     self)._after_run_process(run_process_result)

    def help(cls, nipype_interface, returnhelp=False):
        """ Method to print the full wrapped nipype interface help.

            a nipype process class
        nipype_instance: nipype interface (mandatory)
            a nipype interface object that will be documented.
        """
        from .nipype_process import nipype_factory
        cls_instance = nipype_factory(nipype_interface)

class InteractiveProcess(Process):
    '''
    Base class for interactive processes. The value of the is_interactive 
    parameter determine if either the process can be run in background
    (eventually remotely) as a standard process (is_interactive = False)
    or if the process must be executed interactively in the user environment
    (is_interactive = False).
    '''
    is_interactive = Bool(False)

class ProcessResult(object):
    """ Object that contains running information a particular Process.

    process : Process class (mandatory)
        A copy of the `Process` class that was called.
    runtime : dict (mandatory)
        Execution attributes.
    returncode: dict (mandatory)
        Execution raw attributes
    inputs :  dict (optional)
        Representation of the process inputs.
    outputs : dict (optional)
        Representation of the process outputs.
    """

    def __init__(self, process, runtime, returncode, inputs=None,
                 outputs=None):
        """ Initialize the ProcessResult class.
        """
        self.process = process
        self.runtime = runtime
        self.returncode = returncode
        self.inputs = inputs
        self.outputs = outputs

##########################################################################
# CAPSUL - Copyright (C) CEA, 2013
# Distributed under the terms of the CeCILL-B license, as published by
# the CEA-CNRS-INRIA. Refer to the LICENSE file or to
# http://www.cecill.info/licences/Licence_CeCILL-B_V1-en.html
# for details.

'''
Utilities to link Capsul and NiPype interfaces

Functions
---------
:func:`nipype_factory`
'''

import sys
import os
import types
import six

from traits.api import Directory, File, List, CTrait, Undefined, TraitError

from .process import NipypeProcess

def nipype_factory(nipype_instance, base_class=NipypeProcess):
    """ From a nipype class instance generate dynamically a process
    instance that encapsulate the nipype instance.

    This function clones the nipye traits (also convert special traits) and
    connects the process and nipype instances traits.

    A new 'output_directory' nipype input trait is created.

    Since nipype inputs and outputs are separated and thus can have
    the same names, the nipype process outputs are prefixed with '_'.

    It also monkey patch some nipype functions in order to execute the
    process in a specific directory:
    the monkey patching has been written for Nipype version '0.10.0'.

    nipype_instance : instance (mandatory)
        a nipype interface instance.

    Returns
    -------
    process_instance : instance
        a process instance.

    See Also
    --------
    _run_interface
    _list_outputs
    _gen_filename
    _parse_inputs
    sync_nypipe_traits
    sync_process_output_traits
    clone_nipype_trait
    """

    # Monkey patching for Nipype version '0.9.2'.

    # Change first level masking explicit in postscript
    def _make_matlab_command(self, content):
        from nipype.interfaces.spm import Level1Design
        return super(Level1Design, self)._make_matlab_command(
            content, postscript=None)
    if (nipype_instance.__class__.__module__.startswith(
            'nipype.interfaces.spm.')
        and nipype_instance.__class__.__name__ == "Level1Design"):
        nipype_instance._make_matlab_command = types.MethodType(
            _make_matlab_command, nipype_instance)

    # Create new instance derived from Process
    if hasattr(base_class, '_nipype_class_type'):
        process_instance = base_class()
    else:
        process_instance = base_class(nipype_instance)

    # Define functions to synchronized the process and interface traits

    def sync_nypipe_traits(process_instance, name, old, value):
        """ Event handler function to update the nipype interface traits

        process_instance: process instance (mandatory)
            the process instance that contain the nipype interface we want
            to update.
            the name of the trait we want to update.
        old: type (manndatory)
            the old trait value
        new: type (manndatory)
            the new trait value
        """
        # Set the new nypipe interface value
        trait_map = getattr(process_instance, '_nipype_trait_mapping', {})
        inames = [iname for iname, pname in trait_map.items() if pname == name]
        capsul_name = name
        if inames:
            name = inames[0]

        if name.startswith("nipype_"):
            setattr(process_instance._nipype_interface.inputs,
                    name[7:],
                    value)

        else:
            try:
                setattr(process_instance._nipype_interface.inputs,
                        name,
                        value)
                # reset old value as the assignation has failed
                setattr(process_instance, capsul_name, old)
                raise

    def _replace_dir(value, directory):
        """ Replace directory in filename(s) in value.

        value may be a string, or a list
        """
        if value in (None, Undefined, ""):
            return value
        if isinstance(value, list):
            value = [_replace_dir(x, directory) for x in value]
        else:
            value = os.path.join(directory, os.path.basename(value))

    def sync_process_output_traits(process_instance, name, value):
        """ Event handler function to update the process instance outputs

        This callback is only called when an input process instance trait is
        modified.

            to update.
        value: type (mandatory)
        """
        # Get all the input traits
        input_traits = process_instance.traits(output=False)
        output_directory \
            = getattr(process_instance, 'output_directory', Undefined)

        # get the interface name from the process trait name

        # Try to update all the output process instance traits values when
        # a process instance input trait is modified or when the dedicated
        # 'synchronize' trait value is modified
        if name in input_traits or name in ("synchronize", 'output_directory'):

            # Try to set all the process instance output traits values from
            # the nipype autocompleted traits values
            try:
                nipype_outputs = (process_instance.
                                  _nipype_interface._list_outputs())
                # don't make it all crash because of a nipype trait assign
                # error
                print('EXCEPTION:', e, file=sys.stderr)
                print('while syncing nipype parameter', name,
                      'on', process_instance.name, file=sys.stderr)
                # when called during exit, the traceback module might have
                # already disappeared
                import traceback
                traceback.print_exc()
                ex_type, ex, tb = sys.exc_info()
                logger.debug(
                    "Something wrong in the nipype output trait "
                    "synchronization:\n\n\tError: {0} - {1}\n"
                    "\tTraceback:\n{2}".format(
                        ex_type, ex, "".join(traceback.format_tb(tb))))
                nipype_outputs = {}

            # Synchronize traits: check file existence
            for out_name, out_value in six.iteritems(nipype_outputs):

                pname = trait_map.get(out_name, '_' + out_name)

                try:
                    # if we have an output directory, replace it
                    if output_directory not in (Undefined, None) \
                            and any([x
                                     for x in trait_ids(process_instance.trait(
                                        pname))
                                     if 'File' in x or 'Directory' in x]):
                        out_value = _replace_dir(out_value, output_directory)
                    # Set the output process trait value
                    process_instance.set_parameter(pname, out_value)

                # If we can't update the output process instance traits values,
                # print a logging debug message.
                    print('EXCEPTION:', e, file=sys.stderr)
                    print('while setting nipype output parameter', pname,
                          'on', process_instance.name, 'with value:',
                          out_value, file=sys.stderr)
                    ex_type, ex, tb = sys.exc_info()
                    logger.debug(
                        "Something wrong in the nipype output trait "
                        "synchronization:\n\n\tError: {0} - {1}\n"
                        "\tTraceback:\n{2}".format(
                            ex_type, ex, "".join(traceback.format_tb(tb))))

        if name in list(input_traits.keys()) + ['synchronize',
                                                'output_directory']:
            names = [name]
            if name == 'output_directory':
                names = list(input_traits.keys())
                # check if the input trait is duplicated as an output
                trait = process_instance.trait(name)
                if trait.copyfile:
                    out_trait = process_instance.trait('_modified_%s' % name)
                    if out_trait:
                        new_value = getattr(process_instance, name)
                        if output_directory not in (Undefined, None):
                            new_value = _replace_dir(new_value,
                                                     output_directory)
                        try:
                            process_instance.set_parameter(
                                "_modified_%s" % name, new_value)
                        # If we can't update the output process instance
                        # traits values, print a logging debug message.
                            print('EXCEPTION:', e)
                            import traceback
                            traceback.print_exc()
                            ex_type, ex, tb = sys.exc_info()
                            logger.debug(
                                "Something wrong in the nipype output trait "
                                "synchronization:\n\n\tError: {0} - {1}\n"
                                "\tTraceback:\n{2}".format(
                                    ex_type, ex, "".join(traceback.format_tb(tb))))

    # Clone nipype traits

    # The following function is not shared since it is too specific
    def clone_nipype_trait(process_instance, nipype_trait):
        """ Create a new trait (cloned and converted if necessary)
        from a nipype trait.

        nipype_trait: trait
            the nipype trait we want to clone and convert if necessary.

        Returns
        -------
        process_trait: trait
            the cloned/converted trait that will be used in the process
            instance.
        """
        # Clone the nipype trait
        process_trait = process_instance._clone_trait(nipype_trait)

        # Copy some information from the nipype trait
        process_trait.desc = nipype_trait.desc
        process_trait.optional = not nipype_trait.mandatory
        process_trait._metadata = {}

        return process_trait

    # beginning of nipype_factory function
    try:
        import nipype.interfaces.base.traits_extension as npe
    except (AttributeError, ImportError):
        # In some situations an AttributeError is raised, with the message:
        # module 'nipype.interfaces' has no attribute 'base'
        # but the module is actually here. Maybe it has not finished loading
        # (how can that happen?)
        if 'nipype.interfaces.base.traits_extension' in sys.modules:
            npe = sys.modules['nipype.interfaces.base.traits_extension']
        else:
            # no nipype, or problem loading it. Give up, use regular traits.
            import traits.api as npe

    # Add nipype traits to the process instance
    # > input traits
    trait_map = getattr(process_instance, '_nipype_trait_mapping', {})

    for trait_name, trait in nipype_instance.input_spec().items():

        # Check if trait name already used in class attributes:
        # For instance nipype.interfaces.fsl.FLIRT has a save_log bool input
        # trait.
        trait_name = trait_map.get(trait_name, trait_name)
        if hasattr(process_instance, trait_name):
            trait_name = "nipype_" + trait_name

        # Relax nipype exists trait constraint

        process_trait = clone_nipype_trait(process_instance, trait)

        # Add the cloned trait to the process instance
        process_instance.add_trait(trait_name, process_trait)

        # Need to copy all the nipype trait information
        process_instance.trait(trait_name).optional = not trait.mandatory
        process_instance.trait(trait_name).desc = trait.desc
        process_instance.trait(trait_name).output = False

        # initialize value with nipype interface initial value, (if we can...)
        try:

            if trait_name.startswith("nipype_"):
                setattr(process_instance,
                        trait_name,
                        getattr(nipype_instance.inputs, trait_name[7:]))

            else:
                setattr(process_instance, trait_name,
                        getattr(nipype_instance.inputs, trait_name))

        except TraitError:
            # the value in the nipype trait is actually invalid...
            pass

        # Add the callback to update nipype traits when a process input
        # trait is modified
        process_instance.on_trait_change(sync_nypipe_traits, name=trait_name)

        ## if copyfile is True, we assume the input will be modified, thus it
        ## also becomes an output
        #if trait.copyfile:
            #process_trait = clone_nipype_trait(process_instance, trait)
            #process_trait.output = True
            #process_trait.input_filename = False
            #process_trait.optional = True
            #private_name = "_" + trait_name + '_out'
            #process_instance.add_trait(private_name, process_trait)

    # Add callback to synchronize output process instance traits with nipype
    # autocompleted output traits
    process_instance.on_trait_change(sync_process_output_traits)

    # > output traits
    for trait_name, trait in nipype_instance.output_spec().items():

        # Relax nipye exists trait constraint

        # Create the output process trait name: nipype trait name prefixed
        # by '_'
        private_name = trait_map.get(trait_name, '_' + trait_name)

        process_instance.add_trait(private_name, process_trait)
        process_trait = process_instance.trait(private_name)

        process_trait.optional = not trait.mandatory
        process_trait.desc = trait.desc
        process_trait.output = True
        process_trait.enabled = False

        # SPM output File traits and lists of File should have the
        # metatata input_filename=False
        if process_instance._nipype_interface_name == 'spm':
            if isinstance(process_trait.trait_type,
                          (File, Directory, npe.File, npe.Directory)):
                process_trait.input_filename = False
            elif isinstance(process_trait.trait_type, List) \
                    and isinstance(process_trait.inner_traits[0].trait_type,
                                   (File, Directory, npe.File, npe.Directory)):
                process_trait.inner_traits[0].output = True
                process_trait.inner_traits[0].input_filename = False

    # allow to save the SPM .m script
    if nipype_instance.__class__.__module__.startswith(
            'nipype.interfaces.spm.'):
        script_name = trait_map.get('_spm_script_file', '_spm_script_file')

        process_instance.add_trait(script_name,
                                   File(output=True, optional=True))

    return process_instance

#
#  This software and supporting documentation were developed by
#      CEA/DSV/SHFJ and IFR 49
#      4 place du General Leclerc
#      91401 Orsay cedex
#      France
#
# This software is governed by the CeCILL license version 2 under
# French law and abiding by the rules of distribution of free software.
# You can  use, modify and/or redistribute the software under the
# terms of the CeCILL license version 2 as circulated by CEA, CNRS
# and INRIA at the following URL "http://www.cecill.info".
#
# As a counterpart to the access to the source code and  rights to copy,
# modify and redistribute granted by the license, users are provided only
# with a limited warranty  and the software's author,  the holder of the
# economic rights,  and the successive licensors  have only  limited
# liability.
#
# In this respect, the user's attention is drawn to the risks associated
# with loading,  using,  modifying and/or developing or reproducing the
# software by the user in light of its specific status of free software,
# that may mean  that it is complicated to manipulate,  and  that  also
# therefore means  that it is reserved for developers  and  experienced
# professionals having in-depth computer knowledge. Users are therefore
# encouraged to load and test the software's suitability as regards their
# requirements in conditions enabling the security of their systems and/or
# data to be ensured and,  more generally, to use and operate it in the
# same conditions as regards security.
#
# The fact that you are presently reading this means that you have had
# knowledge of the CeCILL license version 2 and that you accept its terms.

"""
capsul.process.runprocess is not a real python module, but rather an executable script with commandline arguments and options parsing. It is provided as a module just to be easily called via the python command in a portable way::

    python -m capsul.process.runprocess <process name> <process arguments>

Classes
-------
:class:`ProcessParamError`
++++++++++++++++++++++++++

Functions
---------
:func:`set_process_param_from_str`
++++++++++++++++++++++++++++++++++
:func:`get_process_with_params`
+++++++++++++++++++++++++++++++
:func:`run_process_with_distribution`
+++++++++++++++++++++++++++++++++++++
:func:`convert_commandline_parameter`
:func:`main`
++++++++++++

"""

from capsul.api import StudyConfig
from capsul.api import capsul_engine
from capsul.api import Pipeline
from soma.qt_gui import qt_backend
import os
import sys
import re
from optparse import OptionParser, OptionGroup
from traits.api import Undefined, List
import subprocess
try:
    import yaml
    yaml = None
    import json
import six

class ProcessParamError(Exception):
    ''' Exception used in the ``runprocess`` module
    '''
    pass

def set_process_param_from_str(process, k, arg):
    """Set a process parameter from a string representation."""
    if '.' in k and hasattr(process, 'nodes'):
        sub_node_name, k2 = k.split('.', 1)
        sub_node = process.nodes.get(sub_node_name)
        if sub_node is not None:
            try:
                return set_process_param_from_str(sub_node, k2, arg)
                sub_proc = getattr(sub_node, 'process', None)
                if sub_proc is not None:
                    return set_process_param_from_str(sub_proc, k2, arg)
    klist = k.split('.')
    k = klist[0]
    if len(klist) == 1 and not process.trait(k):
        if hasattr(process, k):
            # print('set non-trait value:', process, k, repr(arg))
            setattr(process, k, arg)
            return
        if hasattr(process, 'pipeline_node') \
                and hasattr(process.pipeline_node, k):
            setattr(process.pipeline_node, k, arg)
            return
        raise ProcessParamError("Unknown parameter {0} for process {1}"
                                .format(k, process.name))

    if len(klist) > 1:
        obj = process
        for k in klist[:-1]:
            obj = getattr(obj, k)
        setattr(obj, klist[-1], arg)
        return

    try:
        evaluate = process.trait(k).trait_type.evaluate
    except AttributeError:
        evaluate = None
    if evaluate:
        arg = evaluate(arg)
    setattr(process, k, arg)
    process.trait(k).forbid_completion = True
        try:
            process.propagate_metadata(process.pipeline_node, k,
                                       {'forbid_completion': True})
            # some traits like pipeline_steps are not parameters
            pass

def parse_pipeline_steps(process, kwargs):
    if not isinstance(process, Pipeline):
        return

    param_steps = kwargs.get('pipeline_steps')
    if param_steps is None:
        return

    del kwargs['pipeline_steps']

    pipeline_steps = getattr(process, 'pipeline_steps')
    if not isinstance(pipeline_steps, Controller):
        return

    if isinstance(param_steps, str):
        param_steps = [s.strip() for s in param_steps.split(',')]
    steps = [[s.strip() for s in p.split('=')] for p in param_steps]
    for s in steps:
        if len(s) == 1:
            s.append(True)
        else:
            val = s[1]
            if val in (0, '0', 'False'):
                val = False
            else:
                val = True
            s[1] = val
    steps = [[[s.strip() for s in p[0].split('-')]] + p[1:] for p in steps]
    if len(steps) >= 1 and steps[0][1]:
        # set all steps to False
        steps.insert(0, [['', ''], False])

    for srange, val in steps:
        if len(srange) == 2:
            if srange[0] == '':
                s0 = 0
            else:
                s0 = pipeline_steps.user_traits().index(srange[0])
            if srange[1] == '':
                s1 = len(pipeline_steps.user_traits()) - 1
            else:
                s1 = pipeline_steps.user_traits().index(srange[1])
            srange = list(pipeline_steps.user_traits())[s0:s1+1]
        for s in srange:
            setattr(pipeline_steps, s, val)

def get_process_with_params(process_name, study_config, iterated_params=[],
                            attributes={}, *args, **kwargs):
    ''' Instantiate a process, or an iteration over processes, and fill in its
    parameters.

    process_name: string
        name (mosule and class) of the process to instantiate
    study_config: StudyConfig instance
    iterated_params: list (optional)
        parameters names which should be iterated on. If this list is not
        empty, an iteration process is built. All parameters values
        corresponding to the selected names should be lists with the same size.
    attributes: dict (optional)
        dictionary of attributes for completion system.
    *args:
        sequential parameters for the process. In iteration, "normal"
        parameters are set with the same value for all iterations, and iterated
        parameters dispatch their values to each iteration.
    **kwargs:
        named parameters for the process. Same as above for iterations.

    Returns
    -------
    process: Process instance
    '''
    process = study_config.get_process_instance(process_name)
    signature = process.user_traits()
    params = list(signature.keys())

    steps = parse_pipeline_steps(process, kwargs)

    # check for iterations
    if iterated_params:

        pipeline = study_config.get_process_instance(Pipeline)
        pipeline.add_iterative_process('iteration', process, iterated_params)
        process = pipeline

        # transform iterated attributes into lists if needed
        for param, value in attributes.items():
            if not isinstance(value, list) and not isinstance(value, tuple):
                attributes[param] = list([value])

    for i, arg in enumerate(args):
        set_process_param_from_str(process, params[i], arg)
    for k, arg in six.iteritems(kwargs):
        set_process_param_from_str(process, k, arg)

    completion_engine = ProcessCompletionEngine.get_completion_engine(process)
    try:
        completion_engine.get_attribute_values().import_from_dict(attributes)
    except KeyError as e:
        print('error in completion attributes settings. Controller attributes '
              'are:', file=sys.stderr)
        print(completion_engine.get_attribute_values().user_traits(),
              file=sys.stderr)
        raise
    completion_engine.complete_parameters()

    return process

def run_process_with_distribution(
        study_config, process, use_soma_workflow=False,
        resource_id=None, password=None, config=None, rsa_key_pass=None,
        queue=None, input_file_processing=None, output_file_processing=None,
        keep_workflow=False, keep_failed_workflow=False,
        write_workflow_only=None, max_running_jobs=None, max_queued_jobs=None):
    ''' Run the given process, either sequentially or distributed through
    Soma-Workflow.

        the process to execute (or pipeline, or iteration...)
    use_soma_workflow: bool or None (default=None)
        if False, run sequentially, otherwise use Soma-Workflow. Its
        configuration has to be setup and valid for non-local execution, and
        additional file transfer options may be used.
    resource_id: string (default=None)
        soma-workflow resource ID, defaults to localhost
    password: string
        password to access the remote computing resource. Do not specify it if
        using a ssh key.
    config: dict (optional)
        Soma-Workflow config: Not used for now...
    rsa_key_pass: string
        RSA key password, for ssh key access
    queue: string
        Queue to use on the computing resource. If not specified, use the
        default queue.
    input_file_processing: brainvisa.workflow.ProcessToSomaWorkflow processing code
        Input files processing: local_path (NO_FILE_PROCESSING),
        transfer (FILE_TRANSFER), translate (SHARED_RESOURCE_PATH),
        or translate_shared (BV_DB_SHARED_PATH).
    output_file_processing: same as for input_file_processing
        Output files processing: local_path (NO_FILE_PROCESSING),
        transfer (FILE_TRANSFER), or translate (SHARED_RESOURCE_PATH).
        The default is local_path.
    keep_workflow: bool
        keep the workflow in the computing resource database after execution.
        By default it is removed.
    keep_failed_workflow: bool
        keep the workflow in the computing resource database after execution,
        if it has failed. By default it is removed.
    write_workflow_only: str
        if specified, this is an output filename where the workflow file will
        be written. The workflow will not be actually run, because int his
        situation the user probably wants to use the workflow on his own.
    max_running_jobs: int
        override the queue settings for OCFG_MAX_JOB_RUNNING in soma-workflow
    max_queued_jobs: int
        override the queue settings for OCFG_MAX_JOB_IN_QUEUE in soma-workflow
    '''
    if write_workflow_only:
        use_soma_workflow = True

    if use_soma_workflow is not None:
        study_config.use_soma_workflow = use_soma_workflow

    if study_config.use_soma_workflow:

        if write_workflow_only:
            # Create soma workflow pipeline
            from capsul.pipeline.pipeline_workflow \
                import workflow_from_pipeline

            workflow = workflow_from_pipeline(process)
            swclient.Helper.serialize(write_workflow_only, workflow)

            return

        swm = study_config.modules['SomaWorkflowConfig']
        resource_id = swm.get_resource_id(resource_id, set_it=True)
        if password is not None or rsa_key_pass is not None:
            swm.set_computing_resource_password(resource_id, password,
                                                rsa_key_pass)
        if queue is not None:
            if not hasattr(
                    study_config.somaworkflow_computing_resources_config,
                    resource_id):
                setattr(study_config.somaworkflow_computing_resources_config,
                        resource_id, {})
            getattr(study_config.somaworkflow_computing_resources_config,
                    resource_id).queue = queue
        if max_running_jobs is not None or max_queued_jobs is not None:
            values = {}
            values['computing_resource'] = resource_id
            if queue is not None:
                values['queue'] = queue
            if max_running_jobs is not None:
                values['max_running_jobs'] = max_running_jobs
            if max_queued_jobs is not None:
                values['max_queued_jobs'] = max_queued_jobs
            engine.load_module('somaworkflow')
            with engine.settings as session:
                query = 'config_id == "somaworkflow"'
                if resource_id is not None:
                    query += f' AND computing_resource == "{resource_id}"'
                conf = session.config('somaworkflow', 'global',
                                      selection=query)
                    session.new_config('somaworkflow', 'global', values)
                else:
                    for k, v in values.items():
                        setattr(conf, k, v)

    res = study_config.run(process)
    return res

def convert_commandline_parameter(i):
    i = i.replace('<undefined>', 'Undefined')
    if len(i) > 0 and (i[0] in '[({' or i in ('None', 'True', 'False',
                                              'Undefined')):
        try:
            res=eval(i)
            res=i
    else:
        res = i
    return res

# main
def main():
    ''' Run the :mod:`capsul.process.runprocess` module as a commandline
    '''

    # disable any GUI
    qt_backend.set_headless(needs_opengl=False)

    usage = '''Usage: python -m capsul [options] processname [arg1] [arg2] ...
    [argx=valuex] [argy=valuey] ...

    Example::

        python -m capsul threshold ~/data/irm.ima /tmp/th.nii threshold1=80

    Named arguments (in the shape argx=valuex) may address sub-processes of a
    pipeline, using the dot separator::

        PrepareSubject.t1mri=/home/myself/mymri.nii

    For a more precise description, please look at the web documentation:
    https://brainvisa.info/capsul/user_guide_tree/index.html

    Configuration:

    The ``--config`` option allows to pass a Capsul configuration file (a JSON
    dict). If you have a configuration in Brainvisa/Axon and want to convert
    it, use the following::

        axon-runprocess capsul://capsul.engine.write_engine_config engine.json

    Then the file ``engine.json`` will be OK.
    Alternatively, using "--config axon" will do this for you internally.

    It is possible to reach sub-process parameters in a pipeline by addressing
    their node in the parent pipeline:

    python -m capsul morphologist.capsul.morphologist Renorm.enabled=False

    Pipeline steps:

    Some pipelines define steps that can be enabled or disabled in order to
    process only part of the pipeline. They are kind of groups of nodes, but
    behave a bit differently for the pipeline parameters completion and checks.

    Steps can be specified as the "pipeline_steps" parameter, with a syntax
    which can mix ranges and values, ex::

        python -m capsul morphologist.capsul.morphologist pipeline_steps="importation,bias_correction"

    will do only these 2 steps. While::

        python -m capsul morphologist.capsul.morphologist pipeline_steps="brain_extraction, head_mesh-, sulci_labelling=False"

    will do brain_extraction, then all steps from head_mesh, except
    sulci_labelling which is disabled.
    '''

    # Set up logging on stderr. This must be called before any logging takes
    # place, to avoid "No handlers could be found for logger" errors.
    logging.basicConfig()

    parser = OptionParser(description='Run a single CAPSUL process',
        usage=usage)
    group1 = OptionGroup(
        parser, 'Config',
        description='Processing configuration, database options')
    group1.add_option('--studyconfig', dest='studyconfig',
        help='load StudyConfig configuration from the given file (JSON)')
    group1.add_option(
        '--config', dest='config',
        help='load Capsul engine configuration from the given file (JSON) '
        '(CapsulEngine shape, not Studyconfig -- use --studyconfig '
        'otherwise). Using "--config axon" will run the conversion process in '
        'Axon and import the config.')
    group1.add_option('-i', '--input', dest='input_directory',
                      help='input data directory (if not specified in '
                      'studyconfig file). If not specified neither on the '
                      'commandline nor study configfile, taken as the same as '
                      'output.')
    group1.add_option('-o', '--output', dest='output_directory',
                      help='output data directory (if not specified in '
                      'input.')
    group1.add_option('--if', '--input-fom', dest='input_fom',
                      help='input FOM (File Organization Model). Decides '
                      'which files and directories layout for the input '
                      'data. Generally "morphologist-bids-1.0" or '
                      '"morphologist-auto-nonoverlap-1.0". Default: '
                      '"morphologist-bids-1.0"')
    group1.add_option('--of', '--output-fom', dest='output_fom',
                      'which files and directories layout for the output '
    group1.add_option('--params', dest='paramsfile', default=None,
                      help='specify a file containing commandline parameters. '
                      'The file will contain arguments for this commandline '
                      '(argv): it is an alternative to providing them here. '
                      'It can be useful to reuse parameters, or when the '
                      'parameters are too long (in a large iteration, '
                      'typically). The file syntax is one line per parameter, '
                      'with no further parsing. It will be processed after '
                      'all the current commandline arguments, not right now '
                      'as the argument appears. But if a parameter has '
                      'already been set (via commandline), it will not be '
                      'replaced: first set arguments have priority. If the '
                      'params file itself contains a --params parameter, '
                      'then another file will be read afterwards, and so on.')
    group1.add_option('--opengl', action='store_true', default=False,
                      help='Tell the process loading system that we will '
                      'require headless OpenGL, which needs proper setup and '
                      'libraries loading tweaks. Without this option, Qt and '
                      'potentially graphical modules will be initialized in '
                      'headless mode, but there will be no check for a '
                      'working OpenGL/GLX implementation. In some cases it '
                      'will work anyway, but in others (no X server) OpenGL '
                      'will require using a virtual X server (Xvfb) and '
                      'possibly loading appropriate OpenGL libraries. This is '
                      'not done systematically because of the '
                      'overhead it brings.')
    group1.add_option('-s', '--show-pipeline', action='store_true',
                      help='Display and edit pipeline structure')
    group1.add_option('-e', '--edit', action='store_true',
                      help='Display and edit process parameters')
    parser.add_option_group(group1)

    group2 = OptionGroup(parser, 'Processing',
                        description='Processing options, distributed execution')
    group2.add_option('--swf', '--soma_workflow', dest='soma_workflow',
                      default=False,
                      action='store_true',
                      help='use soma_workflow. Soma-Workflow '
                      'configuration has to be setup and valid for non-local '
                      'execution, and additional file transfer options '
                      'may be used. The default is *not* to use SWF and '
                      'process mono-processor, sequential execution.')
    group2.add_option('-r', '--resource_id', dest='resource_id', default=None,
                      help='soma-workflow resource ID, defaults to localhost')
    group2.add_option('-w', '--write-workflow-only', dest='write_workflow',
                      default=None,
                      help='if specified, this is an output '
                      'filename where the workflow file will be written. The '
                      'workflow will not be actually run, because in this '
                      'situation the user probably wants to use the workflow '
                      'on his own.')
    group2.add_option('-n', '--name', dest='workflow_name', default=None,
                      help='workflow name')
    group2.add_option('-p', '--password', dest='password', default=None,
                      help='password to access the remote computing resource. '
                      'Do not specify it if using a ssh key')
    group2.add_option('--rsa-pass', dest='rsa_key_pass', default=None,
                      help='RSA key password, for ssh key access')
    group2.add_option('--queue', dest='queue', default=None,
                      help='Queue to use on the computing resource. If not '
                      'specified, use the default queue.')
    group2.add_option('--max_running', type=int, default=None,
                      help='set maximum running jobs (override the queue '
                      'settings in config file)')
    group2.add_option('--max_queued', type=int, default=None,
                      help='set maximum queued jobs (override the queue '
    #group2.add_option('--input-processing', dest='input_file_processing',
                      #default=None, help='Input files processing: local_path, '
                      #'transfer, translate, or translate_shared. The default is '
                      #'local_path if the computing resource is the localhost, or '
                      #'translate_shared otherwise.')
    #group2.add_option('--output-processing', dest='output_file_processing',
                      #default=None, help='Output files processing: local_path, '
                      #'transfer, or translate. The default is local_path.')
    group2.add_option('--keep-succeeded-workflow', dest='keep_succeded_workflow',
                      action='store_true', default=False,
                      help='keep the workflow in the computing resource '
                      'database after execution. By default it is removed.')
    group2.add_option('--delete-failed-workflow', dest='delete_failed_workflow',
                      help='delete the workflow in the computing resource '
                      'database after execution, if it has failed. By default '
                      'it is kept.')
    parser.add_option_group(group2)

    group3 = OptionGroup(parser, 'Iteration',
                        description='Iteration')
    group3.add_option('-I', '--iterate', dest='iterate_on', action='append',
                      help='Iterate the given process, iterating over the '
                      'given parameter(s). Multiple parameters may be '
                      'iterated jointly using several -I options. In the '
                      'process parameters, values are replaced by lists, all '
                      'iterated lists should have the same size.\n'
                      'Ex:\n'
                      'python -m capsul -I par_a -I par_c a_process '
                      'par_a="[1, 2]" par_b="something" '
                      'par_c="[\\"one\\", \\"two\\"]"')
    parser.add_option_group(group3)

    group4 = OptionGroup(parser, 'Attributes completion')
    group4.add_option('-a', '--attribute', dest='attributes', action='append',
                      default=[],
                      help='set completion (including FOM) attribute. '
                      'Syntax: attribute=value, value the same syntax as '
                      'process parameters (python syntax for lists, for '
                      'instance), with proper quotes if needed for shell '
                      'escaping.\n'
                      'Ex: -a acquisition="default" '
                      '-a subject=\'["s1", "s2"]\'')
    parser.add_option_group(group4)

    group5 = OptionGroup(parser, 'Help',
                        description='Help and documentation options')
    group5.add_option('--process-help', dest='process_help',
        action='store_true', default=False,
        help='display specified process help')
    parser.add_option_group(group5)

    parser.disable_interspersed_args()
    (options, args) = parser.parse_args()

    default_fom = 'morphologist-bids-1.0'

    gui = False
    if options.show_pipeline or options.edit:
        gui = True
    qt_backend.set_headless(headless_mode=not gui, needs_opengl=options.opengl)
        from soma.qt_gui.qt_backend import Qt
        qapp = Qt.QApplication([])

    while options.paramsfile:
        pfile = options.paramsfile
        options.paramsfile = None
        with open(pfile) as f:
            new_argv = [l.strip() for l in f.readlines()]
        new_options, new_args = parser.parse_args(new_argv)
        for k, v in new_options.__dict__.items():
            if not getattr(options, k, None):
                setattr(options, k, v)
        args += new_args

    engine = capsul_engine()
    engine.load_modules(['fom', 'axon', 'somaworkflow'])
    study_config = engine.study_config

    if options.config:
        config_file = options.config
        tmp = None
        if options.config == 'axon':
            tmp = tempfile.mkstemp(prefix='capsul_conf', suffix='.json')
            os.close(tmp[0])
            cmd = ['axon-runprocess',
                    'capsul://capsul.engine.write_engine_config',
                    tmp[1]]
            subprocess.check_call(cmd)
            config_file = tmp[1]
        with open(config_file) as f:
            if yaml:
                conf = yaml.load(f, Loader=yaml.SafeLoader)
            else:
                conf = json.load(f)
        if tmp:
            os.unlink(tmp[1])
        for env, c in conf.items():
            engine.import_configs(env, c)
    elif options.studyconfig:
        with open(options.studyconfig) as f:
            if yaml:
                scdict = yaml.load(f, Loader=yaml.SafeLoader)
            else:
                scdict = json.load(f)
        study_config.set_study_configuration(scdict)
    else:
        study_config = StudyConfig(
            modules=StudyConfig.default_modules + ['FomConfig'])
        study_config.read_configuration()
        study_config.use_fom = True

    if options.input_directory:
        study_config.input_directory = options.input_directory
    if options.output_directory:
        study_config.output_directory = options.output_directory
    if study_config.output_directory in (None, Undefined, '') \
            and study_config.input_directory not in (None, Undefined, ''):
        study_config.output_directory = study_config.input_directory
    if study_config.input_directory in (None, Undefined, '') \
            and study_config.output_directory not in (None, Undefined, ''):
        study_config.input_directory = study_config.output_directory
    study_config.somaworkflow_keep_succeeded_workflows \
        = options.keep_succeded_workflow
    study_config.somaworkflow_keep_failed_workflows \
        = not options.delete_failed_workflow

    with engine.settings as session:
        config = session.config('fom', 'global')
        if options.input_fom is not None:
            config.input_fom = options.input_fom
        else:
            try:
                config.input_fom = default_fom
                global sys  # WHY does the global import not work here ??
                print('Warning: FOM', default_fom, 'could not be loaded.',
        if options.output_fom is not None:
            config.output_fom = options.output_fom
        else:
            try:
                config.output_fom = default_fom
                pass

    kwre = re.compile(r'([a-zA-Z_](\.?[a-zA-Z0-9_])*)\s*=\s*(.*)$')

    attributes = {}
    for att in options.attributes:
        m = kwre.match(att)
        if m is None:
            raise SyntaxError('syntax error in attribute definition: %s' % att)
        attributes[m.group(1)] = convert_commandline_parameter(m.group(3))

    args = tuple((convert_commandline_parameter(i) for i in args))
    kwargs = {}
    todel = []
    for arg in args:
        if isinstance(arg, six.string_types):
            m = kwre.match(arg)
            if m is not None:
                kwargs[m.group(1)] = convert_commandline_parameter(m.group(3))
                todel.append(arg)
    args = [arg for arg in args if arg not in todel]

    if not args:
        parser.print_usage()
        sys.exit(2)

    # get the main process
    process_name = args[0]
    args = args[1:]

    iterated = options.iterate_on
    try:
        process = get_process_with_params(process_name, study_config, iterated,
                                          attributes,
                                          *args, **kwargs)
    except ProcessParamError as e:
        print("error: {0}".format(e), file=sys.stderr)
        sys.exit(1)

    if options.process_help:
        process.help()

        print()

        attribs = completion_engine.get_attribute_values()
        aval = attribs.export_to_dict()
        print('Completion attributes:')
        print('----------------------')
        print()
        print('(note: may differ depending on study config file contents, '
              'completion rules (FOM)...)')
        print()

        skipped = set(['generated_by_parameter', 'generated_by_process'])
        for name, value in six.iteritems(aval):
            if name in skipped:
                continue
            ttype = attribs.trait(name).trait_type.__class__.__name__
            if isinstance(attribs.trait(name).trait_type, List):
                ttype += '(%s)' \
                    % attribs.trait(name).inner_traits[
                        0].trait_type.__class__.__name__
            print('%s:' % name, ttype)
            if value not in (None, Undefined):
                print('   ', value)

        print()

        del aval, attribs, completion_engine, process
        sys.exit(0)

    if options.show_pipeline:
        from capsul.qt_gui.widgets import PipelineDeveloperView

        mpv = PipelineDeveloperView(process, allow_open_controller=True,
                                    show_sub_pipelines=True,
                                    enable_edition=True)
        mpv.show()

    if options.edit:
        from capsul.qt_gui.widgets.attributed_process_widget \
            import AttributedProcessWidget

        pc = ProcessCompletionEngine.get_completion_engine(process)
        pcvi = AttributedProcessWidget(
            process, enable_attr_from_filename=True, enable_load_buttons=True)

        pcvi.show()

    if gui:
        qapp.exec()

    resource_id = options.resource_id
    password = options.password
    rsa_key_pass = options.rsa_key_pass
    queue = options.queue
    file_processing = []
    max_running_jobs = options.max_running
    max_queued_jobs = options.max_queued

    study_config.use_soma_workflow = options.soma_workflow

    if options.soma_workflow:
        file_processing = [None, None]

    else:

    if options.workflow_name is not None:
        process.name = options.workflow_name

    res = run_process_with_distribution(
        study_config, process, options.soma_workflow, resource_id=resource_id,
        password=password, rsa_key_pass=rsa_key_pass,
        queue=queue, input_file_processing=file_processing[0],
        output_file_processing=file_processing[1],
        write_workflow_only=options.write_workflow,
        max_running_jobs=max_running_jobs, max_queued_jobs=max_queued_jobs)

    # if there was no exception, we assume the process has succeeded.
    # sys.exit(0)
    # no error, do a dirty exit, but avoid cleanup crashes after the process
    # has succeeded...
    os._exit(0)

    # otherwise it has raised an exception, exit "normally"

    main()

'''
Read and write a Process as an XML file.

Classes
=======
:class:`XMLProcess`

Functions
=========
:func:`string_to_value`
:func:`trait_from_xml`
:func:`create_xml_process`

Decorator
=========
:func:`xml_process`
'''

import sys
from ast import literal_eval

from soma.utils.functiontools import getArgumentsSpecification

from traits.api import (Int, Float, String, Unicode, File, Directory, Enum,
                        Bool, List, Any, Undefined)

_known_values = {
    'Undefined': Undefined,
    'None': Undefined
}
    
def string_to_value(string):
    """ Converts a string into a Python value without executing code.
    """
    value = _known_values.get(string)
    if value is None:
        try:
            value = literal_eval(string)
        except ValueError as e:
            raise ValueError('%s: %s' % (str(e), repr(string)))
    return value

class XMLProcess(Process):
    """ Base class of all generated classes for processes defined as a Python
    function decorated with an XML string.
    """    
        """ Execute the XMLProcess. Runs the function and check its result
        to set the output attributes values accordingly.
        """
        # Build function parameters with identified input attributes
        kwargs = dict((i, getattr(self, i)) for i in self._function_inputs)
        # Calls the function and get the result
        result = self._function(**kwargs)
        if self._function_return:
            # Process was declared to return a single value
            setattr(self, self._function_return, result)
        elif self._function_outputs:
            # Process was declared to return several output values
            if isinstance(result, (list, tuple)):
                # Return value is a list, maps the output parameter names in
                # their declaration order
                if len(result) > len(self._function_outputs):
                    raise ValueError('Too many values (%d instead of %d) '
                                     'returned by process %s' % 
                                     (len(result), len(self._function_outputs),
                                      self.id))
                for i in range(len(self._function_outputs)):
                    setattr(self, self._function_outputs[i], result[i])
            elif isinstance(result, dict):
                # Return value is a dict, set the output parameter values.
                for i in self._function_outputs:
                    setattr(self, i, result[i])
            else:
                raise ValueError('Value returned by process %s must be a list,'
                                 ' tuple or dict' % self.id)

string_to_trait = {
    'int': (Int, {}),
    'float': (Float, {}),
    'string': (String, {}),
    'unicode': (Unicode, {}),
    'bool': (Bool, {}),
    'file': (File, {}),
    'directory': (Directory, {}),
    'any': (Any, {}),
    'list_int': (List, {'trait': Int}),
    'list_float': (List, {'trait': Float}),
    'list_string': (List, {'trait': String}),
    'list_unicode': (List, {'trait': Unicode}),
    'list_bool': (List, {'trait': Bool}),
    'list_file': (List, {'trait': File}),
    'list_directory': (List, {'trait': Directory}),
    'list_any': (List, {'trait': Any}),
}
def trait_from_xml(element, order=None):
    """ Creates a trait from an XML element type (<input>, <output> or
    <return>).
    """
    type = element.get('type')
    trait_args = ()
    t = string_to_trait.get(type)
    if t:
        trait_class, trait_kwargs = t
    elif type == 'enum':
        trait_class = Enum
        trait_args = string_to_value(element.get('values'))
        trait_kwargs = {}
    else:
        raise ValueError('Invalid parameter type: %s' % type)
    trait_kwargs = trait_kwargs.copy()
    trait_kwargs['output'] = (element.tag != 'input')
    doc = element.get('doc')
    if doc:
        trait_kwargs['desc'] = doc
    optional = element.get('optional')
    if optional is not None:
        trait_kwargs['optional'] = bool(optional == 'true')
    allowed_ext = element.get('allowed_extensions')
    if allowed_ext is not None:
        trait_kwargs['allowed_extensions'] = eval(allowed_ext)
    if type in ('file', 'list_file'):
        input_filename = element.get('input_filename')
        if input_filename is not None:
            trait_kwargs['input_filename'] = input_filename
        else:
            trait_kwargs['input_filename'] = False
    if order is not None:
        trait_kwargs['order'] = order
    return trait_class(*trait_args, **trait_kwargs)

def create_xml_process(module, name, function, xml):
    """
    Create a new process class given a Python function and a string containing
    the corresponding Capsul XML 2.0 definition.

        name of the module for the created Process class (the Python module is
        name of the new process class
    function: callable (mandatory)
        function to call to execute the process.
    xml: str (mandatory)
        XML definition of the function.

    Returns
    -------
    results:  XMLProcess subclass
        created process class.
    """
    xml_process = ET.fromstring(xml)
    
    class_kwargs = {
        "__module__": module,
    }
    if function.__doc__:
        class_kwargs['__doc__'] = function.__doc__
    
    args, varargs, varkw, defaults = getArgumentsSpecification(function)
    if defaults:
        default_values = dict((args[-1-i], defaults[-1-i]) for i in range(len(defaults)))
    else:
        default_values = {}
    class_kwargs['default_values'] = default_values
    version = xml_process.get('capsul_xml')
    
    role = xml_process.get('role')
    if role:
        class_kwargs['role'] = role
    
    function_inputs = []
    function_outputs = []
    function_return = None
    trait_count = 0 # used to order traits in class
    for child in xml_process:
        if child.tag in ('input', 'output'):
            n = child.get('name')
            trait = trait_from_xml(child, trait_count)
            trait_count += 1
            class_kwargs[n] = trait
            if trait.output and trait.input_filename is False:
                function_outputs.append(n)
            else:
                function_inputs.append(n)
        elif child.tag == 'return':
            if n:
                trait = trait_from_xml(child, trait_count)
                trait_count += 1
                class_kwargs[n] = trait
                function_return = n
            else:
                for parameter in child:
                    n = parameter.get('name')
                    trait = trait_from_xml(parameter, trait_count)
                    trait_count += 1
                    class_kwargs[n] = trait
                    function_outputs.append(n)
        else:
            raise ValueError('Invalid tag in <process>: %s' % child.tag)
    class_kwargs['_function'] = staticmethod(function)
    class_kwargs['_function_inputs'] = function_inputs
    class_kwargs['_function_outputs'] = function_outputs
    class_kwargs['_function_return'] = function_return
    
    # Get the process instance associated to the function
    process_class = type(str(name), (XMLProcess, ), class_kwargs)
    return process_class

def xml_process(xml):
    """ Decorator used to associate a Python function to its Process XML
    representation.    
    """
    def set_capsul_xml(function):
        function.capsul_xml = xml
        return function
    return set_capsul_xml

'''
Classes
=======

:class:`CapsulMainWindow`
'''

import os
import six

from soma.qt_gui.qt_backend import QtCore, QtGui, QtWebKit
from soma.qt_gui.controller_widget import ScrollControllerWidget

from capsul.qt_apps.utils.window import MyQUiLoader
from capsul.qt_apps.utils.fill_treectrl import fill_treectrl
from capsul.qt_gui.widgets import (PipelineDeveloperView, PipelineUserView)
from capsul.qt_gui.board_widget import BoardWidget
from capsul.api import get_process_instance
from capsul.study_config.study_config import StudyConfig

class CapsulMainWindow(MyQUiLoader):
    """ Capsul main window.
    """
    def __init__(self, pipeline_menu, ui_file, default_study_config=None):
        """ Method to initialize the Capsul main window class.

        pipeline_menu: hierachic dict
            each key is a sub module of the module. Leafs contain a list with
            the url to the documentation.
        ui_file: str (mandatory)
            a filename containing the user interface description
        default_study_config: ordered dict (mandatory)
            some parameters for the study configuration
        """
        # Inheritance: load user interface window
        MyQUiLoader.__init__(self, ui_file)

        self.pipeline_menu = pipeline_menu
        self.pipelines = {}
        self.path_to_pipeline_doc = {}

        # Define dynamic controls
        self.controls = {
            QtGui.QAction: ["actionHelp", "actionQuit", "actionBrowse",
                            "actionLoad", "actionChangeView",
                            "actionParameters", "actionRun",
                            "actionStudyConfig", "actionQualityControl"],
            QtGui.QTabWidget: ["display", ],
            QtGui.QDockWidget: ["dockWidgetBrowse", "dockWidgetParameters",
                                "dockWidgetStudyConfig", "dockWidgetBoard"],
            QtGui.QWidget: ["dock_browse", "dock_parameters",
                            "dock_study_config", "dock_board"],
            QtGui.QTreeWidget: ["menu_treectrl", ],
            QtGui.QLineEdit: ["search", ],
        }

        # Add ui class parameter with the dynamic controls and initialize
        self.add_controls_to_ui()
        self.ui.display.setTabsClosable(True)

        # Create the study configuration
        self.study_config = StudyConfig(default_study_config)

        # Create the controller widget associated to the study
        # configuration controller
        self.study_config_widget = ScrollControllerWidget(
            self.study_config, live=True)
        self.ui.dockWidgetStudyConfig.setWidget(self.study_config_widget)

        # Create the pipeline menu
        fill_treectrl(self.ui.menu_treectrl, self.pipeline_menu)

        # Signal for window interface
        self.ui.actionHelp.triggered.connect(self.onHelpClicked)
        self.ui.actionChangeView.triggered.connect(self.onChangeViewClicked)

        # Signal for tab widget
        self.ui.display.currentChanged.connect(self.onCurrentTabChanged)
        self.ui.display.tabCloseRequested.connect(self.onCloseTabClicked)

        # Signal for dock widget
        self.ui.actionBrowse.triggered.connect(self.onBrowseClicked)
        self.ui.actionParameters.triggered.connect(self.onParametersClicked)
        self.ui.actionStudyConfig.triggered.connect(self.onStudyConfigClicked)
        self.ui.actionQualityControl.triggered.connect(self.onQualityControlClicked)

        # Initialize properly the visibility of each dock widget
        self.onBrowseClicked()
        self.onParametersClicked()
        self.onStudyConfigClicked()
        self.onQualityControlClicked()

        # Signal for the pipeline creation
        self.ui.search.textChanged.connect(self.onSearchClicked)
        self.ui.menu_treectrl.currentItemChanged.connect(
            self.onTreeSelectionChanged)
        self.ui.actionLoad.triggered.connect(self.onLoadClicked)

        # Signal for the execution
        self.ui.actionRun.triggered.connect(self.onRunClicked)

        # Set default values

        # Set some tooltips

    def show(self):
        """ Shows the widget and its child widgets.
        """
        self.ui.show()

    def add_controls_to_ui(self):
        """ Method to find dynamic controls
        """
        # Error message template
        error_message = "{0} has no attribute '{1}'"

        # Got through the class dynamic controls
        for control_type, control_item in six.iteritems(self.controls):

            # Get the dynamic control name
            for control_name in control_item:

                # Try to set the control value to the ui class parameter
                try:
                    value = self.ui.findChild(control_type, control_name)
                    if value is None:
                        logger.error(error_message.format(
                            type(self.ui), control_name))
                    setattr(self.ui, control_name, value)
                    logger.error(error_message.format(
                        type(self.ui), control_name))

    ###########################################################################
    # Slots   

    def onRunClicked(self):
        """ Event to execute the process/pipeline.
        """
        self.study_config.run(self.pipeline, executer_qc_nodes=True, verbose=1)

    def onBrowseClicked(self):
        """ Event to show / hide the browse dock widget.
        """
        # Show browse dock widget
        if self.ui.actionBrowse.isChecked():
            self.ui.dockWidgetBrowse.show()

        # Hide browse dock widget
        else:
            self.ui.dockWidgetBrowse.hide()

    def onParametersClicked(self):
        """ Event to show / hide the parameters dock widget.
        """
        # Show parameters dock widget
        if self.ui.actionParameters.isChecked():
            self.ui.dockWidgetParameters.show()

        # Hide parameters dock widget
        else:
            self.ui.dockWidgetParameters.hide()

    def onStudyConfigClicked(self):
        """ Event to show / hide the study config dock widget.
        """
        # Show study configuration dock widget
        if self.ui.actionStudyConfig.isChecked():
            self.ui.dockWidgetStudyConfig.show()

        # Hide study configuration dock widget
        else:
            self.ui.dockWidgetStudyConfig.hide()

    def onQualityControlClicked(self):
        """ Event to show / hide the board dock widget.
        """
        # Create and show board dock widget
        if self.ui.actionQualityControl.isChecked():

            # Create the board widget associated to the pipeline controller
            # Create on the fly in order to get the last status
            # ToDo: add callbacks
            if self.pipeline is not None:
                # board_widget = BoardWidget(
                #     self.pipeline, parent=self.ui.dockWidgetParameters,
                #     name="board")
                board_widget = ScrollControllerWidget(
                    self.pipeline, name="outputs", live=True,
                    hide_labels=False, select_controls="outputs",
                    disable_controller_widget=True)
                #board_widget.setEnabled(False)
                self.ui.dockWidgetBoard.setWidget(board_widget)

            # Show the board widget
            self.ui.dockWidgetBoard.show()

        # Hide board dock widget
        else:
            self.ui.dockWidgetBoard.hide()

    def onSearchClicked(self):
        """ Event to refresh the menu tree control that contains the pipeline
        modules.
        """
        # Clear the current tree control
        self.ui.menu_treectrl.clear()

        # Build the new filtered tree control
        fill_treectrl(self.ui.menu_treectrl, self.pipeline_menu,
                      self.ui.search.text().lower())

    def onTreeSelectionChanged(self):
        """ Event to refresh the pipeline load button status.
        """
        # Get the current item
        item = self.ui.menu_treectrl.currentItem()
        if item is None:
            return

        # Check if we have selected a pipeline in the tree and enable / disable
        # the load button
        url = item.text(2)
        if url == "None":
            self.ui.actionLoad.setEnabled(False)
        else:
            self.ui.actionLoad.setEnabled(True)

    def onRunStatus(self):
        """ Event to refresh the run button status.

        When all the controller widget controls are correctly filled, enable
        the user to execute the pipeline.
        """
        # Get the controller widget
        controller_widget = self.ui.dockWidgetParameters.widget().controller_widget

        # Get the controller widget status
        is_valid = controller_widget.is_valid()

        # Depending on the controller widget status enable / disable
        # the run button
        self.ui.actionRun.setEnabled(is_valid)

    def onLoadClicked(self):
        """ Event to load and display a pipeline.
        """
        # Get the pipeline instance from its string description
        description_list = [str(x) for x in [item.text(1), item.text(0)]
                            if x != ""]
        process_description = ".".join(description_list)
        self.pipeline = get_process_instance(process_description)

        # Create the controller widget associated to the pipeline
        # controller
        pipeline_widget = ScrollControllerWidget(
            self.pipeline, live=True, select_controls="inputs")
        self.ui.dockWidgetParameters.setWidget(pipeline_widget)

        # Add observer to refresh the run button
        controller_widget = pipeline_widget.controller_widget
        for control_name, control \
                in six.iteritems(controller_widget._controls):

            # Unpack the control item
            trait, control_class, control_instance, control_label = control

            # Add the new callback
            control_class.add_callback(self.onRunStatus, control_instance)

        # Refresh manually the run button status the first time
        self.onRunStatus()

        # Store the pipeline documentation root path
        self.path_to_pipeline_doc[self.pipeline.id] = item.text(2)

        # Store the pipeline instance
        self.pipelines[self.pipeline.name] = (
            self.pipeline, pipeline_widget)

        # Create the widget
        widget = PipelineDeveloperView(self.pipeline)
        self._insert_widget_in_tab(widget)

        # Connect the subpipeline clicked signal to the
        # onLoadSubPipelineClicked slot
        widget.subpipeline_clicked.connect(self.onLoadSubPipelineClicked)

    def onLoadSubPipelineClicked(self, name, sub_pipeline, modifiers):
        """ Event to load and display a sub pipeline.
        """
        # Store the pipeline instance in class parameters
        self.pipeline = self.pipeline.nodes[name].process

        # Create the controller widget associated to the sub pipeline
        # controller: if the sub pipeline is a ProcessIteration, disable
        # the corresponding controller widget since this pipeline is generated
        # on the fly an is not directly synchronized with the rest of the
        # pipeline.
        is_iterative_pipeline = False
        if isinstance(self.pipeline, ProcessIteration):
            is_iterative_pipeline = True
            self.pipeline, live=True, select_controls="inputs",
            disable_controller_widget=is_iterative_pipeline)

        # Store the sub pipeline instance

    def onCloseTabClicked(self, index):
        """ Event to close a pipeline view.
        """
        # Remove the pipeline from the intern pipeline list
        pipeline, pipeline_widget = self.pipelines[
            self.ui.display.tabText(index)]
        pipeline_widget.close()
        pipeline_widget.deleteLater()
        del self.pipelines[self.ui.display.tabText(index)]

        # Remove the table that contains the pipeline
        self.ui.display.removeTab(index)

    def onCurrentTabChanged(self, index):
        """ Event to refresh the controller controller widget when a new
        tab is selected
        """
        # If no valid tab index has been passed
        if index < 0:
            self.ui.actionRun.setEnabled(False)

        # A new valid tab is selected
        else:
            # Get the selected pipeline widget
            self.pipeline, pipeline_widget = self.pipelines[
                self.ui.display.tabText(index)]

            # Set the controller widget associated to the pipeline
            # controller
            self.ui.dockWidgetParameters.setWidget(pipeline_widget)

            # Refresh manually the run button status the first time
            self.onRunStatus()

    def onHelpClicked(self):
        """ Event to display the documentation of the active pipeline.
        """
        # Create a dialog box to display the html documentation
        win = QtGui.QDialog()
        win.setWindowTitle("Pipeline Help")

        # Build the pipeline documentation location
        # Possible since common tools generate the sphinx documentation
        if self.pipeline:

            # Generate the url to the active pipeline documentation
            path_to_active_pipeline_doc = os.path.join(
                self.path_to_pipeline_doc[self.pipeline.id], "generated",
                self.pipeline.id.split(".")[1], "pipeline",
                self.pipeline.id + ".html")

            # Create and fill a QWebView
            help = QtWebKit.QWebView()
            help.load(QtCore.QUrl(path_to_active_pipeline_doc))
            help.show()

            # Create and set a layout with the web view
            layout = QtGui.QHBoxLayout()
            layout.addWidget(help)
            win.setLayout(layout)

            # Display the window
            win.exec_()

        # No Pipeline loaded, can't show the documentation message
        # Display a message box
        else:
            QtGui.QMessageBox.information(
                self.ui, "Information", "First load a pipeline!")

    def onChangeViewClicked(self):
        """ Event to switch between simple and full pipeline views.
        """
        # Check if a pipeline has been loaded
        if self._is_active_pipeline_valid():

            # Check the current display mode
            # Case PipelineDeveloperView
            if isinstance(self.ui.display.currentWidget(),
                          PipelineDeveloperView):

                # Switch to PipelineUserView display mode
                widget = PipelineUserView(self.pipeline)
                self._insert_widget_in_tab(widget)

            # Case PipelineUserView
            else:

                # Switch to PipelineDeveloperView display mode
                widget = PipelineDeveloperView(self.pipeline)

        # No pipeline loaded error
        else:
            logger.error("No active pipeline selected. "
                          "Have you forgotten to click the load pipeline "
                          "button?")

    #####################
    # Private interface #

    def _insert_widget_in_tab(self, widget):
        """ Insert a new widget or replace an existing widget.

        widget: a widget (mandatory)
            the widget we want to draw
        """
        # Search if the tab corresponding to the widget has already been created
        already_created = False
        index = 0

        # Go through all the tabs
        for index in range(self.ui.display.count()):

            # Check if we have a match: the tab name is equal to the current
            #pipeline name
            if (self.ui.display.tabText(index) == self.pipeline.name):
                already_created = True
                break

        # If no match found, add a new tab with the widget
        if not already_created:
            self.ui.display.addTab(
                widget, six.text_type(self.pipeline.name))
            self.ui.display.setCurrentIndex(
                self.ui.display.count() - 1)

        # Otherwise, replace the widget from the match tab
        else:
            # Delete the tab
            self.ui.display.removeTab(index)

            # Insert the new tab
            self.ui.display.insertTab(
                index, widget, six.text_type(self.pipeline.name))

            # Set the corresponding index
            self.ui.display.setCurrentIndex(index)

    def _is_active_pipeline_valid(self):
        """ Method to ceack that the active pipeline is valid

        Returns
        -------
        is_valid: bool
            True if the active pipeline is valid
        """
        return self.pipeline is not None

'''
Classes
=======
:class:`PipelineViewerApp`
'''

import os

from capsul.qt_apps.utils.application import Application
from capsul.qt_apps.main_window import CapsulMainWindow
from capsul.qt_apps.utils.find_pipelines import find_pipelines_from_description
import capsul.qt_apps.resources as resources
from capsul.plugins import PLUGS

class PipelineViewerApp(Application):
    """ CAPSULVIEW Application.
    """

    # Load some meta information
    from capsul.info import __version__ as _version
    from capsul.info import NAME as _application_name
    from capsul.info import ORGANISATION as _organisation_name

    def __init__(self, *args, **kwargs):
        """ Method to initialize the PipelineViewerApp class.
        """
        # Inhetritance
        super(PipelineViewerApp, self).__init__(*args, **kwargs)

        # Initialize the application
        self.window = None
        self.init_window()

    def init_window(self):
        """ Method to initialize the main window.
        """
        # First set some meta information
        self.setApplicationName(self._application_name)
        self.setOrganizationName(self._organisation_name)
        self.setApplicationVersion(self._version)

        # Get the user interface description from capsul resources
        ui_file = os.path.join(resources.__path__[0], "capsul.ui")

        # List capsul declared plugins (set of pipelines).
        if self.options.test:
            pipeline_menu = {
                "capsul": {
                    "utils": {
                        "test": {
                            "pipeline": {
                                "XmlPipeline": [""]
                            }
                        }
                    }
                }
            }
        else:
            pipeline_menu = {}
        for module_name, doc_url in PLUGS:
            pipeline_menu.update(
                find_pipelines_from_description(module_name, doc_url)[0])

        # Create and show the main window
        self.window = CapsulMainWindow(pipeline_menu, ui_file)
        self.window.show()

'''
Classes
=======
:class:`Application`
:class:`EmittingStream`
'''

import sys
import optparse
import warnings

from soma.qt_gui.qt_backend import Qt, QtCore, QtWidgets

class Application(QtWidgets.QApplication):
    """ Base Application class.

    Used to set some user options
    """

    def __init__(self, extra_options=None):
        """ Method to initialize the Application class.

        The capsulview application can be executed with command
        line options (that can also be passed to the class constructor
        as extra_options). From the command line, we can set the
        debug level with the -d option:
        * debug
        * info
        * warning
        * error
        * critical

        For example:
        >>> capsulview -d debug

        The default mode is error.

        From the command line we can also redirect all messages
        to a graphical message box with the -r option:
        >>> capsulview -r

        extra_options: list (optional)
            some additional options that are not passed through the command
            line.
        """
        QtWidgets.QApplication.__init__(self, [])

        # Extra application options
        extra_options = extra_options or []

        # Define a mapping to the logging level
        levels = {
            "debug": logging.DEBUG,
            "info": logging.INFO,
            "warning": logging.WARNING,
            "error": logging.ERROR,
            "critical": logging.CRITICAL
        }

        # Parse command line and internal options
        parser = optparse.OptionParser()
        parser.add_option("-d", "--debug", dest="debug",
                          help="Set the logging level "
                               "(debug, info, warning, error, or critical",
                          metavar="LEVEL")
        parser.add_option("-r", "--redirect-to-messagebox", dest="redirect",
                          action="store_true", default=False,
                          help="Redirect all messages to the console")
        parser.add_option("-t", "--test", dest="test",
                          help="Add a set of test pipelines")
        for args, kwargs in extra_options:
            parser.add_option(*args, **kwargs)
        self.options, self.arguments = parser.parse_args()

        # Logging format
        logging_format = ("[%(asctime)s] "
                          "{%(pathname)s:%(lineno)d} "
                          "%(levelname)s - %(message)s")
        date_format = "%Y-%m-%d %H:%M:%S"

        # If someone tried to log something before basicConfig is called,
        # Python creates a default handler that goes to the console and
        # will ignore further basicConfig calls: we need to remove the
        # handlers if there is one.
        while len(logging.root.handlers) > 0:
            logging.root.removeHandler(logging.root.handlers[-1])

        # If the logging level is specified
        if self.options.debug is not None:

            # Get the real logging level from the mapping
            level = levels.get(self.options.debug, None)

            # If a no valid logging level is found raise an Exception
            if level is None:
                raise Exception("Warning : unknown logging level "
                                "{0}".format(self.options.debug))

            # Configure the logging module
            logging.basicConfig(level=level, format=logging_format,
                                datefmt=date_format)           

            # Disable deprecation warnings if we are not in the debug mode
            if level != logging.DEBUG:
                warnings.simplefilter("ignore", DeprecationWarning)

        # Set the default logging level
        else:
            logging.basicConfig(level=logging.ERROR, format=logging_format,
                                datefmt=date_format)

            # Disable deprecation warnings
            warnings.simplefilter("ignore", DeprecationWarning)

        # Check if the redirection option is found: redirecect stdout and
        # stderr to a message box
        if self.options.redirect:

            # Create a message box
            self.message_box = Qt.QTextEdit()

            # Redirect stdout and stderr
            sys.stdout = EmittingStream()
            sys.stderr = EmittingStream()

            # Connect with text written signal
            self.connect(sys.stdout,
                         Qt.SIGNAL('textWritten(QString)'),
                         self._on_text_print)
            self.connect(sys.stderr,

            # Update root logger handler
            root_logger = logging.getLogger()
            h = root_logger.handlers[0]
            h.stream = sys.stdout

    def _on_text_print(self, text):
        """Append text to the QTextEdit.

        text: str (mandatory)
            the text to write
        """
        self.message_box.append(text)
        self.message_box.show()

class EmittingStream(QtCore.QObject):
    """ Logging emitting string basic handler.
    """
    def write(self, text):
        self.textWritten.emit(str(text))

'''
Functions
=========
:func:`fill_treectrl`
:func:`add_tree_nodes`
:func:`search_in_menu`
'''

import six
from soma.qt_gui.qt_backend import QtGui

# Global parameters
font = QtGui.QFont("", 9, QtGui.QFont.Bold)

def fill_treectrl(treectrl, menu, match=""):
    """ Fill a tree control with the different menu items.

    This procedure is able to filter the menu items.
    Loadable pipelines appear in bold in the tree control.
    Insert three elements (current module name, url, parent module name)
    When the url and then parent module name are different from 'None' we have
    reached a leaf that contain a pipeline description.

    treectrl: QTreeControl (mandatory)
        the tree control where we want to insert the menu
    menu: hierachic dict (mandatory)
        each key is a sub module of the module. Leafs contain a list with
        the url to the documentation.
    match: str (optional)
        the string used to filter the menu items
    """
    treectrl.headerItem().setText(0, "Pipelines")
    add_tree_nodes(treectrl, menu, match)

def add_tree_nodes(parent_item, menu, match, parent_module=""):
    """ Add the menu to tree control if match in current module name or
    child modules.

    The match is insensitive to the cast.

    parent_item: QTreeWidgetItem (mandatory)
        a tree control item where we want to insert the menu
    match: str (mandatory)
    parent_module: str (optional)
        the parent module string description ('module.sub_module')
    """
    # Go through the current module sub modules
    for module_name, child_modules in six.iteritems(menu):

        # Filtering: check if we need to add this module in the tree
        if (match == "" or match in module_name.lower() or
            search_in_menu(child_modules, match)):

            # Add the module name to the tree control
            if isinstance(child_modules, dict):
                tree_item = QtGui.QTreeWidgetItem(
                    parent_item, [module_name, "None", "None"])
                if parent_module:
                    current_module = parent_module + "." + module_name
                else:
                    current_module = module_name
                add_tree_nodes(tree_item, child_modules, match, current_module)
            else:
                    parent_item,
                    [module_name, parent_module, child_modules[0]])
                tree_item.setFont(0, font)

def search_in_menu(menu, match):
    """ Recursive search in tree.

    The search procedure is insensitive to the cast.

    Returns
    -------
    is_included: bool
        True if we found match in the tree, False otherwise.
    """
    # Initialize the default value: match not found
    is_included = False

    # If we are on a leaf, check in the module list
    if isinstance(menu, list):
        return is_included

        # Stop criteria
        if isinstance(child_modules, list):
            return is_included or match in module_name.lower()

        # Recursive search
        is_included = (
            is_included or match in module_name.lower() or
            search_in_menu(child_modules, match))

        if is_included:
            return is_included

    return is_included

'''
Functions
=========
:func:`find_pipelines_from_description`
:func:`find_pipeline_and_process`
:func:`lists2dict`
'''

import os
import sys
from setuptools import find_packages
from inspect import isclass

# CAPSUL import

def find_pipelines_from_description(module_name, url=None):
    """ Function that list all the pipeline of a module.

    module_name: str (mandatory)
        the name of the module we want to go through in order to find all
        pipeline classes.
    url: str (optional)
        the url to the module documentation.

    Returns
    -------
    structured_pipelines: hierachic dict
    pipelines: list
        a list a pipeline string descriptions.        
    """
    # Try to import the module
    try:
        __import__(module_name)
    except ImportError:
        logger.error("Can't load module {0}".format(module_name))
        return {}, []

    # Get the module path
    module = sys.modules[module_name]
    module_path = module.__path__[0]

    # Build the expected pipeline description file
    description_file = os.path.join(
        module_path, "{0}.capsul".format(module_name))

    # Load the description file
    if os.path.isfile(description_file):
        with open(description_file) as json_file:
            pipelines = json.load(json_file)

        # Organize the pipeline string description by module names
        structured_pipelines = {}
        lists2dict([x.split(".") for x in pipelines], url, structured_pipelines)

        return structured_pipelines, pipelines

    # No description found
    else:

def find_pipeline_and_process(module_name):
    """ Function that return all the Pipeline and Process classes of a module.

    All the mdoule path are scanned recuresively. Any pipeline or process will
    be added to the output.

    Returns
    -------
    output: dict
        a dictionary with a list of pipeline and process string descriptions
        found in the module.
    """

    try:

    if hasattr(module, '__path__'):
        module_path = module.__path__[0]
    else:
        module_path = os.path.dirname(module.__file__)

    # Use setuptools to go through the module
    sub_modules = find_packages(where=module_path, exclude=("doc", ))
    sub_modules = [module_name + "." + x for x in sub_modules]
    sub_modules.insert(0, module_name)
    logger.debug("Modules found with setuptools: '{0}'.".format(sub_modules))

    # Shift
    shift = len(module_name.split("."))

    # Create a set with all pipelines and process
    pip_and_proc = [set(), set()]
    for sub_module in sub_modules:
        # Get the sub module path
        sub_module_path = os.path.join(
            module_path, *sub_module.split(".")[shift:])

        # List all the module in submodule path
        sub_sub_module_names = [
            sub_module + "." + x[:-3] for x in os.listdir(sub_module_path)
            if (x.endswith(".py") and not x.startswith("_"))]

        # Try to import the sub sub module
        for sub_sub_module_name in sub_sub_module_names:
            try:
                __import__(sub_sub_module_name)
                exc_info = sys.exc_info()
                logger.error("".join(traceback.format_exception(*exc_info)))
                logger.error("Can't load module "
                              "{0}".format(sub_sub_module_name))
                continue

            # Get the module
            sub_sub_module = sys.modules[sub_sub_module_name]

            # From all the tools, find Pipeline instance
            for tool_name in dir(sub_sub_module):
                if tool_name.startswith("_"):
                    continue
                tool = getattr(sub_sub_module, tool_name)
                # Check all the authorized derived class
                parent_classes = [Pipeline, Process]
                for cnt, parent_class in enumerate(parent_classes):
                    if (isclass(tool) and issubclass(tool, parent_class)) \
                            and tool not in parent_classes:
                        pip_and_proc[cnt].add(
                            sub_sub_module_name + "." + tool_name)
                        break
    # Format output
    output = {
        "pipeline_descs": list(pip_and_proc[0]),
        "process_descs": list(pip_and_proc[1])
    }

    return output

def lists2dict(list_of_pipeline_description, url, d):
    """ Convert a list of split module names to a hierarchic dictionary with
    list leafs that contain the url to the module documentation.

    list_of_pipeline_description: list of list of str (mandatory)
        the split module names to organize by modules
    url: str (mandatory)
        the url to the module documentation

    Returns
    -------
    d: hierachic dict
    """
    # Go through all pipeline descriptions
    for l in list_of_pipeline_description:

        # Reach a leaf (a pipeline)
        if len(l) == 1:
            d.setdefault(l[0], []).append(url or "")

        # Continue the recursion
        else:
            if not l[0] in d:
                d[l[0]] = lists2dict([l[1:]], url, {})
            else:
                d[l[0]].update(lists2dict([l[1:]], url, d[l[0]]))

    return d

'''
Classes
=======
:class:`MyQUiLoader`
'''

from capsul.qt_apps.resources.icones import *

class MyQUiLoader(object):
    """ Base window class based on ui file description.
    """

    def __init__(self, uifile):
        """ Method to initialize the base window.

        uifile: str (mandatory)
            a filename containing the user interface description.
        """
        self.ui = qt_backend.loadUi(uifile)

'''
Classes
=======
:class:`BoardWidget`
'''

import six

from soma.qt_gui.qt_backend import QtGui, QtCore
from soma.qt_gui.controller_widget import ControllerWidget

from capsul.qt_gui.widgets.viewer_widget import ViewerWidget
from capsul.pipeline.pipeline_nodes import Switch, PipelineNode

class BoardWidget(QtGui.QWidget):
    """ Class that create a widget to visualize the controller status.
    """

    def __init__(self, controller, parent=None, name=None):
        """ Method to initialize the ControllerWidget class.

        controller: derived Controller instance (mandatory)
            a class derived from the Controller class we want to parametrize
            with a widget.
        parent: QtGui.QWidget (optional, default None)
            the controller widget parent widget.
        name: (optional, default None)
            the name of this controller widget
        """
        super(BoardWidget, self).__init__(parent)

        self.controller = controller

        # If possible, set the widget name
        if name:
            self.setObjectName(name)

        # Create the layout of the board widget
        # Three item layout: output_controller_widget - board_tree - viewer_tree
        self._grid_layout = QtGui.QGridLayout()
        self._grid_layout.setAlignment(QtCore.Qt.AlignTop)
        self._grid_layout.setSpacing(3)
        self._grid_layout.setContentsMargins(5, 5, 5, 5)
        self.setLayout(self._grid_layout)

        # Create all the controls
        self.create_output_widget()
        self.create_viewer_tree()
        self.create_board_tree()

        # Fill the grid layout
        self._grid_layout.addWidget(self.board_tree, 0, 0, 1, 2)
        self._grid_layout.addWidget(self.output_controller_widget, 1, 0, 1, 1)
        self._grid_layout.addWidget(self.viewer_tree, 1, 1, 1, 1)

        # Create the board
        self._fill_trees()

    # Methods   

    def create_output_widget(self):
        """ Method to create the output controller widget built 
        from the class controller output traits
        """
        self.output_controller_widget = ControllerWidget(
                self.controller, parent=self, name="outputs", live=True,
                hide_labels=False, select_controls="outputs")
        self.output_controller_widget.setEnabled(False)

    def create_viewer_tree(self):
        """ Method to create a tree with two columns (pipeline name - viewers)
        that will summarize all the available quality control nodes
        """
        # Create the tree widget
        self.viewer_tree = QtGui.QTreeWidget(parent=self)

        # Initialize the tree widget
        self.viewer_tree.setColumnCount(2)
        self.viewer_tree.headerItem().setText(0, "Pipeline Name")
        self.viewer_tree.headerItem().setText(1, "Viewers")

    def create_board_tree(self):
        """ Method to create a tree with five columns (processings - status -
        execution time - memory - logs) that will summarize all the available
        quality control nodes
        """
        self.board_tree = QtGui.QTreeWidget(parent=self)

        self.board_tree.setColumnCount(5)
        self.board_tree.headerItem().setText(0, "Processings")
        self.board_tree.headerItem().setText(1, "Status")
        self.board_tree.headerItem().setText(2, "Execution Time")
        self.board_tree.headerItem().setText(3, "Memory")
        self.board_tree.headerItem().setText(4, "Logs")

    # Private methods   

    def _title_for(self, title):
        """ Method to tune a title name.

        Juste replace the '_' character by a blank ' '.

        title: str (mandatory)
            the title name we want to tune.

        Returns
        -------
        output: str
            the tuned name
        """
        return title.replace("_", " ")

    def _fill_trees(self):
        """ Method to insert processing parameters in the class trees.
        """
        # Generate structures that contain all viewers and all processings
        # status - metainforamtion
        viewers_struct = {}
        processings_struct = []
    
        # Go through all the controller (pipeline) nodes.
        for node_name, node in six.iteritems(self.controller.nodes):

            # If the current node is a processing node
            if node_name != "" and node.node_type != "view_node":

                # First browse the current node to get processings and viewers
                process_nodes = []
                view_nodes = []
                self.browse_node(
                    node, process_nodes, view_nodes, self.controller)

                # Set process logs
                #for process_node in process_nodes:
                #    widget = LogWidget(process_node)
                #    widget.setParent(root.treeWidget())
                #    child.treeWidget().setItemWidget(child, 3, widget)

                # Fill the processing structure
                for processing_node in process_nodes:
                    processings_struct.append({
                        "name": processing_node.name,
                        "log": processing_node.process.log_file or "No log"})

                # Fill the viewer structure
                for viewer_node, pipeline in view_nodes:

                    # Create a viewer widget (a simple press button)
                    widget = ViewerWidget(viewer_node.name, pipeline, None)
                                          #self._study_config)

                    # Store the widget in the corresponding structure
                    title = self._title_for(pipeline.name)
                    if title not in viewers_struct:
                        viewers_struct[title] = []
                    viewers_struct[title].append(widget)

            # If the current node is a viewer node
            elif node.node_type == "view_node":

                # Create a viewer widget (a simple press button)
                widget = ViewerWidget(node_name, self.controller, None)
                                      #self._study_config)

                # Store the widget in the corresponding structure
                title = self._title_for(self.controller.name)
                if title not in viewers_struct:
                    viewers_struct[title] = []
                viewers_struct[title].append(widget)

        # Fill the viewer tree widget
        viewer_parent = self.viewer_tree.invisibleRootItem()
        for pipeline_title, viewer_widgets in six.iteritems(viewers_struct):

            # Create a new tree item
            viewer_child = QtGui.QTreeWidgetItem(viewer_parent)
            viewer_child.setText(0, pipeline_title)

            # Set the viewer widgets in a layout
            widget_layout = QtGui.QHBoxLayout()
            widget_layout.setSpacing(0)
            widget_layout.setSizeConstraint(QtGui.QLayout.SetMinimumSize)
            widget_layout.setContentsMargins(0, 0, 0, 0)
            widget_layout.addStretch(1)
            for widget in viewer_widgets:
                widget = QtGui.QToolButton()
                widget_layout.addWidget(widget)

            # Set the final widget tree item
            widget = QtGui.QWidget(viewer_child.treeWidget())
            widget.setLayout(widget_layout)
            viewer_child.treeWidget().setItemWidget(viewer_child, 1, widget)

        # Fill the pboard tree widget
        board_parent = self.board_tree.invisibleRootItem()
        for process_info in processings_struct:

            board_child = QtGui.QTreeWidgetItem(board_parent)
            board_child.setText(0, process_info["name"])
            board_child.setText(4, process_info["log"])

    def browse_node(self, node, process_nodes, view_nodes, parent_pipeline):
        """ Find view_node and leaf nodes, ie. Process nodes

        node: Node
            a capsul node
        process_nodes: Node
            node of type processing_node
        view_nodes: 2-uplet
            contains the node of type view_node and the pipeline where this node
            is defined
        """
        # Skip Switch nodes
        if not isinstance(node, Switch):

            # Browse recursively pipeline nodes
            if (isinstance(node.process, Pipeline) and 
                node.node_type != "view_node"):

                pipeline = node.process
                for sub_node in pipeline.nodes.values():
                    if not isinstance(sub_node, PipelineNode):
                        self.browse_node(sub_node, process_nodes, view_nodes,
                                         pipeline)
            # Update the results according to the node type
            else:
                if node.node_type == "view_node":
                    view_nodes.append((node, parent_pipeline))
                else:
                    process_nodes.append(node)

'''
A tool to debug and inspect nodes activation / deactivation in a pipeline.

Classes
=======
:class:`ActivationInspectorApp`
:class:`ActivationInspector`
'''

import os
import re
import six

from capsul.qt_gui.widgets import PipelineDeveloperView

class ActivationInspectorApp(Application):
    """ ActivationInspector Application.
    While developing a pipeline, nodes are connected through links. Nodes will be automatically activated or disabled depending on their connections (a mandatory link to a disabled node will disable the current one).
    You will often wonder why a node will not be activated. This tool helps to determine when and why by "playing" activation rules sequences step-by-step and displaying nodes which activate or deactivate at each step.
    """

    def __init__(self, pipeline_path, record_file=None, *args, **kwargs):
        """ Method to initialize the ActivationInspectorApp class.

        pipeline_path: str (mandatory)
            the name of the pipeline we want to load.
        record_file: str (optional)
            a file where the pipeline activation steps are stored.
        """
        super(ActivationInspectorApp, self).__init__(*args, **kwargs)

        # Load the pipeline
        self.pipeline = get_process_instance(pipeline_path)

        self.record_file = record_file

        """

        ui_file = os.path.join(
            os.path.dirname(__file__), "activation_inspector.ui")
        #ui_file = os.path.join(resources.__path__[0], "activation_inspector.ui")

        # Create and show the activation/pipeline/controller windows
        self.pipeline_window = PipelineDeveloperView(self.pipeline, show_sub_pipelines=True)
        self.controller_window = ScrollControllerWidget(self.pipeline,live=True)
        self.activation_window = ActivationInspector(
            self.pipeline, ui_file, self.record_file,
            developper_view=self.pipeline_window)
        self.pipeline_window.show()
        self.activation_window.show()
        self.controller_window.show()

class ActivationInspector(QtGui.QWidget):
    """ A Widget to display the pipeline activation process step by step.
    """
    def __init__(self, pipeline, ui_file=None, record_file=None,
                 developper_view=None, parent=None):
        """ Initialize the ActivationInspector class.

        pipeline: capsul.Pipeline (mandatory)
            the pipeline we want to inspect.
        ui_file: str (optional)
            the path to the qt user interface description file.
            If not specified, it will find its standard one.
            a file path where the activation steps are recorded.
            If not specified (None), it will create a temporary file.
        developper_view: PipelineDeveloperView (optional)
            if specified it is possible to click on a plug to set a filter
            pattern and to update the pipeline activation accordingly.
        """
        # Inheritance: create the application
        QtGui.QWidget.__init__(self, parent)

        # load the user interface window
        if ui_file is None:
            ui_file = os.path.join(
                os.path.dirname(__file__), "activation_inspector.ui")

        self.ui = qt_backend.loadUi(ui_file)

            QtGui.QListWidget: ["events"],
            QtGui.QPushButton: ["btnUpdate", "next", "previous"],
            QtGui.QLineEdit: ["pattern"]
        }

        if record_file is None:
            record_file_s = tempfile.mkstemp()
            record_file = record_file_s[1]
            os.close(record_file_s[0])
            print('temporary record file:', record_file)
            class AutoDeleteFile(object):
                def __init__(self, record_file):
                    self.record_file = record_file
                    try:
                        os.unlink(self.record_file)
                        pass
            self._autodelete_record_file = AutoDeleteFile(record_file)

        # Store class parameters
        self.pipeline = pipeline
        self.developper_view = developper_view

        # Set the pipeline record file if folder exists
        if os.path.isdir(os.path.dirname(self.record_file)):
            self.pipeline._debug_activations = self.record_file
        else:
                "The record file '{0}' can't be created since the "
                "base directory does not exist.".format(self.record_file))

        # Execute the pipeline activation method

        # Refresh the pipeline activation displayed list
        self.refresh_activation_from_record()

        # Signals for window interface
        self.ui.events.currentRowChanged.connect(
            self.update_pipeline_activation)
        self.ui.btnUpdate.clicked.connect(
            self.refresh_activation_from_record)
        self.ui.next.clicked.connect(self.find_next)
        self.ui.previous.clicked.connect(self.find_previous)

        # Dynamically select a filter rule by clicking on the pipeline view
        # plugs
        if developper_view is not None:
          developper_view.plug_clicked.connect(self.set_filter_pattern)

        """

        """

                try:

    # Slots   

    def refresh_activation_from_record(self):
        """ Method to display pipeline activation steps from the recorded file.
        """
        # Open the last recorded activation file
        with open(self.record_file) as openrecord:

            # Get the header of the file that contains the pipeline identifier
            # of the recorded activation
            record_pipeline_id = openrecord.readline().strip()
            if record_pipeline_id != self.pipeline.id:
                    "'{0}' recorded activations for pipeline '{1}' but not for "
                    "'{2}'".format(self.record_file, record_pipeline_id, 
                                   self.pipeline.id))

            # Clear the list where the recorded activation is displayed
            self.ui.events.clear()

            # Parse the recorded activation file

            # > Store the activation stack step by step to dynamically replay
            # the activation
            self.activations = []
            current_activations = {}

            # > Go through all the activation steps
            parser = re.compile(r"(\d+)([+-=])([^:]*)(:([a-zA-Z_0-9]+))?")
            for activation_step in openrecord.readlines():

                # > Parse the line
                iteration, activation, node, x, plug = parser.match(
                    activation_step.strip()).groups()
                plug = plug or ""

                # > Store the current activation stack
                if activation == "+":
                    current_activations["{0}:{1}".format(node, plug)] = True
                else:
                    del current_activations["{0}:{1}".format(node, plug)]
                self.activations.append(current_activations.copy())

                # > Add a line to the activation display
                self.ui.events.addItem("{0}{1} {2}:{3}".format(
                    iteration, activation, node, plug))

            # Select the last activation step so the pipeline will be
            # in his final configuration
            self.ui.events.setCurrentRow(self.ui.events.count() - 1)

    def update_pipeline_activation(self, index):
        """ Method that is used to replay the activation step by step.

        When a specific activation step is selected, the pipeline will reflect
        the selected activation status
        """
        # Get the activation associated to the 'index' stack level
        activations = self.activations[index]

        # Update the pipeline activation to meet the current selection
        for node in self.pipeline.all_nodes():

            # Restore the plugs and nodes activations
                plug.activated = activations.get(
                    "{0}:{1}".format(node_name, plug_name), False)
            node.activated = activations.get("{0}:".format(node_name), False)

    def find_next(self):
        """ Forward search for a pattern in the activation list.

        Returns
        -------
        is_found: int
            1 if a match has been found, 0 otherwise.
        """
        # Build the search pattern
        pattern = re.compile(self.ui.pattern.text())

        # Forward search
        # > Get the next (n+1) activation row
        next_activation_row = self.ui.events.currentRow() + 1
        # > Search recursively until a match is found
        while next_activation_row < self.ui.events.count():
            if pattern.search(self.ui.events.item(next_activation_row).text()):
                self.ui.events.setCurrentRow(next_activation_row)
                return 1
            next_activation_row += 1

        return 0
        
    def find_previous(self):
        """ Backward search for a pattern in the activation list.

        Returns
        -------
        is_found: bool
            True if a match has been found, False otherwise.
        """

        # Backward search
        # > Get the previous (n-1) activation row
        previous_activation_row = self.ui.events.currentRow() - 1
        while previous_activation_row > 0:
            if pattern.search(self.ui.events.item(previous_activation_row).text()):
                self.ui.events.setCurrentRow(previous_activation_row)
                return 1
            previous_activation_row -= 1

        return 0

    def set_filter_pattern(self, filter_pattern):
        """ Method that set a filter pattern.

        Try a forward search followed by a backward one.

        filter_pattern: str (mandatory)
            the filter pattern we want to set.
        """
        # Display the filter pattern
        self.ui.pattern.setText(filter_pattern)

        # Try to select an item corresponding to the filter pattern
        if not self.find_next():
            self.find_previous()

if __name__ == "__main__":
    """ Command example:
    >>> python activation_inspector.py -p funtk.connectivity.conn.Conn 
        -r ~/tmp/conn_activation.txt
    """
    # Create a tool we can control easily
    import sys
    from optparse import OptionParser

    # Define activation inspector options
    parser = OptionParser()
    parser.add_option("-p", "--pipeline",
                      dest="pipeline_path",
                      help=("the pipeline path we want to investigate: "
                            "module1.module2.Pipeline."))
    parser.add_option("-r", "--record",
                      dest="record_file",
                      help="the file where the activation steps are recorded.")
    sys.argv = []

    # Start the application
    if QtGui.QApplication.instance() is None:
        app = ActivationInspectorApp(options.pipeline_path, options.record_file)
        sys.exit(app.exec_())

'''
Process or pipeline parameters view with attributes handling.

Classes
=======
:class:`AttributedProcessWidget`
'''

import six
from soma.qt_gui.controller_widget \
    import ControllerWidget, ScrollControllerWidget
from traits.api import File, HasTraits, Any, Directory, Undefined, List

class AttributedProcessWidget(QtGui.QWidget):
    """Process interface with attributes completion handling"""
    def __init__(self, attributed_process, enable_attr_from_filename=False,
                 enable_load_buttons=False, override_control_types=None,
                 separate_outputs=True, user_data=None, userlevel=0,
                 scroll=True):
        """
        attributed_process: Process instance
            process with attributes to be displayed
        enable_attr_from_filename: bool (optional)
            if enabled, it will be possible to specify an input filename to
            build attributes from
        override_control_types: dict (optional)
            if given, this is a "factory" dict assigning new controller editor
            types to some traits types in the parameters controller.
        separate_outputs: bool
            if True, inputs and outputs (traits with output=True set) will
            be separated into two boxes.
        user_data: any type (optional)
            optional user data that can be accessed by individual control
            editors
        userlevel: int
            the current user level: some traits may be marked with a non-zero userlevel, and will only be visible if the ControllerWidget userlevel is more than (or equal) the trait level.
        scroll: bool
            if True, the widget includes scrollbars in the parameters and
            attributes sections when needed, otherwise it will be a fixed size
            widget.
        """
        super(AttributedProcessWidget, self).__init__()
        self.setLayout(QtGui.QVBoxLayout())
        self.layout().setContentsMargins(0, 0, 0, 0)
        self.attributed_process = attributed_process
        self._show_completion = False
        self.user_data = user_data
        self.separate_outputs = separate_outputs
        self._userlevel = userlevel

        process = attributed_process
        completion_engine = getattr(process, 'completion_engine', None)

            splitter = QtGui.QSplitter(QtCore.Qt.Vertical)
            self.layout().addWidget(splitter)
            spl_up = QtGui.QWidget()
            spl_up.setLayout(QtGui.QVBoxLayout())
            splitter.addWidget(spl_up)
            spl_down = QtGui.QWidget()
            spl_down.setLayout(QtGui.QVBoxLayout())
            splitter.addWidget(spl_down)
        else:
            spl_up = self
            spl_down = self

        filename_widget = None
        if enable_attr_from_filename and completion_engine is not None:
            c = Controller()
            c.add_trait('attributes_from_input_filename', File(optional=True))
            filename_widget = ControllerWidget(c, live=True,
                                               user_data=user_data)
            spl_up.layout().addWidget(filename_widget)
            self.input_filename_controller = c
            c.on_trait_change(self.on_input_filename_changed,
                              'attributes_from_input_filename', dispatch='ui')
            filename_widget.setSizePolicy(QtGui.QSizePolicy.Expanding,
                                          QtGui.QSizePolicy.Fixed)

        # groupbox area to show attributes
        attrib_widget = QtGui.QGroupBox('Attributes:')
        attrib_widget.setFlat(True)
        attrib_widget.setAlignment(QtCore.Qt.AlignLeft)
        attrib_widget.setLayout(QtGui.QVBoxLayout())
        self.attrib_widget = attrib_widget
        spl_up.layout().addWidget(attrib_widget)
        attrib_widget.setSizePolicy(QtGui.QSizePolicy.Expanding,
                                    QtGui.QSizePolicy.Preferred)

        hlay = QtGui.QHBoxLayout()
        spl_up.layout().addLayout(hlay)
        # CheckBox to completion rules or not
        self.checkbox_fom = QtGui.QCheckBox('Follow completion rules')
        self.checkbox_fom.setChecked(True)
        self.checkbox_fom.stateChanged.connect(self.on_use_fom_change)
        hlay.addWidget(self.checkbox_fom)

        # Button Show/Hide completion
        self.btn_show_completion = QtGui.QCheckBox('Show completion')
        self.btn_show_completion.setSizePolicy(QtGui.QSizePolicy.Fixed,
                                               QtGui.QSizePolicy.Fixed)
        hlay.addWidget(self.btn_show_completion)
        self.btn_show_completion.stateChanged.connect(self.on_show_completion)

        params = QtGui.QWidget()
        playout = QtGui.QVBoxLayout()
        params.setLayout(playout)
        if scroll:
            scroll_a = QtGui.QScrollArea()
            scroll_a.setWidgetResizable(True)
            scroll_a.setWidget(params)
            spl_up.layout().addWidget(scroll_a)
            scroll_a.setSizePolicy(QtGui.QSizePolicy.Preferred,
                                   QtGui.QSizePolicy.Preferred)
            params.setSizePolicy(QtGui.QSizePolicy.Expanding,
                                 QtGui.QSizePolicy.Preferred)
            CWidgetClass = ScrollControllerWidget
        else:
            spl_up.layout().addWidget(params)
            CWidgetClass = ControllerWidget

        # groupbox area to show completion
        if separate_outputs:
            param_widget = QtGui.QGroupBox('Inputs:')
        else:
            param_widget = QtGui.QGroupBox('Parameters:')
        param_widget.setFlat(True)
        param_widget.setAlignment(QtCore.Qt.AlignLeft)
        playout.addWidget(param_widget)
        param_widget.setLayout(QtGui.QVBoxLayout())
        param_widget.setSizePolicy(QtGui.QSizePolicy.Expanding,
                                   QtGui.QSizePolicy.Expanding)
            out_widget = QtGui.QGroupBox('Outputs:')
            out_widget.setFlat(True)
            out_widget.setAlignment(QtCore.Qt.AlignLeft)
            playout.addWidget(out_widget)
            out_widget.setLayout(QtGui.QVBoxLayout())
            out_widget.setSizePolicy(QtGui.QSizePolicy.Expanding,
                                     QtGui.QSizePolicy.Expanding)

        # use concise shape for lists GUI
        from  soma.qt_gui.controls import OffscreenListControlWidget
        control_types_a = {'List': OffscreenListControlWidget}
        control_types_p = {'List': OffscreenListControlWidget}
        if override_control_types:
            control_types_p.update(override_control_types)
        #ControllerWidget._defined_controls['List'] = OffscreenListControlWidget

        # Create controller widget for process and object_attribute
        sel = None
            sel = 'inputs'
        self.controller_widget = ControllerWidget(process, live=True,
            parent=param_widget, override_control_types=control_types_p,
            user_data=user_data, userlevel=userlevel, select_controls=sel)
            self.outputs_cwidget = ControllerWidget(process, live=True,
            parent=out_widget, override_control_types=control_types_p,
            user_data=user_data, userlevel=userlevel,
            select_controls='outputs')

        show_ce = (completion_engine is not None
                   and len(
                      completion_engine.get_attribute_values().user_traits())
                          != 0)

            self.controller_widget2 = CWidgetClass(
                completion_engine.get_attribute_values(),
                live=True, parent=attrib_widget,
                override_control_types=control_types_a, user_data=user_data,
                userlevel=userlevel)
            completion_engine.get_attribute_values().on_trait_change(
                completion_engine.attributes_changed, 'anytrait')
        else:
                Controller(), override_control_types=control_types_a,
                user_data=user_data, userlevel=userlevel)

        # Set controller of attributes and controller of process for each
        # corresponding area
        param_widget.layout().addWidget(self.controller_widget)
            out_widget.layout().addWidget(self.outputs_cwidget)
        attrib_widget.layout().addWidget(self.controller_widget2)

        if enable_load_buttons and completion_engine is not None:
            io_lay = QtGui.QHBoxLayout()
            self.layout().addLayout(io_lay)
            self.btn_load_json = QtGui.QPushButton('Load attributes')
            io_lay.addWidget(self.btn_load_json)
            self.btn_load_json.clicked.connect(self.on_btn_load_json)
            self.btn_save_json = QtGui.QPushButton('Save attributes')
            io_lay.addWidget(self.btn_save_json)
            self.btn_save_json.clicked.connect(self.on_btn_save_json)

        if not show_ce:
            if filename_widget:
                filename_widget.hide()
            attrib_widget.hide()
            self.checkbox_fom.hide()
            self.btn_show_completion.hide()
            if hasattr(self, 'btn_load_json'):
                self.btn_load_json.hide()
                self.btn_save_json.hide()
            self.show_completion(True) # hide file parts
        else:
            self.show_completion(False) # hide file parts

            completion_engine.on_trait_change(
                self._completion_progress_changed, 'completion_progress',
                dispatch='ui')

        completion_engine = getattr(self.attributed_process,
                                   'completion_engine', None)
                completion_engine.attributes_changed, 'anytrait', remove=True)
                remove=True)

    @property
    def userlevel(self):
        return getattr(self, '_userlevel', 0)

    @userlevel.setter
    def userlevel(self, value):
        self._userlevel = value
        cw = getattr(self, 'controller_widget', None)
        if cw:
            cw.userlevel = value
        cw = getattr(self, 'outputs_cwidget', None)
        if cw:
        cw = getattr(self, 'controller_widget2', None)
        if cw:
        # re-hide file params if needed
        self.show_completion(self._show_completion)

    def on_input_filename_changed(self, text):
        '''
        Input file path to guess completion attributes changed: update
        attributes
        '''
            print('set attributes from path:', text)
            try:
                completion_engine.path_attributes(six.text_type(text))
            except ValueError as e:
                print(e)
                traceback.print_stack()

    def on_btn_load_json(self):
        """Load attributes from a json file"""
        if completion_engine is None:
            print('No completion engine with attributes in this process.')
            return
        # ask for a file name
        filename = qt_backend.getOpenFileName(
            self, 'Select a .json attributes file', '',
            'JSON files (*.json)')
        if filename is None:
            return
        print('load', filename)
        attributes = json.load(open(filename))
        print("loaded:", attributes)

    def on_btn_save_json(self):
        """Save attributes in a json file"""
            print('No attributes in this process.')
            return
        filename = qt_backend.getSaveFileName(
            return
        json.dump(completion_engine.get_attribute_values().export_to_dict(),
                  open(filename, 'w'))

    def set_use_fom(self):
        '''
        Setup the FOM doing its job
        '''
        ret = QtGui.QMessageBox.critical(self, "Critical",
            'Going back to completion rules will reset all path files. '
            'Are you sure?',
            QtGui.QMessageBox.Ok, QtGui.QMessageBox.Cancel)

        if ret == QtGui.QMessageBox.Ok:
            #reset attributes and trait of process
            process = self.attributed_process
            completion_engine = getattr(self.attributed_process,
                                       'completion_engine', None)
            if completion_engine is None:
                return
            try:
                # WARNING: is it necessary to reset all this ?
                # create_completion() will do the job anyway ?
                #for name, trait in six.iteritems(process.user_traits()):
                    #if trait.is_trait_type(File) \
                            #or trait.is_trait_type(Directory):
                        #setattr(process,name, Undefined)
                completion_engine.complete_parameters()

                if hasattr(self, 'input_filename_controller') \
                        and self.input_filename_controller. \
                            attributes_from_input_filename \
                        != '':
                    completion_engine.path_attributes(
                        self.input_filename_controller.attributes_from_input_filename)
                print(e)
            self.attrib_widget.show()

        else:
            # reset it in a timer callback, otherwise the checkbox state is not
            # correctly recorded, and next time its state change will not
            # trigger the on_use_fom_change slot.
            QtCore.QTimer.singleShot(0, self._reset_fom_checkbox)

    def _reset_fom_checkbox(self):
        self.checkbox_fom.setChecked(False)

    def on_use_fom_change(self, state):
        '''
        Use completion checkbox callback
        '''
        if state == QtCore.Qt.Checked:
            self.set_use_fom()
        else:
            self.attrib_widget.hide()
                                        'completion_engine', None)
            if completion_engine is not None:
                completion_engine.get_attribute_values().on_trait_change(
                    completion_engine.attributes_changed, 'anytrait',
                    remove=True)
                self.btn_show_completion.setChecked(True)

    def show_completion(self, visible=None):
        '''
        Show or hide completion (File, Directory, or Any parameters)

        visible: bool (optional)
            show/hide. If None, switch the current visibility state.
        '''

        if visible is None:
            visible = not self._show_completion
        self._show_completion = visible
        cwidgets = [self.controller_widget]
        if self.separate_outputs:
            cwidgets.append(self.outputs_cwidget)
        for controller_widget in cwidgets:
            for control_name, control_groups in \
                    six.iteritems(
                        controller_widget._controls):
                for group, control in six.iteritems(control_groups):
                    trait, control_class, control_instance, control_label \
                        = control
                    if not isinstance(trait.trait_type,
                                      (File, Any, Directory)) \
                            and (not isinstance(trait.trait_type, List)
                                or not isinstance(
                                    trait.inner_traits[0].trait_type,
                                    (File, Directory, Any))):
                        continue
                    if trait.forbid_completion:
                        # when completion is disable, parameters are always
                        # visible
                        is_visible = True
                    else:
                        hidden = trait.hidden \
                              or (trait.userlevel is not None
                                  and trait.userlevel > self.userlevel)
                        is_visible = visible and not hidden
                    control_instance.setVisible(is_visible)
                    if isinstance(control_label, tuple):
                        for cl in control_label:
                            cl.setVisible(is_visible)
                    else:
                        control_label.setVisible(is_visible)
            for group, group_widget in six.iteritems(
                    controller_widget._groups):
                if [x for x in group_widget.hideable_widget.children()
                    if isinstance(x, QtGui.QWidget) and not x.isHidden()]:
                    group_widget.show()
                else:
                    group_widget.hide()

    def on_show_completion(self, visible):
        '''
        Toggle the visibility of paths parameters
        '''
        self.show_completion(visible)

    def _completion_progress_changed(self, obj, name, old, new):
                                    'completion_engine', None)
            if not hasattr(self, 'progressdialog'):
                self.progressdialog = QtGui.QWidget()
                self.layout().insertWidget(1, self.progressdialog)
                layout = QtGui.QHBoxLayout()
                self.progressdialog.setLayout(layout)
                layout.addWidget(QtGui.QLabel('Completion progress:'))
                self.progressbar = QtGui.QProgressBar()
                layout.addWidget(self.progressbar)
                self.progressbar.setRange(0, 100)
            value = int(round(100 * completion_engine.completion_progress
                        / completion_engine.completion_progress_total))
            self.progressbar.setValue(value)
            if value != 100:
                self.progressdialog.show()
                QtGui.qApp.processEvents()
            else:
                self.progressdialog.hide()

'''
Tool to debug and understand process / pipeline parameters links

Classes
=======
:class:`CapsulLinkDebuggerView`
'''

import os
import re

class CapsulLinkDebuggerView(QtGui.QWidget):
    """ A Widget to display the links propagation when values are set in a
    pipeline
    """
    CAUSE = 1
    PLUG = 2
    PROPAGATE = 3
    VALUE = 4

    def __init__(self, pipeline, ui_file=None, record_file=None, parent=None):
        super(CapsulLinkDebuggerView, self).__init__(parent)

                os.path.dirname(__file__), "links_debugger.ui")

        self.ui.help.hide()
        self.ui.help.setParent(None)
        self.help_geom = None
        table_header = self.ui.links_table.horizontalHeader()
        table_header.setResizeMode(QtGui.QHeaderView.ResizeToContents)
        table_header_v = self.ui.links_table.verticalHeader()
        table_header_v.setResizeMode(QtGui.QHeaderView.ResizeToContents)

                    try:
                        pass

        self.set_pipeline(pipeline)
        self.update_links_view()
        self.ui.links_table.cellClicked.connect(self.activateCell)
        self.ui.actionPrevious.activated.connect(self.go_previous_line)
        self.ui.actionNext.activated.connect(self.go_next_line)
        self.ui.actionFollow.activated.connect(self.go_follow_link)
        self.ui.actionRefresh.activated.connect(self.update_links_view)
        self.ui.actionClear.activated.connect(self.clear_view)
        self.ui.actionHelp.activated.connect(self.help)

        self.release_pipeline()

        """

    def set_pipeline(self, pipeline):
        if self.pipeline is not None:
            self.release_pipeline()
        pipeline.uninstall_links_debug_handler()
        record_stream = pipeline.install_links_debug_handler(
            log_file=open(self.record_file, 'w'), handler=None, prefix='')
        self.record_stream = record_stream

    def release_pipeline(self):
            self.pipeline.uninstall_links_debug_handler()

    def update_links_view(self):
        self.ui.links_table.clearContents()
        self.ui.links_table.setRowCount(0)  # IMPORTANT otherwise perf drops
        table_header.setResizeMode(QtGui.QHeaderView.Interactive)
        l = 0
        self.record_stream.flush()
        f = open(self.record_file)
        lines = f.readlines()
        self.ui.links_table.setRowCount(len(lines))
        linkre = re.compile('^value link: from: ([^ ,]+) *to: ([^ ]+) *, value: ([^ ]+).*$')
        links_orgs = {}
        for line in lines:
            match = linkre.match(line)
            if match:
                link_source = match.group(1)
                link_dest = match.group(2)
                plug_value = match.group(3)
                self.ui.links_table.setItem(
                    l, 0, QtGui.QTableWidgetItem('%04d' % l))
                    l, self.PLUG, QtGui.QTableWidgetItem(link_source))
                    l, self.PROPAGATE, QtGui.QTableWidgetItem(link_dest))
                    l, self.VALUE, QtGui.QTableWidgetItem(plug_value))
                links_orgs.setdefault(link_dest, []).append(l)
                if link_source in links_orgs:
                    org = links_orgs[link_source][0]
                    self.ui.links_table.setItem(
                        l, self.CAUSE,
                        QtGui.QTableWidgetItem(
                            self.ui.links_table.item(org, 2)))
                l += 1
        self.links_orgs = links_orgs
        #table_header.setResizeMode(1, QtGui.QHeaderView.ResizeToContents)
        #table_header.setResizeMode(2, QtGui.QHeaderView.ResizeToContents)
        #table_header.setResizeMode(3, QtGui.QHeaderView.ResizeToContents)
        #table_header.setResizeMode(4, QtGui.QHeaderView.ResizeToContents)
        #table_header.setResizeMode(QtGui.QHeaderView.Interactive)
        QtGui.qApp.processEvents()
        #table_header.resizeSection(0, table_header.sectionSizeHint(0))
        #table_header.resizeSection(1, table_header.sectionSizeHint(1))
        #table_header.resizeSection(2, table_header.sectionSizeHint(2))
        #table_header.resizeSection(3, table_header.sectionSizeHint(3))
        #table_header.resizeSection(4, table_header.sectionSizeHint(4))

    def clear_view(self):
        self.ui.links_table.setRowCount(0)
        self.record_stream.seek(0)
        self.record_stream.truncate(0)

    def activateCell(self, row, column):
        if self.ui.links_table.item(row, column) is None:
            return
        if column == self.CAUSE:
            self.go_previous(row)
        elif column == self.PLUG:
            self.highlight_plug(row)
        elif column == self.PROPAGATE:
            self.go_next(row)
        elif column == self.VALUE:
            self.highlight_value(row)

    def go_previous(self, row):
        self.ui.links_table.clearSelection()
        plug = self.ui.links_table.item(row, self.PLUG)
        plug.setSelected(True)
        self.ui.links_table.item(row, self.CAUSE).setSelected(True)
        items = self.ui.links_table.findItems(
            plug.text(), QtCore.Qt.MatchExactly)
        for item in items:
            if item.column() == self.PROPAGATE:
                item.setSelected(True)

    def go_next(self, row):
        next_item = self.ui.links_table.item(row, self.PROPAGATE)
        if next_item:
            next_item.setSelected(True)
            items = self.ui.links_table.findItems(
                next_item.text(), QtCore.Qt.MatchExactly)
            for item in items:
                if item.column() == self.PLUG and item.row() >= row:
                    item.setSelected(True)

    def highlight_plug(self, row):
        if plug:
            plug.setSelected(True)
                plug.text(), QtCore.Qt.MatchExactly)
                if item.column() in (self.CAUSE, self.PROPAGATE):

    def highlight_value(self, row):
        value_item = self.ui.links_table.item(row, self.VALUE)
        if value_item:
            value_item.setSelected(True)
                value_item.text(), QtCore.Qt.MatchExactly)
                if item.column() == self.VALUE:

    def go_next_line(self):
        row = self.ui.links_table.currentRow() + 1
        self.ui.links_table.setCurrentCell(row, self.PLUG)
        self.highlight_plug(row)

    def go_previous_line(self):
        row = self.ui.links_table.currentRow() - 1
        if row >= 0:
            self.ui.links_table.setCurrentCell(row, self.PLUG)

    def go_follow_link(self):
        row = self.ui.links_table.currentRow()
        plug = self.ui.links_table.item(row, self.PROPAGATE)
        if plug:
                if item.column() == self.PLUG and item.row() > row:
                    self.ui.links_table.setCurrentCell(item.row(), self.PLUG)
                    break
        self.go_next(row)

    def help(self):
        if self.help_geom is None or self.ui.help.isVisible():
            self.help_geom = self.ui.help.geometry()
            set_geom = False
        else:
            set_geom = True
        self.ui.help.setVisible(not self.ui.help.isVisible())
        if set_geom:
            self.ui.help.setGeometry(self.help_geom)

'''
A Pipeline structure viewer widget, which displays pipeline nodes as boxes and links as lines, and provides pipeline editor features.

The only main class you should look at is the :class:`PipelineDeveloperView` widget, the remaining is internal infrastructure::

    pv = PipelineDeveloperView(pipeline, allow_open_controller=True,
                                enable_edition=True,show_sub_pipelines=True)
    pv.show()

Classes
=======
:class:`ColorType`
:class:`EmbeddedSubPipelineItem`
:class:`boxItem`
----------------
:class:`NodeGWidget`
:class:`HandleItem`
:class:`Link`
:class:`PipelineScene`
:class:`PipelineDeveloperView`

'''

import os
from pprint import pprint
import sys
import inspect
import six
import io
import html

from soma.qt_gui.qt_backend import QtCore, QtGui, Qt
from soma.qt_gui.qt_backend.Qt import QMessageBox
from capsul.api import Switch, PipelineNode, OptionalOutputSwitch
from capsul.pipeline.pipeline_nodes import Node, ProcessNode
from soma.qt_gui.qt_backend.Qt import QGraphicsView
from capsul.qt_gui.widgets.pipeline_file_warning_widget \
    import PipelineFileWarningWidget
import capsul.pipeline.xml as capsulxml
from capsul.study_config import process_instance
from soma.utils.weak_proxy import proxy_method

from traits import api as traits

qt_backend.init_traitsui_handler()

from capsul.qt_gui.widgets.attributed_process_widget \
    import AttributedProcessWidget

# -----------------------------------------------------------------------------
# Globals and constants

GRAY_1 = QtGui.QColor.fromRgbF(0.7, 0.7, 0.8, 0.1)
GRAY_2 = QtGui.QColor.fromRgbF(0.4, 0.4, 0.4, 1)
LIGHT_GRAY_1 = QtGui.QColor.fromRgbF(0.2, 0.2, 0.2, 1)
LIGHT_GRAY_2 = QtGui.QColor.fromRgbF(0.2, 0.2, 0.2, 1)

# Colors for links and plugs

ORANGE_1 = QtGui.QColor.fromRgb(220, 80, 20)
ORANGE_2 = QtGui.QColor.fromRgb(220, 120, 20)
BLUE_1 = QtGui.QColor.fromRgb(50, 150, 250)
BLUE_2 = QtGui.QColor.fromRgb(50, 50, 250)
PURPLE_2 = QtGui.QColor.fromRgb(200, 0, 200)
RED_2 = QtGui.QColor.fromRgb(200, 0, 0)
GREEN_2 = QtGui.QColor.fromRgb(0, 100, 0)
BLACK_2 = QtGui.QColor.fromRgb(10, 10, 10)
WHITE_2 = QtGui.QColor.fromRgb(255, 255, 255)

ANTHRACITE_1 = QtGui.QColor.fromRgbF(0.05, 0.05, 0.05)
LIGHT_ANTHRACITE_1 = QtGui.QColor.fromRgbF(0.25, 0.25, 0.25)

# Classes and functions

class ColorType(object):

        pass

    def colorLink(self, x):
        if not isinstance(x, str):
            # x is a trait
            trait_type_str = x.trait_type.__class__.__name__
            if x.output and x.input_filename is False:
                trait_type_str = 'File_out'
            x = trait_type_str
        return {
            'Str': PURPLE_2,
            'Float': ORANGE_1,
            'Int': BLUE_2,
            'List': RED_2,
            'File': ORANGE_2,
            'File_out': GREEN_2,
        }[x]

class Plug(QtGui.QGraphicsPolygonItem):

    def __init__(self, color, name, height, width, activated=True,
                 optional=False, parent=None):
        super(Plug, self).__init__(parent)
        #         self.color = self._color(activated, optional)
        self.color = color
        if optional:
            brush = QtGui.QBrush(QtCore.Qt.SolidPattern)
            brush.setColor(self.color)
            polygon = QtGui.QPolygonF([QtCore.QPointF(0, 0),
                                       QtCore.QPointF(width / 1.5, 0),
                                       QtCore.QPointF(width / 1.5,
                                                      (height - 5)),
                                       QtCore.QPointF(0, (height - 5))
                                       ])
        #             self.setPen(QtGui.QPen(QtCore.Qt.NoPen))
        else:
                                       QtCore.QPointF(
                                           width, (height - 5) / 2.0),
                                       QtCore.QPointF(0, height - 5)
                                       ])
        self.setPolygon(polygon)
        self.setBrush(brush)
        self.setZValue(3)
        self.setAcceptedMouseButtons(QtCore.Qt.LeftButton)

    #     def _color(self, activated, optional):
    #         if optional:
    #             if activated:
    #                 color = QtCore.Qt.darkGreen
    #             else:
    #                 color = QtGui.QColor('#BFDB91')
    #         else:
    #                 color = QtCore.Qt.black
    #             else:
    #                 color = QtCore.Qt.gray
    #         return color

    #     def update_plug(self, activated, optional):
    #         color = self._color(activated, optional)
    #         brush = QtGui.QBrush(QtCore.Qt.SolidPattern)
    #         brush.setColor(color)
    #         self.setBrush(brush)

    def update_plug(self, color):
        brush = QtGui.QBrush(QtCore.Qt.SolidPattern)
        brush.setColor(color)

    def get_plug_point(self):
        point = QtCore.QPointF(
            self.boundingRect().size().width() / 2.0,
            self.boundingRect().size().height() / 2.0)
        return self.mapToParent(point)

    def mousePressEvent(self, event):
        super(Plug, self).mousePressEvent(event)
        if event.button() == QtCore.Qt.LeftButton:
            self.scene().plug_clicked.emit(self.name)
            event.accept()
        elif event.button() == QtCore.Qt.RightButton:
            # print('plug: right click')
            self.scene().plug_right_clicked.emit(self.name)

class EmbeddedSubPipelineItem(QtGui.QGraphicsProxyWidget):
    '''
    QGraphicsItem containing a sub-pipeline view
    '''

    def __init__(self, sub_pipeline_wid):
        super(EmbeddedSubPipelineItem, self).__init__()
        old_height = sub_pipeline_wid.sizeHint().height()
        sizegrip = QtGui.QSizeGrip(None)
        new_height = old_height \
                     + sub_pipeline_wid.horizontalScrollBar().height()
        sub_pipeline_wid.setCornerWidget(sizegrip)
        sub_pipeline_wid.setHorizontalScrollBarPolicy(
            QtCore.Qt.ScrollBarAlwaysOn)
        sub_pipeline_wid.resize(sub_pipeline_wid.sizeHint().width(), new_height)
        self.setWidget(sub_pipeline_wid)

class boxItem(QtGui.QGraphicsRectItem):
    def __init__(self, parent=None):
        super(boxItem, self).__init__(parent)
        #         self.setFlags(self.ItemIsFocusable)
        self.penBox = 0;
        self.name = ""

    def focusInEvent(self, event):
        self.setPen(QtGui.QPen(QtGui.QColor(150, 150, 250), 3, QtCore.Qt.DashDotLine))
        return QtGui.QGraphicsRectItem.focusInEvent(self, event)

    def focusOutEvent(self, event):
        self.setPen(self.penBox)
        return QtGui.QGraphicsRectItem.focusOutEvent(self, event)

    def keyPressEvent(self, event):
        if event.key() == QtCore.Qt.Key_Delete:
            self.scene()._node_keydelete_clicked(self)
        else:
            super(boxItem, self).keyPressEvent(event)

class NodeGWidget(QtGui.QGraphicsItem):
    def __init__(self, name, parameters, pipeline,
                 parent=None, process=None, sub_pipeline=None,
                 colored_parameters=True,
                 logical_view=False, labels=[],
                 show_opt_inputs=True, show_opt_outputs=True,
                 userlevel=0):
        super(NodeGWidget, self).__init__(parent)

        self.infoActived = QtGui.QGraphicsTextItem('', self)
        self.colType = ColorType()

        self.setFlags(self.ItemIsSelectable)
        self.setCursor(Qt.QCursor(QtCore.Qt.PointingHandCursor))

        self.style = 'default'
        #print('GNode userlevel:', self.userlevel)
        #print([(pname, param) for pname, param in six.iteritems(parameters)
             #if not getattr(param, 'hidden', False)
             #and (getattr(param, 'userlevel', None) is None
                  #or param.userlevel <= self.userlevel)]
            controller = process.process
        else:
            controller = process
        self.parameters = SortedDictionary()
        for pname, param in six.iteritems(parameters):
            show = True
            if controller:
                trait = controller.trait(pname)
                if getattr(trait, 'hidden', False):
                    show = False
                elif getattr(trait, 'userlevel', None) is not None:
                    if trait.userlevel > self.userlevel:
                        show = False
            if show:
                self.parameters[pname] = param

        self.setFlag(QtGui.QGraphicsItem.ItemIsMovable)
        self.in_plugs = SortedDictionary()
        self.in_params = {}
        self.out_plugs = SortedDictionary()
        self.out_params = {}
        self.sub_pipeline = sub_pipeline
        self.embedded_subpipeline = None
        self.colored_parameters = colored_parameters
        self.logical_view = logical_view

        # Added to choose to visualize optional parameters
        self.show_opt_inputs = show_opt_inputs
        self.show_opt_outputs = show_opt_outputs

        self.labels = []
        self.scene_labels = labels
        self.label_items = []
        my_labels = []
        steps = getattr(pipeline, 'pipeline_steps', None)
        if steps:
            for step_name, step in six.iteritems(steps.user_traits()):
                step_nodes = step.nodes
                if name in step_nodes:
                    my_labels.append('step: %s' % step_name)
        selects = pipeline.get_processes_selections()
        for sel_plug in selects:
            groups = pipeline.get_processes_selection_groups(sel_plug)
            for group, nodes in six.iteritems(groups):
                if name in nodes:
                    my_labels.append('select: %s' % sel_plug)

        for label in my_labels:
            self._get_label(label)

        self._set_brush()
        self.setAcceptedMouseButtons(
            QtCore.Qt.LeftButton | QtCore.Qt.RightButton | QtCore.Qt.MiddleButton)

        self._build()

        self._update_param_timer = Qt.QTimer()
        self._update_param_timer.setSingleShot(True)
        self._update_param_timer.timeout.connect(
            proxy_method(self, 'update_parameters_now'))

        if colored_parameters:
            process.on_trait_change(self._repaint_parameter, dispatch='ui')
        process.on_trait_change(self.update_parameters, 'user_traits_changed',
                                dispatch='ui')

        # print('NodeGWidget.__del__')
        self._release()
        # super(NodeGWidget, self).__del__()

    @property
        return self._userlevel

        self.update_parameters()

    def _release(self):
        # release internal connections / callbacks / references in order to
        # allow deletion of self
        self.process.on_trait_change(self.update_parameters,
                                     'user_traits_changed', remove=True)
        if self.colored_parameters:
            try:
                self.process.on_trait_change(self._repaint_parameter, remove=True)
                pass
            self.colored_parameters = None
        self.sizer = None

    def get_title(self):
        if self.sub_pipeline is None:
        else:
            return "[{0}]".format(self.name)

    def update_parameters(self):
        self._update_param_timer.start(20)

    def update_parameters_now(self):
        if isinstance(self.process, ProcessNode):
            controller = self.process.process
        else:
            controller = self.process
        for pname, param in six.iteritems(self.process.user_traits()):
            show = True
            if self.name == 'inputs' and param.output:
                continue
            elif self.name == 'outputs' and not param.output:
                continue
            if show:

        self.update_node()

    def update_labels(self, labels):
        ''' Update colored labels
        '''
        for item in self.label_items:
            item.deleteLater()  # FIXME there should be another way !
        for label in labels:
        self._create_label_marks()

    def _get_label(self, label, register=True):
        class Label(object):
            def __init__(self, label, color):
                self.text = label
                self.color = color

        for l in self.scene_labels:
            if label == l.text:
                if register and l not in self.labels:
                    self.labels.append(l)
                return l
        color = self.new_color(len(self.scene_labels))
        label_item = Label(label, color)
        if register:
            self.labels.append(label_item)
        self.scene_labels.append(label_item)
        return label_item

    def new_color(self, num):
        colors = [[1, 0.3, 0.3],
                  [0.3, 1, 0.3],
                  [0.3, 0.3, 1],
                  [1, 1, 0],
                  [0, 1, 1],
                  [1, 0, 1],
                  [1, 1, 1],
                  [1, 0.7, 0],
                  [1, 0, 0.7],
                  [1, 0.7, 0.7],
                  [0.7, 1, 0],
                  [0., 1., 0.7],
                  [0.7, 1, 0.7],
                  [0.7, 0, 1],
                  [0., 0.7, 1],
                  [0.7, 0.7, 1],
                  [1, 1, 0.5],
                  [0.5, 1, 1],
                  [1, 0.5, 1]]
        c = colors[num % len(colors)]
        code = (int(c[0] * 255), int(c[1] * 255), int(c[2] * 255))
        return code

    def _repaint_parameter(self, param_name, new_value):
        if self.logical_view or param_name not in self.parameters:
            return
        param_text = self._parameter_text(param_name)
        param_item = self.in_params.get(param_name)
        if param_item is None:
            param_item = self.out_params[param_name]
        if isinstance(param_item, QtGui.QGraphicsProxyWidget):
            # colored parameters are widgets
            param_item.widget().findChild(
                QtGui.QLabel, 'label').setText(param_text)
        else:
            param_item.setHtml(param_text)

    def _build(self):
        margin = 0
        self.title = QtGui.QGraphicsTextItem(self.get_title(), self)
        #         font = self.title.font()
        font = QtGui.QFont("Times", 11, QtGui.QFont.Bold)
        #         font.setWeight(QtGui.QFont.Bold)
        self.title.setFont(font)
        self.title.setPos(margin, margin)
        self.title.setZValue(2)
        self.title.setDefaultTextColor(QtCore.Qt.white)
        self.title.setParentItem(self)

        if self.logical_view:
            self._build_logical_view_plugs()
        else:
            self._build_regular_view_plugs()

        ctr = self.contentsRect()
        self.wmin = ctr.width()
        self.hmin = ctr.height()

        font1 = QtGui.QFont("Times", 12, QtGui.QFont.Normal)
        font1.setItalic(True)
        self.infoActived.setFont(font1)
        self.infoActived.setZValue(2)
        self.infoActived.setDefaultTextColor(QtCore.Qt.red)
        self.infoActived.setParentItem(self)

        self.box = boxItem(self)
        self.box.setFlags(self.box.ItemIsFocusable)
        self.box.setBrush(self.bg_brush)
        self.box.setPen(QtGui.QPen(QtCore.Qt.NoPen))
        self.box.setZValue(-1)
        self.box.penBox = self.box.pen()
        self.box.name = self.name
        self.box.setParentItem(self)

        self.sizer = HandleItem(self)
        self.sizer.wmin = self.wmin
        self.sizer.hmin = self.hmin
        self.sizer.setPos(ctr.width(), ctr.height())
        self.sizer.posChangeCallbacks.append(proxy_method(self, 'changeSize'))
        self.sizer.setFlag(self.sizer.ItemIsSelectable, True)

        self.box_title = QtGui.QGraphicsRectItem(self)
        self.box_title.setBrush(self.title_brush)
        self.box_title.setPen(QtGui.QPen(QtCore.Qt.NoPen))
        self.box_title.setZValue(1)
        self.box_title.setParentItem(self)

        self.changeSize(ctr.width(), ctr.height())

    def changeSize(self, w, h):
        limit = False
        factor_h = 35

        if h < self.hmin:
            h = self.hmin
            limit = True

        if w < self.wmin:
            w = self.wmin

        winMax, woutMax = 0, 0

        self.box.setRect(0.0, 0.0, w, h)

        self.box_title.setRect(0.0, 0.0, w, 30)
        self.title.setPos(w / 2 - self.title.boundingRect().size().width() / 2,
                          0)
        self.infoActived.setPos(
            w / 2 - self.infoActived.boundingRect().size().width() / 2, h + 2)

        #         rect = self.title.mapRectToParent(self.title.boundingRect())
        #         rect.setWidth(w)
        #         self.box_title.setRect(rect)

        y1 = h / (len(self.in_plugs) + 1)
        dy = y1
        for inp in self.in_plugs:
            self.in_plugs[inp].setPos(0, y1)
            self.in_params[inp].setPos(self.in_plugs[inp].boundingRect().size().width(), y1)
            if winMax < self.in_params[inp].boundingRect().size().width():
                winMax = self.in_params[inp].boundingRect().size().width()
            y1 += dy

        y2 = h / (len(self.out_plugs) + 1)
        dy = y2
        for outp in self.out_plugs:
            self.out_plugs[outp].setPos(w, y2)
            self.out_params[outp].setPos(w - self.out_params[outp].boundingRect().size().width() - 5, y2)
            if woutMax < self.out_params[outp].boundingRect().size().width():
                woutMax = self.out_params[outp].boundingRect().size().width()
            y2 += dy

        if w < winMax + woutMax + 15:
            w = winMax + woutMax + 15
            self.updateSize(w, h)
            #self.sizer.setPos(w, h)
            self.wmin = w

        if limit:
            self.sizer.setPos(w, h)

        self.update_labels([l.text for l in self.labels])

        # if self.hmin < factor_h * len(self.in_plugs):
        #     self.hmin = factor_h * len(self.in_plugs)
        #     self.updateSize(w, self.hmin)
        # if self.hmin < factor_h * len(self.out_plugs):
        #     self.hmin = factor_h * len(self.out_plugs)

    def updateSize(self, w, h):
        # print("wmin =",self.wmin,", w=",w)
        if w<self.wmin:
            w=self.wmin
        margin = 20
        factor_h = 35.0
        h = factor_h * len(self.in_plugs)+margin
        self.hmin=h

        if h < factor_h * len(self.out_plugs):
            h = factor_h * len(self.out_plugs) + margin
            self.hmin = h
        self.sizer.hmin = h
        self.changeSize(w, h + margin)
        self.sizer.setPos(w, h + margin)
        
    def _colored_text_item(self, label, text=None, margin=2):
        labelc = self._get_label(label, False)
        color = labelc.color
        if text is None:
            text = label
        # I can't make rounded borders with appropriate padding
        # without using 2 QLabels. This is probably overkill. We could
        # replace this code of we find a simpler way.
        label_w = QtGui.QLabel('')
        label_w.setStyleSheet("background: rgba(255, 255, 255, 0);")
        lay = QtGui.QVBoxLayout()
        lay.setContentsMargins(margin, margin, margin, margin)
        label_w.setLayout(lay)
        label2 = QtGui.QLabel(text)
        label2.setObjectName('label')
        label2.setStyleSheet(
            "background: rgba({0}, {1}, {2}, 255); "
            "border-radius: 7px; border: 0px solid; "
            "padding: 1px;".format(*color))
        lay.addWidget(label2)
        label_item = QtGui.QGraphicsProxyWidget(self)
        label_item.setWidget(label_w)

    def _build_regular_view_plugs(self):
        margin = 5
        plug_width = 12
        pos = margin + margin + self.title.boundingRect().size().height()
        pos0 = pos
        if self.name == 'inputs':
            selections = self.pipeline.get_processes_selections()
        else:
            selections = []

        for in_param, pipeline_plug in six.iteritems(self.parameters):
            output = (not pipeline_plug.output if self.name in (
                'inputs', 'outputs') else pipeline_plug.output)
            if output or (not self.show_opt_inputs and pipeline_plug.optional):
                continue
            param_text = self._parameter_text(in_param)
            param_name = QtGui.QGraphicsTextItem(self)
            param_name.setHtml(param_text)

            plug_name = '%s:%s' % (self.name, in_param)

            try:
                #                 color = self.colorLink(trait_type_str)
                color = self.colType.colorLink(self.process.trait(in_param))
                color = ORANGE_2

            plug = Plug(color, plug_name,
                        param_name.boundingRect().size().height(),
                        plug_width, activated=pipeline_plug.activated,
                        optional=pipeline_plug.optional, parent=self)
            param_name.setZValue(2)
            plug.setPos(margin, pos)
            param_name.setPos(plug.boundingRect().size().width() + margin, pos)
            param_name.setParentItem(self)
            plug.setParentItem(self)
            self.in_plugs[in_param] = plug
            self.in_params[in_param] = param_name
            pos = pos + param_name.boundingRect().size().height()

        pos = pos0
        for out_param, pipeline_plug in six.iteritems(self.parameters):
            if not output or (not self.show_opt_outputs and pipeline_plug.optional):
                continue
            param_text = self._parameter_text(out_param)
            if out_param in selections:
                param_name = self._colored_text_item('select: ' + out_param,
                                                     param_text, 0)
            else:
                param_name = QtGui.QGraphicsTextItem(self)
                param_name.setHtml(param_text)

            plug_name = '%s:%s' % (self.name, out_param)

            try:
                color = self.colType.colorLink(self.process.trait(out_param))

            plug.setPos(plug.boundingRect().size().width() + margin +
                        param_name.boundingRect().size().width() + margin, pos)
            self.out_plugs[out_param] = plug
            self.out_params[out_param] = param_name

    def change_input_view(self):
        self.show_opt_inputs = not self.show_opt_inputs

    def change_output_view(self):
        self.show_opt_outputs = not self.show_opt_outputs

    def _build_logical_view_plugs(self):
        margin = 5

        has_input = False

            if output:
            else:
                has_input = True
            if has_input and has_output:
                break

        if has_input:
            param_name.setHtml('')
            plug_name = '%s:inputs' % self.name

            color = QtCore.Qt.black

                        plug_width, activated=True,
                        optional=False, parent=self)
            self.in_plugs['inputs'] = plug
            self.in_params['inputs'] = param_name

            plug_name = '%s:outputs' % self.name

            plug.setPos(self.title.boundingRect().width()
                        - plug.boundingRect().width(), pos)
            self.out_plugs['outputs'] = plug
            self.out_params['outputs'] = param_name

    def _create_label_marks(self):
        labels = self.labels
        if labels:
            margin = 5
            plug_width = 12
            xpos = margin + plug_width
            ypos = None
            params = dict(self.in_params)
            params.update(self.out_params)
            child = None
            for param in params.values():
                y = self.mapRectFromItem(param, param.boundingRect()).bottom()
                if ypos is None or ypos < y:
                    ypos = y
                    child = param
            #if ypos is None:
                #ypos = margin * 2 + self.title.boundingRect().size().height()
            if child is None:
                child = self.childItems()[-1]
                item_rect = self.mapRectFromItem(child, child.boundingRect())
                ypos = item_rect.bottom()
            for label in labels:
                color = label.color
                text = label.text
                label_item = self._colored_text_item(label.text, label.text)
                label_item.setPos(xpos, ypos)
                label_item.setParentItem(self)
                self.label_items.append(label_item)
                ypos = self.mapRectFromItem(
                    label_item, label_item.boundingRect()).bottom()

    def clear_plugs(self):
        for plugs, params in ((self.in_plugs, self.in_params),
                              (self.out_plugs, self.out_params)):
            for plug_name, plug in six.iteritems(plugs):
                param_item = params[plug_name]
                self.scene().removeItem(param_item)
                self.scene().removeItem(plug)
        self.in_plugs = {}
        self.out_plugs = {}

    def updateInfoActived(self, state):
        if state:
            self.infoActived.setPlainText('')
        else:
            self.infoActived.setPlainText('disabled')

    def fonced_viewer(self, det):
        if det:
            #             color=QtGui.QColor(150, 150, 250)
            self.setOpacity(0.2)
        else:
            #             color=self.color
            self.setOpacity(1)

    #         self._set_pen(self.active, self.weak, color)

    def _set_brush(self):
        pipeline = self.pipeline
        if self.name in ('inputs', 'outputs'):
            node = pipeline.pipeline_node
        else:
            node = pipeline.nodes[self.name]
        color_1, color_2, color_3, style = pipeline_tools.pipeline_node_colors(
            pipeline, node)
        self.style = style
        color_1 = QtGui.QColor.fromRgbF(*color_1)
        color_2 = QtGui.QColor.fromRgbF(*color_2)
        #         color_1 = ANTHRACITE_1
        #         color_2 = LIGHT_ANTHRACITE_1
        gradient = QtGui.QLinearGradient(0, 0, 0, 50)
        gradient.setColorAt(0, color_1)
        gradient.setColorAt(1, color_2)
        self.bg_brush = QtGui.QBrush(gradient)

            #             color_1 = GRAY_1
            #             color_2 = GRAY_2
            self.updateInfoActived(True)

        else:
            #             color_1 = LIGHT_GRAY_1
            #             color_2 = LIGHT_GRAY_2
            self.updateInfoActived(False)

        if node in pipeline.disabled_pipeline_steps_nodes():
            color_1 = self._color_disabled(color_1)
            color_2 = self._color_disabled(color_2)

        gradient = QtGui.QLinearGradient(0, 2, 5, 100)
        gradient.setColorAt(1, GRAY_1)
        gradient.setColorAt(0, GRAY_2)
        self.title_brush = QtGui.QBrush(LIGHT_GRAY_2)

    def _color_disabled(self, color):
        target = [220, 240, 220]
        new_color = QtGui.QColor(int((color.red() + target[0]) / 2),
                                 int((color.green() + target[1]) / 2),
                                 int((color.blue() + target[2]) / 2))

    def _create_parameter(self, param_name, pipeline_plug):
        margin = 5
        output = (not pipeline_plug.output if self.name in (
            'inputs', 'outputs') else pipeline_plug.output)
            if output:
                param_name = 'outputs'
            else:
                param_name = 'inputs'
        if self.name == 'inputs' and not self.logical_view \
                and 'select: ' + param_name in \
                [l.text for l in self.scene_labels]:
            param_name_item = self._colored_text_item('select: ' + param_name,
                                                      param_text, 0)
        else:
            param_name_item = QtGui.QGraphicsTextItem(self)
            param_name_item.setHtml(param_text)
        plug_name = '%s:%s' % (self.name, param_name)

        color = QtCore.Qt.black

        plug = Plug(color, plug_name,
                    param_name_item.boundingRect().size().height(),
                    plug_width, activated=pipeline_plug.activated,
                    optional=pipeline_plug.optional, parent=self)
        param_name_item.setZValue(2)
        if output:
            plugs = self.out_plugs
            params = self.out_params
            params_size = len(params) + len(self.in_params)
            # FIXME: sub-pipeline size
            xpos = plug.boundingRect().size().width() + margin
            pxpos = plug.boundingRect().size().width() + margin * 2 \
                    + param_name_item.boundingRect().size().width()
        else:
            plugs = self.in_plugs
            params = self.in_params
            params_size = len(params)
            pxpos = margin
            params_size = 0
            if output:
                pxpos = self.title.boundingRect().width() \
                        - plug.boundingRect().width()
        pos = margin * 2 + self.title.boundingRect().size().height() \
              + param_name_item.boundingRect().size().height() * params_size
        param_name_item.setPos(xpos, pos)
        plug.setPos(pxpos, pos)
        param_name_item.setParentItem(self)
        plug.setParentItem(self)
        plugs[param_name] = plug
        params[param_name] = param_name_item
        if output:
            self._shift_params()

        self.updateSize(self.box.boundingRect().size().width(), self.box.boundingRect().size().height())
        self.sizer.setPos(self.box.boundingRect().size().width(), self.box.boundingRect().size().height())

    #         self.hmin=self.box.boundingRect().size().height()

    def _shift_params(self):
        margin = 5
        if not self.in_params:
            if not self.out_params:
                param_item = None
            else:
                param_item = list(self.out_params.values())[0]
        else:
            param_item = list(self.in_params.values())[0]
        ni = 0
        no = 0
        bottom_pos = 0
        if param_item:
            for param_name, pipeline_plug in six.iteritems(self.parameters):
                output = (not pipeline_plug.output if self.name in (
                    'inputs', 'outputs') else pipeline_plug.output)
                if output:
                    # Added to choose to visualize optional parameters
                    if not pipeline_plug.optional or (self.show_opt_outputs and pipeline_plug.optional):
                        params = self.out_params
                        plugs = self.out_plugs
                        npos = no + len(self.in_params)
                        no += 1
                    else:
                        continue
                else:
                    if not pipeline_plug.optional or (self.show_opt_inputs and pipeline_plug.optional):
                        params = self.in_params
                        plugs = self.in_plugs
                        npos = ni
                        ni += 1
                    else:
                        continue
                pos = margin * 2 + self.title.boundingRect().size().height() \
                      + param_item.boundingRect().size().height() * npos
                new_param_item = params.get(param_name)
                if new_param_item is None:
                    continue
                param_item = new_param_item
                plug = plugs[param_name]
                ppos = param_item.pos()
                param_item.setPos(ppos.x(), pos)
                ppos = plug.pos()
                plug.setPos(ppos.x(), pos)
                pos += param_item.boundingRect().size().height()
                bottom_pos = max(pos, bottom_pos)
            if self.logical_view:
                nparams = 1
            else:
                nparams = len(self.in_params) + len(self.out_params)
            pos = margin * 2 + self.title.boundingRect().size().height() \
                  + param_item.boundingRect().size().height() * nparams
        else:
            pos = margin * 2 + self.title.boundingRect().size().height()
        for label_item in self.label_items:
            ppos = label_item.pos()
            label_item.setPos(ppos.x(), pos)
            pos += label_item.boundingRect().size().height()

    def _remove_parameter(self, param_name):
        if param_name in self.in_params:
        else:
        param_item = params[param_name]
        if self.scene():
            self.scene().removeItem(param_item)
        plug = plugs[param_name]
            self.scene().removeItem(plug)
        del params[param_name]
        del plugs[param_name]
        self._shift_params()

    def _parameter_text(self, param_name):
            return ''
        pipeline_plug = self.parameters[param_name]
        # output = (not pipeline_plug.output if self.name in (
        # 'inputs', 'outputs') else pipeline_plug.output)
        output = pipeline_plug.output
        if output:
            param_text = '<font color="#400000"><b>%s</b></font>' % param_name
        else:
            param_text = '<font color="#111111"><b>%s</b></font>' % param_name
        try:
            value = getattr(self.process, param_name)
            value = traits.Undefined
        if value is None or value is traits.Undefined or value == '':
            param_text = '<em>%s</em>' % param_text
        else:
            trait = self.process.user_traits()[param_name]
            if (isinstance(trait.trait_type, traits.File) \
                or isinstance(trait.trait_type, traits.Directory)) \
                param_text = '<b>%s</b>' % param_text
        return param_text

    def update_node(self):
        # this is needed before the box is resized, otherwise some bad things
        # may happen, especially segfaults...
        self.prepareGeometryChange()
        for param, pipeline_plug in six.iteritems(self.parameters):
            if output:
                plugs = self.out_plugs
                params = self.out_params
                if self.logical_view:
                    param = 'outputs'
            else:
                plugs = self.in_plugs
                params = self.in_params
                    param = 'inputs'
            gplug = plugs.get(param)
            if gplug is None:  # new parameter ?
                self._create_parameter(param, pipeline_plug)
                gplug = plugs.get(param)
            if not self.logical_view:
                #                 gplug.update_plug(pipeline_plug.activated,
                #                                   pipeline_plug.optional)

                try:
                    #                     color = self.colorLink(trait_type_str)
                    color = self.colType.colorLink(self.process.trait(param))

                    color = ORANGE_2

                gplug.update_plug(color)

                if isinstance(params[param], QtGui.QGraphicsProxyWidget):
                    # colored parameters are widgets
                    params[param].widget().findChild(
                        QtGui.QLabel, 'label').setText(
                        self._parameter_text(param))
                else:
                    params[param].setHtml(self._parameter_text(param))

        if not self.logical_view:
            # check removed params
            to_remove = []

            # Added to choose to visualize optional parameters
            for param, pipeline_plug in six.iteritems(self.parameters):
                if output:
                    if pipeline_plug.optional and not self.show_opt_outputs:
                        to_remove.append(param)
                else:
                    if pipeline_plug.optional and not self.show_opt_inputs :

            for param in self.in_params:
                if param not in self.parameters:
                    to_remove.append(param)
            for param in self.out_params:
            for param in to_remove:
                self._remove_parameter(param)

        #         margin = 5
        #         brect = self.boundingRect()
        #         brect.setWidth(brect.right() - margin)
        #         rect.setWidth(brect.width())
        #         self.box.setRect(self.boundingRect())
        
        ################a dd by Irmage OM #############################################
        try :
            dim = self.scene().dim.get(self.box.name)
            if isinstance(dim, Qt.QPointF):
                dim=(dim.x(),dim.y())
            
            self.updateSize(dim[0],dim[1])
#             self.scene().dim[self.box.name] = (dim[0],dim[1]) 
#             print("update_node : self.scene().dim ",dim)

            dim = (self.box.boundingRect().size().width(), self.box.boundingRect().size().height())
#             print("update_node : boundingRect()")
        ##############################################################################

    def contentsRect(self):
        brect = QtCore.QRectF(0, 0, 0, 0)
        excluded = []
        for name in ('box', 'box_title'):
            if hasattr(self, name):
                excluded.append(getattr(self, name))
        for child in self.childItems():
            if not hasattr(child, 'isVisible'):
                # we sometimes get some QObject here, I don't know who they are
                continue
            if not child.isVisible() or child in excluded:
                continue
            item_rect = self.mapRectFromItem(child, child.boundingRect())
            if first:
                first = False
                brect = item_rect
            else:
                if child is self.embedded_subpipeline:
                    margin = 5
                    item_rect.setBottom(item_rect.bottom() + margin)
                if item_rect.left() < brect.left():
                    brect.setLeft(item_rect.left())
                if item_rect.top() < brect.top():
                    brect.setTop(item_rect.top())
                if item_rect.right() > brect.right():
                    brect.setRight(item_rect.right())
                if item_rect.bottom() > brect.bottom():
                    brect.setBottom(item_rect.bottom())
        return brect

    def boundingRect(self):
        margin = 0
        brect = self.contentsRect()
        if self.embedded_subpipeline and self.embedded_subpipeline.isVisible():
            margin = 5
        brect.setRight(brect.right())
        brect.setBottom(brect.bottom() + margin)

    def paint(self, painter, option, widget=None):
        pass

    def postscript(self, file_name):
        printer = QtGui.QPrinter(QtGui.QPrinter.HighResolution)
        printer.setOutputFormat(QtGui.QPrinter.PostScriptFormat)
        printer.setOutputFileName(file_name)
        # qreal xmargin = contentRect.width()*0.01;
        # qreal ymargin = contentRect.height()*0.01;
        # printer.setPaperSize(10*contentRect.size()*1.02,QPrinter::DevicePixel);
        # printer.setPageMargins(xmargin,ymargin,xmargin,ymargin,QPrinter::DevicePixel);
        painter = QtGui.QPainter()
        painter.begin(printer)
        painter.setPen(QtCore.Qt.blue)
        painter.setFont(QtGui.QFont('Arial', 30))
        painter.drawText(0, 0, 'Ca marche !')
        # render(&painter,QRectF(QPointF(0,0),10*contentRect.size()),contentRect);
        painter.end()

    def resize_subpipeline_on_show(self):
        margin = 5
        param_width = self.in_params_width()
        pos = margin * 2 + self.title.boundingRect().size().height()
        opos = param_width \
               + self.embedded_subpipeline.boundingRect().width()  # + margin ?
        for name, param in six.iteritems(self.out_params):
            param.setPos(opos, param.pos().y())
            plug = self.out_plugs[name]
            plug.setPos(opos + margin + param.boundingRect().size().width(),
                        plug.pos().y())
        #         rect = self.box_title.boundingRect()
        rect = self.box.boundingRect()
        rect.setWidth(self.contentsRect().width())
        self.box.setRect(self.boundingRect())

    def resize_subpipeline_on_hide(self):
        margin = 5
            param.setPos(plug.boundingRect().width() + margin, param.pos().y())
                        param.boundingRect().size().width() + margin, plug.pos().y())

    def in_params_width(self):
        margin = 5
        width = 0
        pwidth = 0
        for param_name, param in six.iteritems(self.in_params):
            if param.boundingRect().width() > width:
                width = param.boundingRect().width()
            if pwidth == 0:
                plug = self.in_plugs[param_name]
                pwidth = plug.boundingRect().width()
        return width + margin + pwidth

    def out_params_width(self):
        width = 0
        for param_name, param in six.iteritems(self.out_params):
        return width

    def add_subpipeline_view(
            self,
            sub_pipeline,
            allow_open_controller=True,
            scale=None):
        if self.embedded_subpipeline:
            if self.embedded_subpipeline.isVisible():
                self.embedded_subpipeline.hide()
                self.resize_subpipeline_on_hide()
            else:
                self.embedded_subpipeline.show()
                self.resize_subpipeline_on_show()
        else:
            sub_view = PipelineDeveloperView(
                sub_pipeline,
                show_sub_pipelines=True,
                allow_open_controller=allow_open_controller,
                enable_edition=self.scene().edition_enabled(),
                userlevel=self.userlevel)
            if scale is not None:
                sub_view.scale(scale, scale)
            pwid = EmbeddedSubPipelineItem(sub_view)
            sub_view._graphics_item = weakref.proxy(pwid)
            margin = 5
            pwid.setParentItem(self)
            pwid.setPos(self.in_params_width(), pos)
            self.embedded_subpipeline = pwid
            self.resize_subpipeline_on_show()
            self.setFiltersChildEvents(False)
            pwid.geometryChanged.connect(self.resize_subpipeline_on_show)

    def mouseDoubleClickEvent(self, event):
        if self.sub_pipeline:
            if isinstance(self.sub_pipeline, weakref.ProxyTypes):
                # get the "real" object
                process = self.sub_pipeline.__init__.__self__
            else:
                process = self.sub_pipeline
            self.scene().subpipeline_clicked.emit(self.name, process,
                                                  int(event.modifiers()))
        else:
            event.ignore()

        item = self.scene().itemAt(event.scenePos(), Qt.QTransform())
        # print('NodeGWidget click, item:', item)
        if isinstance(item, Plug):
            item.mousePressEvent(event)
            return
        super(NodeGWidget, self).mousePressEvent(event)
        if isinstance(self.process, weakref.ProxyTypes):
            process = self.process.__init__.__self__  # get the "real" object
        else:
        if event.button() == QtCore.Qt.RightButton and process is not None:
            self.scene().node_right_clicked.emit(self.name, process)

        if event.button() == QtCore.Qt.LeftButton and process is not None:
            if isinstance(process, Process):
                self.scene().process_clicked.emit(self.name, process)
            else:
                self.scene().node_clicked.emit(self.name, process)

        if event.button() == QtCore.Qt.LeftButton and event.modifiers() == QtCore.Qt.ControlModifier:
            self.scene().node_clicked_ctrl.emit(self.name, process)
            self.scene().clearSelection()
            self.box.setSelected(True)
            return QtGui.QGraphicsItem.mousePressEvent(self, event)

        super(NodeGWidget, self).keyPressEvent(event)

        if event.key() == QtCore.Qt.Key_Up:
            self.setPos(self.x(), self.y() - 1)
        if event.key() == QtCore.Qt.Key_Down:
            self.setPos(self.x(), self.y() + 1)
        if event.key() == QtCore.Qt.Key_Left:
            self.setPos(self.x() - 1, self.y())
        if event.key() == QtCore.Qt.Key_Right:
            self.setPos(self.x() + 1, self.y())

        return QtGui.QGraphicsItem.keyPressEvent(self, event)
        event.accept()

class HandleItem(QtGui.QGraphicsRectItem):
    """ A handle that can be moved by the mouse """

        super(HandleItem, self).__init__(Qt.QRectF(-10.0, -10.0, 10.0, 10.0), parent)
        #         self.setRect(Qt.QRectF(-4.0,-4.0,4.0,4.0))
        self.posChangeCallbacks = []
        self.setPen(QtGui.QPen(QtCore.Qt.NoPen))
        self.setBrush(QtGui.QBrush(QtCore.Qt.yellow))
        self.setFlag(self.ItemIsMovable, True)
        self.setFlag(self.ItemSendsScenePositionChanges, True)
        self.setCursor(QtGui.QCursor(QtCore.Qt.SizeFDiagCursor))
        self.wmin = 0.0
        self.hmin = 0.0
        self.hmax = 0.0

        self.effectiveOpacity()
        self.setOpacity(0.01)

    def itemChange(self, change, value):
        if change == self.ItemPositionChange:
            self.x, self.y = value.x(), value.y()
            if self.x < self.wmin:
                self.x = self.wmin
            if self.y < self.hmin:
                self.y = self.hmin
            # TODO: make this a signal?
            # This cannot be a signal because this is not a QObject
            for cb in self.posChangeCallbacks:
                res = cb(self.x, self.y)

                if res:
                    self.x, self.y = res
                    if self.x < self.wmin:
                        self.x = self.wmin
                    if self.y < self.hmin:
                        self.y = self.hmin

                    value = QtCore.QPointF(self.x, self.y)
            #                     value = Qt.QPointF(x, y)      #### ??
            self.hmax = value.y()
        # Call superclass method:

        return super(HandleItem, self).itemChange(change, value)

    def mouseReleaseEvent(self, mouseEvent):
        self.setSelected(False)
        self.setPos(self.x, self.y)
        return QtGui.QGraphicsRectItem.mouseReleaseEvent(self, mouseEvent)

class Link(QtGui.QGraphicsPathItem):

    def __init__(self, origin, target, active, weak, color, parent=None):
        super(Link, self).__init__(parent)

        self._set_pen(active, weak, color)

        self.setFlag(QtGui.QGraphicsItem.ItemIsSelectable, False)
        self.setFlag(QtGui.QGraphicsItem.ItemIsFocusable, True)

        path = QtGui.QPainterPath()
        path.moveTo(origin.x(), origin.y())
        path.cubicTo(origin.x() + 90, origin.y(),
                     target.x() - 90, target.y(),
                     target.x() - 5, target.y())
        self.setPath(path)
        self.setZValue(0.5)
        self.active = active
        self.weak = weak

    def _set_pen(self, active, weak, color):
        self.pen = QtGui.QPen()
        self.pen.setWidth(3)

        if active:
            self.pen.setBrush(color)
        else:
            self.pen.setBrush(QtCore.Qt.gray)
        if weak:
            self.pen.setStyle(QtCore.Qt.DashLine)
        self.pen.setCapStyle(QtCore.Qt.RoundCap)
        self.pen.setJoinStyle(QtCore.Qt.RoundJoin)
        self.setPen(self.pen)

    def update(self, origin, target):

    def update_activation(self, active, weak, color):
        if color == 'current':
            color = self.color

        if det:
        else:

        # print('Link click, item:', item)
        if event.button() == QtCore.Qt.RightButton:
            # not a signal since we don't jhave enough identity information in
            # self: the scene has to help us.
            self.scene()._link_right_clicked(self)
        else:
            super(Link, self).mousePressEvent(event)

        self.setPen(QtGui.QPen(QtGui.QColor(150, 150, 250), 3, QtCore.Qt.DashDotDotLine))
        return QtGui.QGraphicsPathItem.focusInEvent(self, event)

        return QtGui.QGraphicsPathItem.focusOutEvent(self, event)

            self.scene()._link_keydelete_clicked(self)
        else:
            super(Link, self).keyPressEvent(event)

class PipelineScene(QtGui.QGraphicsScene):
    # Signal emitted when a sub pipeline has to be open.
    subpipeline_clicked = QtCore.Signal(str, Process,
                                        QtCore.Qt.KeyboardModifiers)
    # Signal emitted when a node box is clicked
    process_clicked = QtCore.Signal(str, Process)
    node_clicked = QtCore.Signal(str, Node)
    # Signal emitted when a node box is clicked with ctrl
    node_clicked_ctrl = QtCore.Signal(str, Process)
    # Signal emitted when a switch box is clicked
    switch_clicked = QtCore.Signal(str, Switch)
    # Signal emitted when a node box is right-clicked
    node_right_clicked = QtCore.Signal(str, Controller)
    # Signal emitted when a plug is clicked
    plug_clicked = QtCore.Signal(str)
    # Signal emitted when a plug is clicked with the right mouse button
    plug_right_clicked = QtCore.Signal(str)
    # Signal emitted when a link is right-clicked
    link_right_clicked = QtCore.Signal(str, str, str, str)

    link_keydelete_clicked = QtCore.Signal(str, str, str, str)

    node_keydelete_clicked = QtCore.Signal(str)

    def __init__(self, parent=None, userlevel=0):
        super(PipelineScene, self).__init__(parent)

        self.gnodes = {}
        self.glinks = {}
        self._pos = 50
        self.pos = {}
        self.dim = {} # add by Irmage OM for recorded dimension of Nodes
        self.colored_parameters = True
        self.logical_view = False
        self._enable_edition = False

        #         pen = QtGui.QPen(QtGui.QColor(250,100,0),2)
        #         self.l = QtCore.QLineF(-10,0,10,0)
        #         self.addLine(self.l,pen)
        #         self.l = QtCore.QLineF(0,-10,0,10)

        self._update_pipeline_timer = Qt.QTimer()
        self._update_pipeline_timer.setSingleShot(True)
        self._update_pipeline_timer.timeout.connect(
          proxy_method(self, 'update_pipeline_now'))

        self.changed.connect(self.update_paths)

        # print('PipelineScene.__del__')
        try:
            self._release()
        except RuntimeError:
            pass  # C++ object deleted, attributes are already destroyed

        # print('PipelineScene._release')
        if hasattr(self, 'pos'):
            del self.pos
        if hasattr(self, 'dim'):
            del self.dim
        if hasattr(self, 'labels'):
            del self.labels
        if hasattr(self, 'glinks'):
            del self.glinks
        if 'gnodes' in self.__dict__:
            from soma.qt_gui.qt_backend import sip
            gnode = None
            for gnode in self.gnodes.values():
                gnode._release()
                self.removeItem(gnode)
                sip.transferback(gnode)
            del gnode
            del self.gnodes

        try:
            self.changed.disconnect()
        except TypeError:
            pass  # already done

        # force delete gnodes: needs to use gc.collect()
        import gc
        gc.collect()

    @property

        if self._userlevel != value:
            self._userlevel = value
            for name, gnode in self.gnodes.items():
                gnode.userlevel = value
            self.update_pipeline()

    def _add_node(self, name, gnode):
        self.addItem(gnode)

        ################# add by Irmage OM ####################
        dim = self.dim.get(name)
#         print("_add_node : dim : ",dim," , type =",type(dim).__name__)

        if dim is not None:

            gnode.updateSize(dim[0],dim[1])
            #gnode.sizer.setPos(dim[0],dim[1])

#         gnode.update_node()

        ######################################################

        pos = self.pos.get(name)
        if pos is None:
            gnode.setPos(2 * self._pos, self._pos)
            self._pos += 100
        else:
            if not isinstance(pos, Qt.QPointF):
                pos = Qt.QPointF(pos[0], pos[1])
            gnode.setPos(pos)
        
        self.gnodes[name] = gnode
        
         #repositioning 'inputs' node
        if name == 'inputs':
            pos_left_most=(0,0)
            for el in self.gnodes:
                if el!='inputs' and el!='outputs':
                    if pos_left_most[0] > self.gnodes[el].pos().x():
                        pos_left_most=(self.gnodes[el].pos().x(),self.gnodes[el].pos().y())
            xl = pos_left_most[0]-(2*self.gnodes[name].boundingRect().size().width())
            yl = pos_left_most[1]
            self.gnodes[name].setPos(xl,yl)
#             gnode.update_node()    
             
        #repositioning 'outputs' node
        if name == 'outputs':
            pos_right_most=(0,0)
                    if pos_right_most[0] < self.gnodes[el].pos().x() + self.gnodes[el].boundingRect().size().width() :
                        pos_right_most=(self.gnodes[el].pos().x() + self.gnodes[el].boundingRect().size().width(),self.gnodes[el].pos().y())
            xl = pos_right_most[0]+self.gnodes[name].boundingRect().size().width()
            yl = pos_right_most[1]
#             gnode.update_node()

        ################" add by Irmage #############################################
        self.setSceneRect(QtCore.QRectF())
        #############################################################################_node_keydelete_clicked

    def add_node(self, node_name, node):
        if not isinstance(node, ProcessNode):
            sub_pipeline = process
        elif process and isinstance(process, ProcessIteration):
            sub_pipeline = process.process
        else:
            sub_pipeline = None
        gnode = NodeGWidget(
            node_name, node.plugs, self.pipeline,
            sub_pipeline=sub_pipeline, process=process,
            colored_parameters=self.colored_parameters,
            logical_view=self.logical_view, labels=self.labels,
            userlevel=self.userlevel)
        self._add_node(node_name, gnode)
        gnode.update_node()
        return gnode

    def add_link(self, source, dest, active, weak):
        #         print("add link ", source, dest)
        source_gnode_name, source_param = source
        if not source_gnode_name:
            source_gnode_name = 'inputs'
        dest_gnode_name, dest_param = dest
        if not dest_gnode_name:
            dest_gnode_name = 'outputs'
            source_param = 'outputs'
            dest_param = 'inputs'
        try:
            typeq = self.typeLink(source_gnode_name, source_param)
            #             color = self.colorLink(typeq)
            color = self.colType.colorLink(typeq)

            color = ORANGE_2
        #         verif=((str(dest_gnode_name), str(dest_param)))
        #         print(str(verif) in str(self.glinks.keys()))
        source_dest = ((str(source_gnode_name), str(source_param)),
                       (str(dest_gnode_name), str(dest_param)))
        if source_dest in self.glinks:
                # keep strongest link representation
                glink = self.glinks[source_dest]
                if active or glink.active:
                    active = True
                if not weak or not glink.weak:
                    weak = False
                if glink.weak != weak or glink.active != active:
                    glink.update_activation(active, weak, "current")
            return  # already done
        source_gnode = self.gnodes[source_gnode_name]
        dest_gnode = self.gnodes.get(dest_gnode_name)

        if dest_gnode is not None:
            if dest_param in dest_gnode.in_plugs \
                    and source_param in source_gnode.out_plugs:
                glink = Link(
                    source_gnode.mapToScene(
                        source_gnode.out_plugs[source_param].get_plug_point()),
                    dest_gnode.mapToScene(
                        dest_gnode.in_plugs[dest_param].get_plug_point()),
                    active, weak, color)
                self.glinks[source_dest] = glink
                self.addItem(glink)

    def _remove_link(self, source_dest):
        source, dest = source_dest
        source_gnode = self.gnodes.get(source_gnode_name)
            # is it useful ?
        new_source_dest = ((str(source_gnode_name), str(source_param)),
                           (str(dest_gnode_name), str(dest_param)))
        glink = self.glinks.get(new_source_dest)
        if glink is not None:
            self.removeItem(glink)
            del self.glinks[new_source_dest]

    def update_paths(self, regions=[]):
        for name, i in six.iteritems(self.gnodes):
            self.pos[i.name] = i.pos()
            br = i.box.boundingRect()
            self.dim[i.name] = (br.width(), br.height())

        dropped = []
        for source_dest, glink in six.iteritems(self.glinks):
            source, dest = source_dest
            source_gnode_name, source_param = source
            dest_gnode_name, dest_param = dest
            source_gnode = self.gnodes[source_gnode_name]
            dest_gnode = self.gnodes[dest_gnode_name]
            if source_param not in source_gnode.out_plugs \
                    or dest_param not in dest_gnode.in_plugs:
                dropped.append(source_dest)
            else:
                glink.update(source_gnode.mapToScene(
                    source_gnode.out_plugs[source_param].get_plug_point()),
                        dest_gnode.in_plugs[dest_param].get_plug_point()))
        for source_dest in dropped:
            self._remove_link(source_dest)

        pipeline_inputs = SortedDictionary()
        pipeline_outputs = SortedDictionary()
        if pipeline is not None:
            for name, plug in six.iteritems(pipeline.nodes[''].plugs):
                    pipeline_outputs[name] = plug
                else:
                    pipeline_inputs[name] = plug
            if pipeline_inputs:
                self._add_node(
                    'inputs', NodeGWidget(
                        'inputs', pipeline_inputs, pipeline,
                        process=pipeline,
                        colored_parameters=self.colored_parameters,
                        logical_view=self.logical_view,
                        userlevel=self.userlevel))
                if not node_name:
                    continue
                self.add_node(node_name, node)
            if pipeline_outputs:
                    'outputs', NodeGWidget(
                        'outputs', pipeline_outputs, pipeline,

            for source_node_name, source_node in six.iteritems(pipeline.nodes):
                for source_parameter, source_plug \
                        in six.iteritems(source_node.plugs):
                        weak_link) in source_plug.links_to:
                        if dest_node is pipeline.nodes.get(dest_node_name):
                            self.add_link(
                                (source_node_name, source_parameter),
                                (dest_node_name, dest_parameter),
                                active=source_plug.activated \
                                      and dest_plug.activated,
                                weak=weak_link)

    def update_pipeline(self):
        self._update_pipeline_timer.start(20)

    def update_pipeline_now(self):
            self._update_logical_pipeline()
        else:
            self._update_regular_pipeline()

    def _update_regular_pipeline(self):
        # normal view
        removed_nodes = []

        #         print(self.gnodes)
        for node_name, gnode in six.iteritems(self.gnodes):
            removed = False
            if gnode.logical_view:
                gnode.clear_plugs()
                gnode.logical_view = False
            if node_name in ('inputs', 'outputs'):
                node = pipeline.nodes['']
                # in case traits have been added/removed
                if node_name == 'inputs':
                    pipeline_inputs = SortedDictionary()
                    for name, plug in six.iteritems(node.plugs):
                        if not plug.output:
                            trait = node.get_trait(name)
                            if not trait.hidden \
                                    and (trait.userlevel is None
                                         or trait.userlevel <= self.userlevel):
                                pipeline_inputs[name] = plug
                    gnode.parameters = pipeline_inputs
                    if len(gnode.parameters) == 0:
                        # no inputs: remove the gnode
                        removed_nodes.append(node_name)
                        removed = True
                else:
                    pipeline_outputs = SortedDictionary()
                        if plug.output:
                                pipeline_outputs[name] = plug
                    gnode.parameters = pipeline_outputs
                        # no outputs: remove the gnode
            else:
                node = pipeline.nodes.get(node_name)
                if node is None:  # removed node
                    removed_nodes.append(node_name)
                    removed = True
                    continue
            if not removed:
                gnode.active = node.activated
                gnode.update_node()

        # handle removed nodes
        for node_name in removed_nodes:
            gnode = self.gnodes[node_name]
            self.removeItem(gnode)
            self.gnodes.pop(node_name, None)
            self.dim.pop(node_name, None)
            self.pos.pop(node_name, None)
            sip.transferback(gnode)
            #import objgraph
            #objgraph.show_backrefs(gnode)
            del gnode

        # check for added nodes
        added_nodes = []
            if node_name == '':
                pipeline_inputs = SortedDictionary()
                pipeline_outputs = SortedDictionary()
                for name, plug in six.iteritems(node.plugs):
                        pipeline_outputs[name] = plug
                    else:
                        pipeline_inputs[name] = plug
                if pipeline_inputs and 'inputs' not in self.gnodes:
                    self._add_node(
                        'inputs', NodeGWidget(
                            'inputs', pipeline_inputs, pipeline,
                            process=pipeline,
                            colored_parameters=self.colored_parameters,
                            logical_view=self.logical_view,
                            userlevel=self.userlevel))
                if pipeline_outputs and 'outputs' not in self.gnodes:
                        'outputs', NodeGWidget(
                            'outputs', pipeline_outputs, pipeline,
            elif node_name not in self.gnodes:
                if isinstance(node, PipelineNode):
                    sub_pipeline = node.process
                else:
                    sub_pipeline = None

        # links
        to_remove = []
            source_node_name, source_param = source
            dest_node_name, dest_param = dest
            if source_node_name == 'inputs':
                source_node_name = ''
            if dest_node_name == 'outputs':
                dest_node_name = ''
            source_node = pipeline.nodes.get(source_node_name)
            if source_node is None:
                to_remove.append(source_dest)
                continue
            source_plug = source_node.plugs.get(source_param)
            dest_node = pipeline.nodes.get(dest_node_name)
            if dest_node is None:
                continue
            dest_plug = dest_node.plugs.get(dest_param)
            remove_glink = False
            if source_plug is None or dest_plug is None:
                # plug[s] removed
                remove_glink = True
            else:
                active = source_plug.activated and dest_plug.activated
                weak = [x[4] for x in source_plug.links_to \
                        if x[:2] == (dest_node_name, dest_param)]
                if len(weak) == 0:
                    # link removed
                    remove_glink = True
                else:
                    weak = weak[0]
            if remove_glink:
            else:
                glink.update_activation(active, weak, "current")
        for source_dest in to_remove:
        # check added links
        for source_node_name, source_node in six.iteritems(pipeline.nodes):
            for source_parameter, source_plug \
                    in six.iteritems(source_node.plugs):
                for (dest_node_name, dest_parameter, dest_node, dest_plug,
                     weak_link) in source_plug.links_to:
                    if dest_node is pipeline.nodes.get(dest_node_name):
                        self.add_link(
                            (source_node_name, source_parameter),
                            (dest_node_name, dest_parameter),
                            active=source_plug.activated \
                                   and dest_plug.activated,
                            weak=weak_link)
        self._update_steps()

    def _update_steps(self):
        if not hasattr(pipeline, 'pipeline_steps'):
            return
        steps = pipeline.pipeline_steps
        if steps is None:
            return
            gnode = self.gnodes.get(node_name)
            if gnode is None:
                continue
            labels = ['step: %s' % n for n in steps.user_traits()
                      if node_name in steps.trait(n).nodes]
            #print('update step labels on', node_name, ':', labels)
            gnode.update_labels(labels)

    def _update_logical_pipeline(self):
        # update nodes plugs and links in logical view mode
        # nodes state
            if not gnode.logical_view:
                gnode.logical_view = True
            else:
                    continue
            gnode.active = node.activated
            gnode.update_node()

            self.removeItem(self.gnodes[node_name])
            sip.transferback(self.gnodes[node_name])
            del self.gnodes[node_name]

                        pipeline_outputs['outputs'] = plug
                    else:
                        pipeline_inputs['inputs'] = plug
                else:

        # links
        # delete all links
        # recreate links

    def set_enable_edition(self, state=True):
        self._enable_edition = state

    def edition_enabled(self):
        return self._enable_edition

        super(PipelineScene, self).keyPressEvent(event)
        if not event.isAccepted():
            if event.key() == QtCore.Qt.Key_P:
                # print position of boxes
                event.accept()
                pview = self.parent()
                pview.print_node_positions()
            elif event.key() == QtCore.Qt.Key_T:
                for item in self.items():
                    if isinstance(item, boxItem):
                        item.focusOutEvent(Qt.QFocusEvent(Qt.QEvent.FocusOut))
                # toggle logical / full view
                pview.switch_logical_view()
            elif event.key() == QtCore.Qt.Key_A:
                # auto-set nodes positions
                pview.auto_dot_node_positions()

        #             elif Qt.QKeySequence(event.key()+int(event.modifiers())) == Qt.QKeySequence("Ctrl+Z"):
        #                 self.undoTyping_clicked.emit()

    def link_tooltip_text(self, source_dest):
        '''Tooltip text for the fiven link

        source_dest: tuple (2 tuples of 2 strings)
            link description:
            ((source_node, source_param), (dest_node, dest_param))
        '''
        source_node_name = source_dest[0][0]
        dest_node_name = source_dest[1][0]
        if source_node_name in ('inputs', 'outputs'):
            proc = self.pipeline
            source_node_name = ''
            source_node = self.pipeline.nodes[source_node_name]
        else:
            proc = source_node
            if hasattr(source_node, 'process'):
                proc = source_node.process
        if dest_node_name in ('inputs', 'outputs'):
            dest_node_name = ''
        splug = source_node.plugs[source_dest[0][1]]
        link = [l for l in splug.links_to \
                if l[0] == dest_node_name and l[1] == source_dest[1][1]][0]
        if splug.activated and link[3].activated:
            active = '<font color="#ffa000">activated</font>'
        else:
            active = '<font color="#a0a0a0">inactive</font>'
        if link[4]:
            weak = '<font color="#e0c0c0">weak</font>'
        else:
            weak = '<b>strong</b>'
        name = source_dest[0][1]
        value = getattr(proc, name)
        # trait = proc.user_traits()[name]
        trait_type = proc.user_traits()[name].trait_type
        trait_type_str = str(trait_type)
        trait_type_str = trait_type_str[: trait_type_str.find(' object ')]
        trait_type_str = trait_type_str[trait_type_str.rfind('.') + 1:]
        inst_type = self.get_instance_type_string(value)
        typestr = ('%s (%s)' % (inst_type, trait_type_str)).replace(
            '<', '').replace('>', '')
        msg = '''<h3>%s</h3>
<table cellspacing="6">
    <tr>
      <td><b>Link:</b></td>
      <td>%s</td>
    </tr>
</table>
<table>
  <tr>
    <td><b>type:</b></td>
    <td>%s</td>
  </tr>
  <tr>
    <td><b>value:</b></td>
  </tr>
''' \
              % (source_dest[0][1], active, weak, typestr,
                 html.escape(str(value)))
        if isinstance(trait_type, traits.File) \
                or isinstance(trait_type, traits.Directory) \
                or isinstance(trait_type, traits.Any):
            if self.is_existing_path(value):
                msg += '''    <tr>
      <td></td>
      <td>existing path</td>
    </tr>
'''
            elif not isinstance(trait_type, traits.Any):
      <td></td>
      <td><font color="#a0a0a0">non-existing path</font></td>
    </tr>
'''
        msg += '</table>'
        return msg

    def get_instance_type_string(value):
            return 'None'
        if value is traits.Undefined:
            return 'Undefined'
        if isinstance(value, (list, traits.TraitListObject)):
            return 'list'
        return type(value).__name__

    def is_existing_path(value):
        if value not in (None, traits.Undefined) \
                and type(value) in (str, six.text_type) and os.path.exists(value):

    def html_doc(doc_text):
        # TODO: sphinx transform
        text = doc_text.replace('<', '&lt;')
        text = text.replace('>', '&gt;')
        return text

    def plug_tooltip_text(self, node, name):
        '''Tooltip text for a node plug
        '''
        if node.name in ('inputs', 'outputs'):
            splug = self.pipeline.pipeline_node.plugs[name]
        else:
            src = self.pipeline.nodes[node.name]
            splug = src.plugs.get(name)
            if not splug:
            proc = src
            if hasattr(src, 'process'):
                proc = src.process
        if splug.output:
            output = '<font color="#d00000">output</font>'
        else:
            output = '<font color="#00d000">input</font>'
        if splug.enabled:
            enabled = 'enabled'
        else:
            enabled = '<font color="#a0a0a0">disabled</font>'
        if splug.activated:
            activated = 'activated'
        else:
            activated = '<font color="#a0a0a0">inactive</font>'
        if splug.optional:
            optional = '<font color="#00d000">optional</font>'
        else:
            optional = 'mandatory'
        trait = proc.user_traits()[name]
        trait_type = trait.trait_type
        trait_type_str = trait_type.__class__.__name__
        if trait.output and trait.input_filename is False:
            trait_type_str += ', output filename'
        typestr = ('%s (%s)' % (self.get_instance_type_string(value),
                                trait_type_str)).replace(
    <tr>
      <td><b>Plug:</b></td>
    </tr>
</table>
<table>
    <tr>
      <td><b>type:</b></td>
    </tr>
    <tr>
      <td><b>value:</b></td>
    </tr>
''' \
              % (name, output, optional, enabled, activated, typestr,
      <td></td>
    </tr>
'''
      <td></td>
    </tr>
'''
        desc = trait.desc
        if desc:
            msg += '\n<h3>Description:</h3>\n'
            msg += self.html_doc(desc)
        return msg

    def node_tooltip_text(self, node):
        msg = getattr(process, '__doc__', '')
        # msg = self.html_doc(doc)
        return msg

    def _parentgnode(self, item):
        if qt_backend.get_qt_backend() != 'PyQt5':
            return item.parentItem()
        # in PyQt5 (certain versions at least, Ubuntu 16.04) parentItem()
        # returns something inappropriate, having the wrong type
        # QGraphicsVideoItem, probably a cast mistake, and which leads to
        # a segfault, so we have to get it a different way.
        nodes = [node for node in self.gnodes.values()
                 if item in node.childItems()]
        if len(nodes) == 1:
            return nodes[0]

    def helpEvent(self, event):
        '''
        Display tooltips on plugs and links
        '''
            event.setAccepted(False)
            super(PipelineScene, self).helpEvent(event)
            return
        item = self.itemAt(event.scenePos(), Qt.QTransform())
        if isinstance(item, Link):
            for source_dest, glink in six.iteritems(self.glinks):
                if glink is item:
                    text = self.link_tooltip_text(source_dest)
                    item.setToolTip(text)
                    break
        elif isinstance(item, Plug):
            node = self._parentgnode(item)
            for name, plug in six.iteritems(node.in_plugs):
                if plug is item:
                    break
            if not found:
                for name, plug in six.iteritems(node.out_plugs):
                    if plug is item:
                        break
            if found:
                text = self.plug_tooltip_text(node, name)
                item.setToolTip(text)
        elif isinstance(item, QtGui.QGraphicsRectItem):
            if isinstance(node, NodeGWidget):
                text = self.node_tooltip_text(node)
        elif isinstance(item, QtGui.QGraphicsProxyWidget):
            # PROBLEM: tooltips in child graphics scenes seem not to popup.
            #
            # to force them we would have to translate the event position to
            # the sub-scene position, and call the child scene helpEvent()
            # method, with a custom event.
            # However this is not possible, since QGraphicsSceneHelpEvent
            # does not provide a public (nor even protected) constructor, and
            # secondarily helpEvent() is protected.

        super(PipelineScene, self).helpEvent(event)

        print(self.gnodes)
        gnode = self.gnodes.get(node_name)
        if gnode is None:
            # already done (possibly via a notification)
            return
        todel = set()
        from soma.qt_gui.qt_backend import sip
        for link, glink in six.iteritems(self.glinks):
            if link[0][0] == node_name or link[1][0] == node_name:
                self.removeItem(glink)
                todel.add(link)
        for link in todel:
            del self.glinks[link]
        self.removeItem(gnode)
        sip.transferback(self.gnodes[node_name])
        del self.gnodes[node_name]

    def _link_right_clicked(self, link):
        # find the link in list
        # print('Scene._link_right_clicked:', link)
            if glink is link:
                self.link_right_clicked.emit(
                    source_dest[0][0], source_dest[0][1],
                    source_dest[1][0], source_dest[1][1])
                break

    def _link_keydelete_clicked(self, link):
                self.link_keydelete_clicked.emit(
                break

    def _node_keydelete_clicked(self, node):
        self.node_keydelete_clicked.emit(node.name)

    def typeLink(self, name_node, name_plug):

        if name_node in ('inputs', 'outputs'):
            splug = self.pipeline.pipeline_node.plugs[name_plug]
        else:
            src = self.pipeline.nodes[name_node]
            splug = src.plugs[name_plug]
            proc = src

        value = getattr(proc, name_plug)

        trait = proc.user_traits()[name_plug]
        return trait_type_str

class PipelineDeveloperView(QGraphicsView):
    '''
    Pipeline representation as a graph, using boxes and arrows.

    Based on Qt QGraphicsView, this can be used as a Qt QWidget.

    Qt signals are emitted on a right click on a node box, and on a double
    click on a sub-pipeline box, to allow handling at a higher level. Default
    behaviors can be enabled using constructor parameters.

    Ctrl + double click opens sub-pipelines in embedded views inside their
    parent box.

    subpipeline_clicked
    node_right_clicked
    process_clicked
    node_clicked
    node_clicked_ctrl
    plug_clicked
    plug_right_clicked
    link_right_clicked
    colored_parameters
    scene

    Methods
    -------
    __init__
    set_pipeline
    is_logical_view
    set_logical_view
    zoom_in
    zoom_out
    openProcessController
    add_embedded_subpipeline
    onLoadSubPipelineClicked
    onOpenProcessController
    enableNode
    enable_step
    disable_preceding_steps
    disable_following_steps
    enable_preceding_steps
    enable_following_steps
    set_switch_value
    disable_done_steps
    enable_all_steps
    check_files
    auto_dot_node_positions
    save_dot_image_ui
    reset_initial_nodes_positions
    window
    '''
    '''Signal emitted when a sub pipeline has to be open.'''
    '''Signal emitted when a node box has to be open.'''
    '''Signal emitted when a node box has to be in the foreground.'''
    '''Signal emitted when a switch box has to be open.'''
    '''Signal emitted when a node box is right-clicked'''
    '''Signal emitted when a plug is clicked'''
    '''Signal emitted when a plug is right-clicked'''
    '''Signal emitted when a link is right-clicked'''
    edit_sub_pipeline = QtCore.Signal(Pipeline)
    '''Signal emitted when a sub-pipeline has to be edited'''
    open_filter = QtCore.Signal(str)
    '''Signal emitted when an Input Filter has to be opened'''
    export_to_db_scans = QtCore.Signal(str)
    '''Signal emitted when an Input Filter has to be linked to database_scans'''

    scene = None
    '''
    type: PipelineScene

    the main scene.
    '''
    colored_parameters = True
    '''
    If enabled (default), parameters in nodes boxes are displayed with color
    codes representing their state, and the state of their values: output
    parameters, empty values, existing files, non-existing files...

    When colored_parameters is set, however, callbacks have to be installed to
    track changes in traits values, so this actually has an overhead.
    When colored_parameters is used, the color code is as follows:

    * black pamameter name: input
    * red parameter name: output
    * italics parameter name: Undefined, None, or empty string value
    * bold parameter name: existing file or directory name
    * regular font parameter name: non-existing file, or non-file parameter type
    * black plug: mandatory
    * green plug: optional
    * grey plug: mandatory, inactive
    * light green plug: optional, inactive
    * grey link: inactive
    * orange link: active
    * dotted line link: weak link
    '''

    class ProcessNameEdit(Qt.QLineEdit):
        ''' A specialized QLineEdit with completion for process name
        '''
        def __init__(self, parent=None,
                     class_type_check=process_instance.is_process):
            super(PipelineDeveloperView.ProcessNameEdit,
                  self).__init__(parent)
            self.compl = QtGui.QCompleter([])
            self.setCompleter(self.compl)
            self.textEdited.connect(self.on_text_edited)
            self.py_cache = {}  # cache for loaded python files
            self.class_type_check = class_type_check

        @staticmethod
        def _execfile(filename):
            glob_dict = {}
            exec(compile(open(filename, "rb").read(), filename, 'exec'),
                 glob_dict, glob_dict)
            return glob_dict

        def load_py(self, filename):
            if filename not in self.py_cache:
                try:
                    self.py_cache[filename] = self._execfile(filename)
                    print('exception while executing file %s:' % filename, e)
                    return {}
            return self.py_cache[filename]

        def get_processes_or_modules(self, filename):
            file_dict = self.load_py(filename)
            processes = []
            for name, item in six.iteritems(file_dict):
                if self.class_type_check(item) or inspect.ismodule(item):
                    processes.append(name)
            return processes

        def on_text_edited(self, text):
            compl = set()
            modpath = str(text).split('.')
            current_mod = None
            paths = []
            sel = set()
            mod = None
            if len(modpath) > 1:
                current_mod = '.'.join(modpath[:-1])
                try:
                    mod = importlib.import_module(current_mod)
                except ImportError:
                    mod = None
                if mod:
                    if os.path.basename(mod.__file__).startswith(
                            '__init__.py'):
                        paths = [os.path.dirname(mod.__file__)]
                    # add process/pipeline objects in current_mod
                    procs = [item for k, item
                             in six.iteritems(mod.__dict__)
                             if self.class_type_check(item)
                             or inspect.ismodule(item)]
                    compl.update(['.'.join([current_mod, c.__name__])
                                  for c in procs])
            if not mod:
                # no current module
                # is it a path name ?
                pathname, filename = os.path.split(str(text))
                if os.path.isdir(pathname):
                    # look for class in python file filename.py#classname
                    elements = filename.split('.py#')
                    if len(elements) == 2:
                        filename = elements[0] + '.py'
                        object_name = elements[1]
                        full_path = os.path.join(pathname, filename)
                        processes = self.get_processes_or_modules(full_path)
                        if object_name != '':
                            processes = [p for p in processes
                                         if p.startswith(object_name)]
                        compl.update(['#'.join((full_path, p))
                                      for p in processes])
                    else:
                        # look for matching xml files
                        for f in os.listdir(pathname):
                            if (f.endswith('.xml') or f.endswith('.json')
                                or os.path.isdir(os.path.join(pathname,
                                                              f))) \
                                    and f.startswith(filename):
                                compl.add(os.path.join(pathname, f))
                            elif f.endswith('.py'):
                else:
                    paths = sys.path
            for path in paths:
                if path == '':
                    path = '.'
                try:
                    for f in os.listdir(path):
                        if f.endswith('.py'):
                            sel.add(f[:-3])
                        elif f.endswith('.pyc') or f.endswith('.pyo'):
                            sel.add(f[:-4])
                        elif f.endswith('.xml') or f.endswith('.json'):
                            sel.add(f)
                        elif '.' not in f \
                                and os.path.isdir(os.path.join(
                            path, f)):
                except OSError:
                    pass
            begin = modpath[-1]
            cm = []
            if current_mod is not None:
                cm = [current_mod]
            compl.update(['.'.join(cm + [f]) for f in sel \
                          if f.startswith(modpath[-1])])
            model = self.compl.model()
            model.setStringList(list(compl))

    def __init__(self, pipeline=None, parent=None, show_sub_pipelines=False,
                 allow_open_controller=False, logical_view=False,
                 enable_edition=False, userlevel=0):
        '''PipelineDeveloperView

        pipeline:  Pipeline (optional)
            pipeline object to be displayed
            If omitted an empty pipeline will be used, and edition mode will be
            activated.
        parent:  QWidget (optional)
            parent widget
        show_sub_pipelines:  bool (optional)
            if set, sub-pipelines will appear as red/pink boxes and a double
            click on one of them will open another window with the sub-pipeline
            structure in it
        allow_open_controller:  bool (optional)
            if set, a right click on any box will open another window with the
            underlying node controller, allowing to see and edit parameters
            values, switches states, etc.
        logical_view:  bool (optional)
            if set, plugs and links between plugs are hidden, only links
            between nodes are displayed.
        enable_edition: bool (optional)
            if set, pipeline edition features are available in GUI and menus:
            adding process boxes, drawing links etc. If pipeline is not
            specified, then edition will be activated anyway.
        '''

        super(PipelineDeveloperView, self).__init__(parent)

        # self.setAlignment(QtCore.Qt.AlignTop | QtCore.Qt.AlignLeft)
        self.setAlignment(QtCore.Qt.AlignCenter)
        self.centerOn(0,0)

        self.setRenderHints(Qt.QPainter.Antialiasing | Qt.QPainter.SmoothPixmapTransform)
        self.setBackgroundBrush(QtGui.QColor(60, 60, 60))
        self.scene = None
        self._show_sub_pipelines = show_sub_pipelines
        self._allow_open_controller = allow_open_controller
        self._logical_view = logical_view
        self._enable_edition = enable_edition
        self._pipeline_filename = ""
        self._restricted_edition = False
        self.disable_overwrite = False
        self.doc_browser = None

        self._grab = False
        self._grab_link = False
        self.plug_clicked.connect(self._plug_clicked)
        self.plug_right_clicked.connect(self._plug_right_clicked)
        self.link_right_clicked.connect(self._link_clicked)
        self.node_clicked_ctrl.connect(self._node_clicked_ctrl)
        self.link_keydelete_clicked.connect(self._link_delete_clicked)
        self.node_keydelete_clicked.connect(self._node_delete_clicked)

    # def __del__(self):
    #     # print('PipelineDeveloperView.__del__')
    #     self.release_pipeline(delete=True)
    #     # super(PipelineDeveloperView, self).__del__()

    @property

        if self.scene is not None and not sip.isdeleted(self.scene):
            self.scene.userlevel = value
        for widget in self.findChildren(QtGui.QWidget):
            if sip.isdeleted(widget):
                continue
            if hasattr(widget, 'userlevel'):
                widget.userlevel = value

    def ensure_pipeline(self, pipeline):
        '''
        Check that we have a pipeline or a process
        '''
        if pipeline is None:
            enable_edition = True

        if not isinstance(pipeline, Pipeline):
            if isinstance(pipeline, Process):
                process = pipeline
                pipeline = Pipeline()
                pipeline.set_study_config(process.get_study_config())
                pipeline.add_process(process.name, process)
                pipeline.autoexport_nodes_parameters()
                pipeline.node_position["inputs"] = (0., 0.)
                pipeline.node_position[process.name] = (300., 0.)
                pipeline.node_position["outputs"] = (600., 0.)
                # pipeline.scene_scale_factor = 0.5
                pipeline.node_dimension[process.name] = (300., 200.) #add by Irmage OM
            else:
                raise Exception("Expect a Pipeline or a Process, not a "
                                "'{0}'.".format(repr(pipeline)))

    def _set_pipeline(self, pipeline):
        pos = {}
        dim = {}
        if self.scene:
            pos = self.scene.pos
            dim = self.scene.dim #add by Irmage OM
            # pprint(dict((i, (j.x(), j.y())) for i, j in six.iteritems(pos)))
        if hasattr(pipeline, 'node_position'):
            for i, j in six.iteritems(pipeline.node_position):
                if isinstance(j, QtCore.QPointF):
                    pos[i] = j
                else:
                    pos[i] = QtCore.QPointF(*j)
        
        ############### add by Irmage OM #######################
        if hasattr(pipeline, 'node_dimension'):
            for i, j in six.iteritems(pipeline.node_dimension):
                    dim[i] = (j.x(), j.y())
                else:
                    dim[i] = j
                    
#         print("_set_pipeline : ",pos," ; ",dim)
        #######################################################            
        self.scene.set_pipeline(pipeline)
        self.scene.pos = pos
        self.scene.dim = dim
            self.setWindowTitle(pipeline.name)
            # Try to initialize the scene scale factor
            if hasattr(pipeline, "scene_scale_factor"):
                self.scale(
                    pipeline.scene_scale_factor, pipeline.scene_scale_factor)

            self.reset_initial_nodes_positions()
        
        self.fitInView(self.sceneRect(), QtCore.Qt.KeepAspectRatio)
        #############################################################################
        
        '''
        Assigns a new pipeline to the view.
        '''
        pipeline = self.ensure_pipeline(pipeline)
        self._set_pipeline(pipeline)
            # Setup callback to update view when pipeline state is modified
            pipeline.on_trait_change(self._reset_pipeline, 'selection_changed',
                                    dispatch='ui')
            pipeline.on_trait_change(self._reset_pipeline,
                                     'user_traits_changed', dispatch='ui')
            if hasattr(pipeline, 'pipeline_steps'):
                pipeline.pipeline_steps.on_trait_change(
                    self._reset_pipeline, dispatch='ui')
                
    def closeEvent(self, event):
        """Ensure pipeline is released before the widget is closed."""
        super().closeEvent(event)

    def release_pipeline(self, delete=False):
        '''
        Releases the pipeline currently viewed (and remove the callbacks)

        If ``delete`` is set, this means the view is within deletion process
        and a new scene should not be built
        '''
        # Setup callback to update view when pipeline state is modified

        try:

            if (
                hasattr(self, 'scene')
                and self.scene is not None
                and hasattr(self.scene, 'pipeline')
            ):
                pipeline = self.scene.pipeline

            if pipeline is not None:

                if hasattr(pipeline, 'pipeline_steps'):
                    pipeline.pipeline_steps.on_trait_change(
                        self._reset_pipeline, remove=True
                    )

                pipeline.on_trait_change(
                    self._reset_pipeline, 'selection_changed', remove=True
                )
                    self._reset_pipeline, 'user_traits_changed', remove=True
                )

            if sip.isdeleted(self):
                # prevent 'C++ object has been deleted' error
                return
        
            self.setScene(None)

            if hasattr(self, 'scene') and self.scene:

                # force destruction of scene internals now that the Qt object
                # still exists
                try:
                    self.scene._release()
    
                    pass
            
                # the scene is not deleted after all refs are released, even
                # after self.setScene(None). This is probably a bug in PyQt:
                # the C++ layer keeps ownership of the scene, whereas it should
                # not: the Qt doc specifies for QGraphicsView.setScene():
                # "The view does not take ownership of scene.", however in PyQt it
                # does, and only releases it when the QGraphicsView is deleted.
                # Thus we have to force it by hand:
                try:
                    sip.transferback(self.scene)

                    pass

                self.scene = None
                import gc
                gc.collect()

            if not delete and (pipeline is not None or self.scene is None):

                try:
                    self.scene = PipelineScene(self, userlevel=self.userlevel)
                    self.scene.set_enable_edition(self._enable_edition)
                    self.scene.logical_view = self._logical_view
                    self.scene.colored_parameters = self.colored_parameters
                    self.scene.subpipeline_clicked.connect(self.subpipeline_clicked)
                    self.scene.subpipeline_clicked.connect(self.onLoadSubPipelineClicked)
                    self.scene.process_clicked.connect(self._node_clicked)
                    self.scene.node_clicked.connect(self._node_clicked)
                    self.scene.node_clicked_ctrl.connect(self._node_clicked_ctrl)
                    self.scene.switch_clicked.connect(self.switch_clicked)
                    self.scene.node_right_clicked.connect(self.node_right_clicked)
                    self.scene.node_right_clicked.connect(self.onOpenProcessController)
                    self.scene.plug_clicked.connect(self.plug_clicked)
                    self.scene.plug_right_clicked.connect(self.plug_right_clicked)
                    self.scene.link_right_clicked.connect(self.link_right_clicked)
                    self.scene.link_keydelete_clicked.connect(self.link_keydelete_clicked)
                    self.scene.node_keydelete_clicked.connect(self.node_keydelete_clicked)
                    self.scene.pos = {}
                    self.scene.dim = {}
                    self.setWindowTitle('<no pipeline>')
                    self.setScene(self.scene)

                    print(
                        f"[release_pipeline] Exception "
                        f"during scene rebuild: {e}"
                    )

            print(f"[release_pipeline] Exception: {e}")

    def is_logical_view(self):
        '''
        in logical view mode, plugs and links between plugs are hidden, only
        links between nodes are displayed.
        '''
        return self._logical_view

    def set_logical_view(self, state):
        '''

        state:  bool (mandatory)
            to set/unset the logical view mode
        '''
        self._logical_view = state
        self._reset_pipeline()

    def _reset_pipeline(self):
        # self._set_pipeline(pipeline)
        self.scene.logical_view = self._logical_view
        self.scene.update_pipeline()

    def zoom_in(self):
        '''
        Zoom the view in, applying a 1.2 zoom factor
        '''
        scf = 1.2
        self.scale(scf, scf)
        cur_pos = self.mapFromGlobal(Qt.QCursor.pos())
        c_pos = Qt.QPointF(self.width() / 2, self.height() / 2)
        p = (cur_pos - c_pos) * (scf - 1.)
        self.horizontalScrollBar().setValue(
            self.horizontalScrollBar().value() + int(p.x()))
        self.verticalScrollBar().setValue(
            self.verticalScrollBar().value() + int(p.y()))

    def zoom_out(self):
        '''
        Zoom the view out, applying a 1/1.2 zool factor
        '''
        scf = 1. / 1.2

        '''
        Get the editable state
        '''

        '''
        Set the editable state. Edition allows to modify a pipeline: adding /
        removing process boxes and switches, drawing links, etc.
        '''
        self.scene.set_enable_edition(state)

    def is_restricted_edition_mode(self):
        '''
        Get the restricted mode status

        Returns
        -------
        enabled: bool
        '''
        return self._restricted_edition

    def set_restricted_edition_mode(self, enabled):
        '''
        Set the restricted edition mode. In restricted mode, some background
        menu actions ("add process", "open node controller"...) are not
        available.

        '''
        self._restricted_edition = enabled

    def wheelEvent(self, event):
        done = False
        if event.modifiers() == QtCore.Qt.ControlModifier:
            item = self.itemAt(event.pos())
            if not isinstance(item, QtGui.QGraphicsProxyWidget):
                done = True
                if qt_backend.get_qt_backend() == 'PyQt5':
                    delta = event.angleDelta().y()
                else:
                    delta = event.delta()
                if delta < 0:
                    self.zoom_out()
                else:
                    self.zoom_in()
        if not done:
            super(PipelineDeveloperView, self).wheelEvent(event)

        super(PipelineDeveloperView, self).mousePressEvent(event)
            if event.button() == QtCore.Qt.RightButton:
                self.open_background_menu()
            else:
                self._grab = True
                self._grabpos = event.pos()

            #             print("background clicked")

            for source_dest, glink in six.iteritems(self.scene.glinks):
                glink.fonced_viewer(False)
            for node_name, gnode in six.iteritems(self.scene.gnodes):
                gnode.fonced_viewer(False)

    def mouseReleaseEvent(self, event):
        if self._grab_link:
            try:
                self._release_grab_link(event)
                print("source to destination types are not compatible")
                print(e)

        super(PipelineDeveloperView, self).mouseReleaseEvent(event)
        self.scene.update()

    def mouseMoveEvent(self, event):
        if self._grab:
            translation = event.pos() - self._grabpos
            self._grabpos = event.pos()
            self.horizontalScrollBar().setValue(
                self.horizontalScrollBar().value() - int(translation.x()))
            self.verticalScrollBar().setValue(
                self.verticalScrollBar().value() - int(translation.y()))
        elif self._grab_link:
            self._move_grab_link(event)
        else:
            super(PipelineDeveloperView, self).mouseMoveEvent(event)

    def dragEnterEvent(self, event):
        """Event handler when the mouse enters the widget.

        :param event: event
        """

        if event.mimeData().hasFormat('component/name'):

    def dragMoveEvent(self, event):
        """Event handler when the mouse moves in the widget.

        """

    def dropEvent(self, event):
        """Event handler when something is dropped in the widget.

        """

            self.click_pos = QtGui.QCursor.pos()
            path = bytes(event.mimeData().data('component/name'))
            self.drop_process(path.decode('utf8'))

    def drop_process(self, path):
        """Find the dropped process in the system's paths.

        :param path: class's path (e.g. "nipype.interfaces.spm.Smooth") (str)
        """

        package_name, process_name = os.path.splitext(path)
        process_name = process_name[1:]
        __import__(package_name)
        pkg = sys.modules[package_name]
        for name, instance in sorted(list(pkg.__dict__.items())):
            if name == process_name:
                if issubclass(instance, Node):
                    # it's a node
                    try:
                        QtGui.QApplication.setOverrideCursor(
                            QtCore.Qt.WaitCursor)
                        self.add_named_node(None, instance)
                        QtGui.QApplication.restoreOverrideCursor()
                        return
                        print(e)
                        return
                try:
                    process = get_process_instance(instance)
                    print(e)
                    return
                else:
                    QtGui.QApplication.setOverrideCursor(QtCore.Qt.WaitCursor)
                    self.add_named_process(instance)
                    QtGui.QApplication.restoreOverrideCursor()

    def add_embedded_subpipeline(self, subpipeline_name, scale=None):
        '''
        Adds an embedded sub-pipeline inside its parent node.
        '''
        gnode = self.scene.gnodes.get(str(subpipeline_name))
        if gnode is not None:
            sub_pipeline \
                = self.scene.pipeline.nodes[str(subpipeline_name)].process
            gnode.add_subpipeline_view(
                sub_pipeline, self._allow_open_controller, scale=scale)

    def onLoadSubPipelineClicked(self, node_name, sub_pipeline, modifiers):
        """ Event to load a open a sub-pipeline view.
        If ctrl is pressed the new view will be embedded in its parent node
        box.
        """
        if self._show_sub_pipelines:
            if modifiers & QtCore.Qt.ControlModifier:
                try:
                    self.add_embedded_subpipeline(node_name)
                    return
                    print('node not found in:')
                    print(list(self.scene.gnodes.keys()))
                show_sub_pipelines=self._show_sub_pipelines,
                allow_open_controller=self._allow_open_controller,
                enable_edition=self.edition_enabled(),
                logical_view=self._logical_view, userlevel=self.userlevel)
            # set self.window() as QObject parent (not QWidget parent) to
            # prevent the sub_view to close/delete immediately
            QtCore.QObject.setParent(sub_view, self.window())
            sub_view.setAttribute(QtCore.Qt.WA_DeleteOnClose)
            sub_view.setWindowTitle(node_name)
            sub_view.doc_browser = self
            self.scene.update()
            sub_view.show()

    def window(self):
        '''
        window() is overloaded from QWidget.window() to handle embedded views
        cases.
        A PipelineDeveloperView may be displayed inside a NodeGWidget.
        In this case, we want to go up to the parent scene's window to the
        "real" top window, where QWidget.window() will end in the current
        graphics scene.
        '''
        if hasattr(self, '_graphics_item'):
            return self._graphics_item.scene().views()[0].window()
        else:
            return super(PipelineDeveloperView, self).window()

    def onOpenProcessController(self, node_name, process):
        """ Event to open a sub-process/sub-pipeline controller
        """
        if self._allow_open_controller:
            self.open_node_menu(node_name, process)

    def openProcessController(self):
        sub_view = QtGui.QScrollArea()
        node_name = self.current_node_name
        if node_name in ('inputs', 'outputs'):
            node_name = ''
        process = self.scene.pipeline.nodes[node_name]
        if hasattr(process, 'process'):
        # force instantiating a completion engine (since
        # AttributedProcessWidget does not force it)
        if hasattr(process, 'get_study_config'):  # exclude custom nodes
            engine = process.get_study_config().engine
            ce = ProcessCompletionEngine.get_completion_engine(process)

        cwidget = AttributedProcessWidget(
            process, enable_attr_from_filename=True, enable_load_buttons=True,
        sub_view.setWidget(cwidget)
        sub_view.setWidgetResizable(True)
        sub_view.setAttribute(QtCore.Qt.WA_DeleteOnClose)
        sub_view.setWindowTitle(self.current_node_name)
        # try to resize to a width that doesn't need an horizontal scrollbar
        sub_view.resize(
            cwidget.controller_widget.parent().parent().sizeHint().width(),
            sub_view.sizeHint().height())
        sub_view.show()
        # set self.window() as QObject parent (not QWidget parent) to
        # prevent the sub_view to close/delete immediately
        QtCore.QObject.setParent(sub_view, self.window())

    def open_node_menu(self, node_name, process):
        """ right-click popup menu for nodes
        """
        node_name = six.text_type(node_name)  # in case it is a QString
        node_type = 'process'
        if isinstance(process, OptionalOutputSwitch):
            node_type = 'opt. output switch'
        elif isinstance(process, Switch):
            node_type = 'switch'
        menu = QtGui.QMenu('Node: %s' % node_name, None)
        title = menu.addAction('Node: %s (%s)' % (node_name, node_type))
        title.setEnabled(False)
        menu.addSeparator()

        self.current_node_name = node_name
        self.current_process = process
        node = self.scene.pipeline.nodes[node_name]

        # Input_Filter
        if process.name == "Input_Filter":
            open_filter = menu.addAction('Open filter')
            open_filter.triggered.connect(self.emit_open_filter)

            export_to_db_scans = menu.addAction('Export to database_scans')
            export_to_db_scans.triggered.connect(self.emit_export_to_db_scans)

            menu.addSeparator()

        if not self._restricted_edition:
            controller_action = QtGui.QAction('open node controller', menu)
            controller_action.triggered.connect(self.openProcessController)
            menu.addAction(controller_action)

        disable_action = QtGui.QAction('Enable/disable node', menu)
        disable_action.setCheckable(True)
        disable_action.setChecked(node.enabled)
        disable_action.toggled.connect(self.enableNode)
        menu.addAction(disable_action)

        steps = getattr(self.scene.pipeline, 'pipeline_steps', None)
        if steps is not None:
            my_steps = [step_name for step_name in steps.user_traits()
                        if node.name in steps.trait(step_name).nodes]
            for step in my_steps:
                step_action = menu.addAction('(enable) step: %s' % step)
                step_action.setCheckable(True)
                step_state = getattr(self.scene.pipeline.pipeline_steps, step)
                step_action.setChecked(step_state)
                step_action.toggled.connect(SomaPartial(self.enable_step,
                                                        step))
            if len(my_steps) != 0:
                step = my_steps[0]
                disable_prec = menu.addAction('Disable preceding steps')
                disable_prec.triggered.connect(SomaPartial(
                    self.disable_preceding_steps, step))
                enable_prec = menu.addAction('Enable preceding steps')
                enable_prec.triggered.connect(SomaPartial(
                    self.enable_preceding_steps, step))
                step = my_steps[-1]
                disable_foll = menu.addAction('Disable following steps')
                disable_foll.triggered.connect(SomaPartial(
                    self.disable_following_steps, step))
                enable_foll = menu.addAction('Enable following steps')
                enable_foll.triggered.connect(SomaPartial(
                    self.enable_following_steps, step))

        enable_all_action = menu.addAction('Enable all steps')
        enable_all_action.triggered.connect(self.enable_all_steps)
        disable_done_action = menu.addAction(
            'Disable steps with existing outputs')
        disable_done_action.triggered.connect(self.disable_done_steps)
        check_pipeline_action = menu.addAction(
            'Check input / output files')
        check_pipeline_action.triggered.connect(self.check_files)

        if isinstance(node, Switch) \
                and not isinstance(node, OptionalOutputSwitch):
            # allow to select switch value from the menu
            submenu = menu.addMenu('Switch value')
            agroup = QtGui.QActionGroup(submenu)
            values = node.trait('switch').trait_type.values
            value = node.switch
            set_index = -1
            for item in values:
                action = submenu.addAction(item)
                action.setCheckable(True)
                action.triggered.connect(SomaPartial(
                    self.set_switch_value, node, item))
                if item == value:
                    action.setChecked(True)

        if not self.get_doc_browser(create=False):
            doc_action = menu.addAction('Show doc')
            doc_action.triggered.connect(self.show_doc)

        if self.edition_enabled():
            del_node_action = menu.addAction('Delete node')
            del_node_action.triggered.connect(self.del_node)
            if node is not self.scene.pipeline.pipeline_node:
                export_mandatory_plugs = menu.addAction(
                    'Export unconnected mandatory plugs')
                export_mandatory_plugs.triggered.connect(
                    self.export_node_unconnected_mandatory_plugs)
                export_all_plugs = menu.addAction(
                    'Export all unconnected plugs')
                export_all_plugs.triggered.connect(
                    self.export_node_all_unconnected_plugs)
                export_mandatory_inputs = menu.addAction(
                    'Export unconnected mandatory inputs')
                export_mandatory_inputs.triggered.connect(
                    self.export_node_unconnected_mandatory_inputs)
                export_all_inputs = menu.addAction(
                    'Export all unconnected inputs')
                export_all_inputs.triggered.connect(
                    self.export_node_all_unconnected_inputs)
                export_mandatory_outputs = menu.addAction(
                    'Export unconnected mandatory outputs')
                export_mandatory_outputs.triggered.connect(
                    self.export_node_unconnected_mandatory_outputs)
                export_all_outputs = menu.addAction(
                    'Export all unconnected outputs')
                export_all_outputs.triggered.connect(
                    self.export_node_all_unconnected_outputs)
                step = None
                if steps is not None:
                    my_steps = [step_name for step_name in steps.user_traits()
                                if node.name in steps.trait(step_name).nodes]
                    if len(my_steps) == 1:
                        step = my_steps[0]
                    elif len(my_steps) >= 2:
                        step = repr(my_steps)
                change_step = menu.addAction('change step: %s' % step)
                change_step.triggered.connect(self._change_step)

        gnode = self.scene.gnodes[self.current_node_name]

        show_opt_inputs = menu.addAction(
            'Show optional inputs')
        show_opt_inputs.setCheckable(True)
        show_opt_inputs.triggered.connect(
            self.show_optional_inputs
        )
        if gnode.show_opt_inputs:
            show_opt_inputs.setChecked(True)

        show_opt_outputs = menu.addAction(
            'Show optional outputs')
        show_opt_outputs.setCheckable(True)
        show_opt_outputs.triggered.connect(
            self.show_optional_outputs
        )
        if gnode.show_opt_outputs:
            show_opt_outputs.setChecked(True)

        # Emit a signal to edit the node if it is a PipelineNode
            edit_sub_pipeline = menu.addAction('Edit sub-pipeline')
            edit_sub_pipeline.triggered.connect(self.emit_edit_sub_pipeline)

        menu.exec_(QtGui.QCursor.pos())
        del self.current_node_name
        del self.current_process

    def emit_export_to_db_scans(self):
        self.export_to_db_scans.emit(self.current_node_name)

    def emit_open_filter(self):
        self.open_filter.emit(self.current_node_name)

    def emit_edit_sub_pipeline(self):
        node = self.scene.pipeline.nodes[self.current_node_name]
        sub_pipeline = node.process
        if isinstance(sub_pipeline, weakref.ProxyTypes):
            # get the "real" object
            sub_pipeline = sub_pipeline.__init__.__self__
        self.edit_sub_pipeline.emit(sub_pipeline)

    def show_optional_inputs(self):
        '''
        Added to choose to visualize optional inputs.
        '''

        connected_plugs = []

        # print(self.current_node_name)
        # print(gnode.in_plugs)

        # for source_dest, link in six.iteritems(self.scene.glinks):
        #     print(source_dest,",",self.current_node_name in str(source_dest))

        # The show_opt_inputs attribute is not changed yet
            # Verifying that the plugs are not connected to another node
            for param, pipeline_plug in six.iteritems(gnode.parameters):
                # print(param," : ",pipeline_plug.activated)
                output = (not pipeline_plug.output if gnode.name in (
                if not output:
                    if pipeline_plug.optional and pipeline_plug.links_from and gnode.show_opt_inputs :
                        connected_plugs.append(param)
            if connected_plugs:
                if len(connected_plugs) == 1:
                    text = "Please remove links from this plug:\n"
                else:
                    text = "Please remove links from these plugs:\n"
                for plug_name in connected_plugs:
                    text += plug_name + ", "
                text = text[:-2] + '.'

                msg = QMessageBox()
                msg.setIcon(QMessageBox.Critical)
                msg.setText(text)
                msg.setWindowTitle("Error while changing the view of the node")
                msg.setStandardButtons(QMessageBox.Ok | QMessageBox.Cancel)
                msg.exec_()
                return

        # Changing the show_opt_outputs attribute
        gnode.change_input_view()

    def show_optional_outputs(self):
        '''
        Added to choose to visualize optional outputs.
        '''

        # The show_opt_outputs attribute is not changed yet
                if output:
                    if pipeline_plug.optional and pipeline_plug.links_to and gnode.show_opt_outputs:
                else:

                return

        gnode.change_output_view()

    def open_background_menu(self):
        '''
        Open the right-click menu when triggered from the pipeline background.
        '''
        self.click_pos = QtGui.QCursor.pos()
        has_dot = shutil.which('dot')
        menu = QtGui.QMenu('Pipeline level menu', None)
        title = menu.addAction('Pipeline level menu')

        if self.is_logical_view():
            logical_view = menu.addAction('Switch to regular parameters view')
        else:
            logical_view = menu.addAction('Switch to logical pipeline view')
        logical_view.triggered.connect(self.switch_logical_view)
        auto_node_pos = menu.addAction('Auto arrange nodes positions')
        auto_node_pos.triggered.connect(self.auto_dot_node_positions)
        if not has_dot:
            auto_node_pos.setEnabled(False)
            auto_node_pos.setText(
                'Auto arrange nodes positions (needs graphviz/dot tool '
                'installed)')
        init_node_pos = menu.addAction('Reset to initial nodes positions')
        init_node_pos.triggered.connect(self.reset_initial_nodes_positions)
        if not hasattr(self.scene.pipeline, 'node_position') \
                or len(self.scene.pipeline.node_position) == 0:
            init_node_pos.setEnabled(False)
            init_node_pos.setText(
                'Reset to initial nodes positions (none defined')
        save_dot = menu.addAction('Save image of pipeline graph')
        save_dot.triggered.connect(self.save_dot_image_ui)
            save_dot.setEnabled(False)
            save_dot.setText(
                'Save image of pipeline graph (needs graphviz/dot tool '
        print_pos = menu.addAction('Print nodes positions')
        print_pos.triggered.connect(self.print_node_positions)

        if self._enable_edition:
            if not self._restricted_edition:
                add_proc = menu.addAction('Add process in pipeline')
                add_proc.triggered.connect(self.add_process)
            add_switch = menu.addAction('Add switch in pipeline')
            add_switch.triggered.connect(self.add_switch)
            add_optional_output_switch = menu.addAction(
                'Add optional output switch in pipeline')
            add_optional_output_switch.triggered.connect(
                self.add_optional_output_switch)
            add_iter_proc = menu.addAction('Add iterative process in pipeline')
            add_iter_proc.triggered.connect(self.add_iterative_process)
            add_node = menu.addAction('Add custom node in pipeline')
            add_node.triggered.connect(self.add_node)

            export_mandatory_plugs = menu.addAction(
                'Export unconnected mandatory plugs')
            export_mandatory_plugs.triggered.connect(
                self.export_unconnected_mandatory_plugs)
            export_all_plugs = menu.addAction(
                'Export all unconnected plugs')
            export_all_plugs.triggered.connect(
                self.export_all_unconnected_plugs)
            export_mandatory_inputs = menu.addAction(
                'Export unconnected mandatory inputs')
            export_mandatory_inputs.triggered.connect(
                self.export_unconnected_mandatory_inputs)
            export_all_inputs = menu.addAction(
                'Export all unconnected inputs')
            export_all_inputs.triggered.connect(
                self.export_all_unconnected_inputs)
            export_mandatory_outputs = menu.addAction(
                'Export unconnected mandatory outputs')
            export_mandatory_outputs.triggered.connect(
                self.export_unconnected_mandatory_outputs)
            export_all_outputs = menu.addAction(
                'Export all unconnected outputs')
            export_all_outputs.triggered.connect(
                self.export_all_unconnected_outputs)
            prune = menu.addAction('Prune unused pipeline plugs')
            prune.triggered.connect(self._prune_plugs)

            new_pipeline = menu.addAction('New pipeline (clear current)')
            new_pipeline.triggered.connect(self.new_pipeline)
            load_pipeline = menu.addAction('Load pipeline (clear current)')
            load_pipeline.triggered.connect(self.load_pipeline)
            save_parameters = menu.addAction('Save pipeline parameters')
            save_parameters.triggered.connect(self.save_pipeline_parameters)
            load_parameters = menu.addAction('Load pipeline parameters')
            load_parameters.triggered.connect(self.load_pipeline_parameters)

        save = menu.addAction('Save pipeline')
        save.triggered.connect(self.save_pipeline)

        del self.click_pos

    def enableNode(self, checked):
        if self.current_node_name in ['inputs', 'outputs']:
        else:
            node_name = self.current_node_name
        self.scene.pipeline.nodes[node_name].enabled = checked

    def enable_step(self, step_name, state):
        setattr(self.scene.pipeline.pipeline_steps, step_name, state)

    def disable_preceding_steps(self, step_name, dummy):
        # don't know why we get this additional dummy parameter (False)
        steps = self.scene.pipeline.pipeline_steps
        for step in steps.user_traits():
            if step == step_name:
                break
            setattr(steps, step, False)

    def disable_following_steps(self, step_name, dummy):
            if found:
                setattr(steps, step, False)
            elif step == step_name:
                found = True

    def enable_preceding_steps(self, step_name, dummy):
                break

    def enable_following_steps(self, step_name, dummy):
            if found:
                setattr(steps, step, True)

    def set_switch_value(self, switch, value, dummy):
        switch.switch = value

    def disable_done_steps(self):
        pipeline_tools.disable_runtime_steps_with_existing_outputs(
            self.scene.pipeline)

    def enable_all_steps(self):
        self.scene.pipeline.enable_all_pipeline_steps()

    def check_files(self):
        overwritten_outputs = pipeline_tools.nodes_with_existing_outputs(
        missing_inputs = pipeline_tools.nodes_with_missing_inputs(
        if len(overwritten_outputs) == 0 and len(missing_inputs) == 0:
                self, 'Pipeline ready', 'All input files are available. '
                                        'No output file will be overwritten.')
        else:
            dialog = QtGui.QWidget()
            layout = QtGui.QVBoxLayout(dialog)
            warn_widget = PipelineFileWarningWidget(
                missing_inputs, overwritten_outputs)
            layout.addWidget(warn_widget)
            hlay = QtGui.QHBoxLayout()
            layout.addLayout(hlay)
            hlay.addStretch()
            ok = QtGui.QPushButton('OK')
            self.ok_button = ok
            hlay.addWidget(ok)
            ok.clicked.connect(dialog.close)
            dialog.show()
            self._warn_files_widget = dialog

    def auto_dot_node_positions(self):
        '''
        Calculate pipeline nodes positions using graphviz/dot, and place the
        pipeline view nodes accordingly.
        '''
        scene = self.scene
        scale = 67.  # dpi
        nodes_sizes = dict([(name,
                             (gnode.boundingRect().width(),
                              gnode.boundingRect().height()))
                            for name, gnode in six.iteritems(scene.gnodes)])
        dgraph = pipeline_tools.dot_graph_from_pipeline(
            scene.pipeline, nodes_sizes=nodes_sizes)
        tfile, tfile_name = tempfile.mkstemp()
        os.close(tfile)
        pipeline_tools.save_dot_graph(dgraph, tfile_name)
        toutfile, toutfile_name = tempfile.mkstemp()
        os.close(toutfile)
        try:
            cmd = ['dot', '-Tplain', '-o', toutfile_name, tfile_name]
            try:
                soma.subprocess.check_call(cmd)
            except FileNotFoundError:
                # dot is not installed in the PATH. Give up
                return

            nodes_pos = self._read_dot_pos(toutfile_name)

            rects = dict([(name, node.boundingRect())
                          for name, node in six.iteritems(scene.gnodes)])
            pos = dict([(name, (-rects[name].width() / 2 + pos[0] * scale,
                                -rects[name].height() / 2 - pos[1] * scale))
                        for id, name, pos in nodes_pos])
            minx = min([x[0] for x in six.itervalues(pos)])
            miny = min([x[1] for x in six.itervalues(pos)])
            pos = dict([(name, (p[0] - minx, p[1] - miny))
                        for name, p in six.iteritems(pos)])
            #         print('pos:')
            #         print(pos)
            scene.pos = pos
            for node, position in six.iteritems(pos):
                gnode = scene.gnodes[node]
                if isinstance(position, Qt.QPointF):
                    gnode.setPos(position)
                else:
                    gnode.setPos(*position)

        finally:
            os.unlink(tfile_name)
            os.unlink(toutfile_name)

    def _read_dot_pos(self, filename):
        '''
        Read the nodes positions from a file generated by graphviz/dot, in
        "plain" text format.

        Returns
        -------
        nodes_pos: dict
            keys are nodes IDs (names), and values are 2D positions
        '''
        fileobj = open(filename)
        nodes_pos = []
            file_iter = fileobj.readlines()
        else:
            file_iter = fileobj
        for line in file_iter:
            if line.startswith('node'):
                line_els0 = line.split()
                line_els = []
                for el in line_els0:
                    if el.startswith('"') and el.endswith('"'):
                        line_els.append(el[1:-1])
                    else:
                        line_els.append(el)
                id = line_els[1]
                pos = tuple([float(x) for x in line_els[2:4]])
                name = line_els[6]
                nodes_pos.append((id, name, pos))
            elif line.startswith('edge'):
                break
        return nodes_pos

    def save_dot_image_ui(self):
        '''
        Ask for a filename using the file dialog, and save a graphviz/dot
        representation of the pipeline.
        The pipeline representation follows the current visualization mode
        ("regular" or "logical" with smaller boxes) with one link of a given
        type (active, weak) between two given boxes: all parameters are not
        represented.
        '''
        file_dialog = QtGui.QFileDialog(filter='Images (*.png *.xpm *.jpg *.ps *.eps);; All (*)')
        file_dialog.setDefaultSuffix('.png')
        file_dialog.setFileMode(QtGui.QFileDialog.AnyFile)
        file_dialog.setAcceptMode(QtGui.QFileDialog.AcceptSave)
        if file_dialog.exec_():
            filename = file_dialog.selectedFiles()

            '''filename = QtGui.QFileDialog.getSaveFileName(
                None, 'Save image of the pipeline', '',
                'Images (*.png *.xpm *.jpg *.ps *.eps);; All (*)')'''
            if filename:
                pipeline_tools.save_dot_image(self.scene.pipeline, filename[0])

    def reset_initial_nodes_positions(self):
        '''
        Set each pipeline node to its "saved" position, ie the one which may
        be found in the "node_position" variable of the pipeline.
        '''
        if scene.pipeline is None:
            return

#         ############## add by Irmage OM ###################
#         dim = getattr(scene.pipeline, 'node_dimension') 
#         if dim is not None:
#             scene.dim = dim
#             print()
#             for node, dimension in six.iteritems(dim):
#                 gnode = scene.gnodes.get(node)
#                 if gnode is not None:
#                     if isinstance(dimension, QtCore.QPointF):
#                         dimension = (dim.x(),dim.y())
# #                     else:
# #                         dimension = dim.width(),dim.height()
#                     gnode.update(0,0,*dimension)
#         #####################################################

        pos = getattr(scene.pipeline, 'node_position')
        if pos is not None:
                gnode = scene.gnodes.get(node)
                if gnode is not None:
                    if isinstance(position, QtCore.QPointF):
                        position = (position.x(), position.y())

    def switch_logical_view(self):
        self.set_logical_view(not self.is_logical_view())

    def print_node_positions(self):
        def conv_pos(p):
            if isinstance(p, Qt.QPointF):
                return (p.x(), p.y())
            return p

        posdict = dict([(key, conv_pos(value)) \
                        for key, value in six.iteritems(self.scene.pos)])
        pprint(posdict)

    def del_node(self, node_name=None):

        pipeline = self.scene.pipeline

        if not node_name:
        if node_name not in ('inputs', 'outputs'):
            pipeline.remove_node(node_name)
        elif node_name in self.scene.gnodes:
            # delete all input or output export plugs in the pipeline
            if node_name == 'inputs':
                plugs = [name
                         for name, plug in
                            six.iteritems(pipeline.pipeline_node.plugs)
                         if not plug.output]
            else:
                         if plug.output]
            for plug in plugs:
                self.scene.pipeline.remove_trait(plug)

        self.scene.pipeline.update_nodes_and_plugs_activation()

    def export_node_plugs(self, node_name, inputs=True, outputs=True,
                          optional=False):
        for parameter_name, plug in six.iteritems(node.plugs):
            if parameter_name in ("nodes_activation", "selection_changed"):
                continue
            if (((node_name, parameter_name) not in pipeline.do_not_export and
                 ((outputs and plug.output and not plug.links_to) or
                  (inputs and not plug.output and not plug.links_from)) and
                 (optional or not node.get_trait(parameter_name).optional))):
                pipeline.export_parameter(node_name, parameter_name)

    def export_plugs(self, inputs=True, outputs=True, optional=False):
        for node_name in self.scene.pipeline.nodes:
                self.export_node_plugs(node_name, inputs=inputs,
                                       outputs=outputs, optional=optional)

    def export_node_unconnected_mandatory_plugs(self):
        self.export_node_plugs(self.current_node_name)

    def export_node_all_unconnected_plugs(self):
        self.export_node_plugs(self.current_node_name, optional=True)

    def export_node_unconnected_mandatory_inputs(self):
        self.export_node_plugs(
            self.current_node_name, inputs=True, outputs=False)

    def export_node_all_unconnected_inputs(self):
            self.current_node_name, inputs=True, outputs=False, optional=True)

    def export_node_unconnected_mandatory_outputs(self):
            self.current_node_name, inputs=False, outputs=True)

    def export_node_all_unconnected_outputs(self):
            self.current_node_name, inputs=False, outputs=True, optional=True)

    def export_unconnected_mandatory_plugs(self):
        self.export_plugs()

    def export_all_unconnected_plugs(self):
        self.export_plugs(optional=True)

    def export_unconnected_mandatory_inputs(self):
        self.export_plugs(inputs=True, outputs=False)

    def export_all_unconnected_inputs(self):
        self.export_plugs(inputs=True, outputs=False, optional=True)

    def export_unconnected_mandatory_outputs(self):
        self.export_plugs(inputs=False, outputs=True)

    def export_all_unconnected_outputs(self):
        self.export_plugs(inputs=False, outputs=True, optional=True)

    def _change_step(self):
        steps_defined = True
            steps = Controller()
            steps_defined = False

        wid = Qt.QDialog()
        wid.setModal(True)
        lay = Qt.QVBoxLayout()
        wid.setLayout(lay)

        listw = Qt.QListWidget()
        listw.setSelectionMode(listw.MultiSelection)
        lay.addWidget(listw)
        n = 0
            listw.addItem(step)
            nodes = steps.trait(step).nodes
            if node_name in nodes:
                item = listw.item(n)
            n += 1
        addlay = Qt.QHBoxLayout()
        lay.addLayout(addlay)
        addb = Qt.QPushButton('+')
        addlay.addWidget(addb)
        remb = Qt.QPushButton('-')
        addlay.addWidget(remb)

        def add_clicked():
            d = Qt.QDialog()
            d.setModal(True)
            la = Qt.QHBoxLayout()
            d.setLayout(la)
            l = Qt.QLineEdit()
            la.addWidget(l)
            l.returnPressed.connect(d.accept)
            r = d.exec_()
            if r:
                name = l.text()
                if name not in steps.user_traits():
                    n = listw.count()
                    listw.addItem(name)
                    listw.item(n).setSelected(True)

        def remove_clicked():
            selected = []
            for i in range(listw.count()):
                item = listw.item(i)
                if item.isSelected():
                    selected.append((i, item.text()))
            if len(selected) != 0:
                r = Qt.QMessageBox.question(
                    wid, 'remove steps',
                    'remove the following steps from the whole pipeline ?\n%s'
                    % repr([s[1] for s in selected]))
                if r == Qt.QMessageBox.Yes:
                    for s in reversed(selected):
                        listw.takeItem(s[0])

        def up_clicked():
                    selected.append(i)
            if len(selected) != 0 and selected[0] != 0:
                for i in selected:
                    item = listw.takeItem(i)
                    listw.insertItem(i-1, item)

        def down_clicked():
            if len(selected) != 0 and selected[-1] != listw.count() - 1:
                for i in reversed(selected):
                    listw.insertItem(i + 1, item)

        addb.clicked.connect(add_clicked)
        remb.clicked.connect(remove_clicked)
        up = Qt.QPushButton('^')
        addlay.addWidget(up)
        down = Qt.QPushButton('v')
        addlay.addWidget(down)
        up.clicked.connect(up_clicked)
        down.clicked.connect(down_clicked)

        oklay = Qt.QHBoxLayout()
        lay.addLayout(oklay)
        ok = Qt.QPushButton('OK')
        oklay.addWidget(ok)
        cancel = Qt.QPushButton('Cancel')
        oklay.addWidget(cancel)
        ok.clicked.connect(wid.accept)
        cancel.clicked.connect(wid.reject)

        res = wid.exec_()
        if res:
            items = set()
            sitems = []
                name = item.text()
                sel = item.isSelected()
                items.add(name)
                sitems.append(name)
                trait = steps.trait(name)
                if sel:
                    if trait is None:
                        self.scene.pipeline.add_pipeline_step(
                            name, [node_name])
                        steps = self.scene.pipeline.pipeline_steps
                    else:
                        nodes = steps.trait(name).nodes
                        if node_name not in nodes:
                            nodes.append(node_name)
                elif trait is not None:
                    if node_name in trait.nodes:
                        trait.nodes.remove(node_name)
            steps = list(steps.user_traits().keys())
            for step in steps:
                if step not in items:
                    self.scene.pipeline.remove_pipeline_step(step)
            # reorder traits if needed
            steps = self.scene.pipeline.pipeline_steps
            if list(steps.user_traits().keys()) != sitems:
                values = [steps.trait(step).nodes for step in sitems]
                for step in sitems:
                    steps.remove_trait(step)
                for step, nodes in zip(sitems, values):
                    self.scene.pipeline.add_pipeline_step(step, nodes)

            self.scene.update_pipeline()

    class ProcessModuleInput(QtGui.QDialog):
        def __init__(self, display_str='process module/name',
            super(PipelineDeveloperView.ProcessModuleInput, self).__init__()
            self.setWindowTitle('%s:' % display_str)
            layout = QtGui.QGridLayout(self)
            layout.addWidget(QtGui.QLabel('module/process:'), 0, 0)
            self.proc_line = PipelineDeveloperView.ProcessNameEdit(
                class_type_check=class_type_check)
            layout.addWidget(self.proc_line, 0, 1)
            layout.addWidget(QtGui.QLabel('node name'), 1, 0)
            self.name_line = QtGui.QLineEdit()
            layout.addWidget(self.name_line, 1, 1)
            # hlay = QtGui.QHBoxLayout()
            # layout.addLayout(hlay, 1, 1)
            layout.addWidget(ok, 2, 0)
            cancel = QtGui.QPushButton('Cancel')
            layout.addWidget(cancel, 2, 1)
            ok.clicked.connect(self.accept)
            cancel.clicked.connect(self.reject)

    def add_process(self):
        '''
        Insert a process node in the pipeline. Asks for the process
        module/name, and the node name before inserting.
        '''

        proc_name_gui = PipelineDeveloperView.ProcessModuleInput()
        proc_name_gui.resize(800, proc_name_gui.sizeHint().height())

        res = proc_name_gui.exec_()
        if res:
            proc_module = six.text_type(proc_name_gui.proc_line.text())
            node_name = str(proc_name_gui.name_line.text())
            self.add_named_process(proc_module, node_name)

    def add_named_process(self, proc_module, node_name=None):

            if isinstance(proc_module, six.string_types):
                class_name = proc_module
            else:
                class_name = proc_module.__name__
            i = 1
            node_name = '%s_%d' % (class_name.lower(), i)

            while node_name in pipeline.nodes and i < 100:
                i += 1
                node_name = '%s_%d' % (class_name.lower(), i)

        engine = pipeline.get_study_config().engine
        try:
            process = engine.get_process_instance(proc_module)
            traceback.print_exc()
            return
        pipeline.add_process(node_name, process)

        gnode = self.scene.add_node(node_name, node)
        gnode.setPos(self.mapToScene(self.mapFromGlobal(self.click_pos)))

        return process

    def add_node(self):
        '''
        Insert a custom node in the pipeline. Asks for the node
        '''

        def is_pipeline_node(item):
            return item is not Node and isinstance(item, Node)

        node_name_gui = PipelineDeveloperView.ProcessModuleInput(
            display_str='node module/name', class_type_check=is_pipeline_node)
        node_name_gui.resize(800, node_name_gui.sizeHint().height())

        res = node_name_gui.exec_()
        if res:
            node_module = six.text_type(node_name_gui.proc_line.text())
            node_name = str(node_name_gui.name_line.text())
            self.add_named_node(node_name, node_module)

    def add_named_node(self, node_name, node_module):
        def configure_node(cls):
            conf_controller = cls.configure_controller()
            w = Qt.QDialog()
            w.setWindowTitle('Custom node parameterization')
            l = Qt.QVBoxLayout()
            w.setLayout(l)
            c = ScrollControllerWidget(conf_controller, live=True)
            l.addWidget(c)
            h = Qt.QHBoxLayout()
            l.addLayout(h)
            ok = Qt.QPushButton('OK')
            h.addWidget(ok)
            cancel = Qt.QPushButton('Cancel')
            h.addWidget(cancel)
            ok.clicked.connect(w.accept)
            cancel.clicked.connect(w.reject)
            res = w.exec_()
            if res:
                c.update_controller()
                return conf_controller
            else:

        def get_node_instance(class_str, pipeline):
            cls_and_name = process_instance.get_node_class(class_str)
            if cls_and_name is None:
            name, cls = cls_and_name
            if hasattr(cls, 'configure_controller'):
                conf_controller = configure_node(cls)
                if conf_controller is None:
                    return None # abort
            else:
                conf_controller = Controller()
            if hasattr(cls, 'build_node'):
                node = cls.build_node(pipeline, name, conf_controller)
            else:
                # probably bound to fail...
                node = cls(pipeline, name, [], [])
            return node

        try:
            node = get_node_instance(node_module, pipeline)
            print(e)
            return
            return

        if not node_name and node:
            class_name = node.__class__.__name__
            i = 1

                i += 1

        pipeline.nodes[node_name] = node
        pipeline._set_subprocess_context_name(node, node_name)

    class IterativeProcessInput(ProcessModuleInput):
        def __init__(self, engine):
            super(PipelineDeveloperView.IterativeProcessInput,
                  self).__init__()
            # hlay = Qt.QHBoxLayout()
            # self.layout().addLayout(hlay)
            lay = self.layout()
            item = lay.itemAtPosition(2, 0)
            widget = item.widget()
            lay.removeItem(item)
            lay.addWidget(widget, 3, 0)
            item = lay.itemAtPosition(2, 1)
            lay.addWidget(widget, 3, 1)
            lay.addWidget(Qt.QLabel('iterative plugs:'), 2, 0)
            self.plugs = Qt.QListWidget()
            self.plugs.setEditTriggers(Qt.QListWidget.NoEditTriggers)
            self.plugs.setSelectionMode(Qt.QListWidget.ExtendedSelection)
            lay.addWidget(self.plugs, 2, 1)
            self.proc_line.textChanged.connect(self.set_plugs)
            # self.proc_line.editingFinished.connect(self.set_plugs)
            self.engine = engine

        def set_plugs(self, text):
            self.plugs.clear()
            try:
                process = self.engine.get_process_instance(text)
                return
            traits = list(process.user_traits().keys())
            self.plugs.addItems(traits)

        def iterative_plugs(self):
            return [item.text() for item in self.plugs.selectedItems()]

    def add_iterative_process(self):
        '''
        Insert an iterative process node in the pipeline. Asks for the process
        module/name, the node name, and iterative plugs before inserting.
        '''
        proc_name_gui = PipelineDeveloperView.IterativeProcessInput(engine)

        if res:
            try:
                process = engine.get_process_instance(
                    six.text_type(proc_name_gui.proc_line.text()))
                print(e)
                return
            iterative_plugs = proc_name_gui.iterative_plugs()
            do_not_export = list(process.user_traits().keys())
            pipeline.add_iterative_process(node_name, process, iterative_plugs,
                                           do_not_export=do_not_export)

            gnode = self.scene.add_node(node_name, node)
            gnode.setPos(self.mapToScene(self.mapFromGlobal(self.click_pos)))

    def add_switch(self):
        '''
        Insert a switch node in the pipeline. Asks for the switch
        inputs/outputs, and the node name before inserting.
        '''

        class SwitchInput(QtGui.QDialog):
                super(SwitchInput, self).__init__()
                self.setWindowTitle('switch parameters/name:')
                layout = QtGui.QGridLayout(self)
                layout.addWidget(QtGui.QLabel('inputs:'), 0, 0)
                self.inputs_line = QtGui.QLineEdit()
                layout.addWidget(self.inputs_line, 0, 1)
                layout.addWidget(QtGui.QLabel('outputs:'), 1, 0)
                self.outputs_line = QtGui.QLineEdit()
                layout.addWidget(self.outputs_line, 1, 1)
                layout.addWidget(QtGui.QLabel('node name'), 2, 0)
                self.name_line = QtGui.QLineEdit()
                layout.addWidget(self.name_line, 2, 1)
                ok = QtGui.QPushButton('OK')
                layout.addWidget(ok, 3, 0)
                cancel = QtGui.QPushButton('Cancel')
                layout.addWidget(cancel, 3, 1)
                ok.clicked.connect(self.accept)
                cancel.clicked.connect(self.reject)

        switch_name_gui = SwitchInput()
        switch_name_gui.resize(600, switch_name_gui.sizeHint().height())

        res = switch_name_gui.exec_()
        if res:
            pipeline = self.scene.pipeline
            node_name = str(switch_name_gui.name_line.text()).strip()
            inputs = str(switch_name_gui.inputs_line.text()).split()
            outputs = str(switch_name_gui.outputs_line.text()).split()
            pipeline.add_switch(node_name, inputs, outputs)
            # add_switch triggers an update
            gnode = self.scene.gnodes[node_name]

    def add_optional_output_switch(self):
        '''
        Insert an optional output switch node in the pipeline. Asks for the
        switch inputs/outputs, and the node name before inserting.
        '''

                layout.addWidget(QtGui.QLabel('input:'), 0, 0)
                layout.addWidget(QtGui.QLabel('output:'), 1, 0)

        if res:
            input = str(switch_name_gui.inputs_line.text()).strip()
            output = str(switch_name_gui.outputs_line.text()).strip()
            if output == '' and node_name != '':
                output = node_name
            elif output != '' and node_name == '':
                node_name = output
            pipeline.add_optional_output_switch(node_name, input, output)
            # add_optional_output_switch does *not* trigger an update
            self._reset_pipeline()

    def _plug_clicked(self, name):
        if self.is_logical_view() or not self.edition_enabled():
            # in logival view, links are not editable since they do not reflect
            # the details of reality
            return
        node_name, plug_name = str(name).split(':')
        plug_name = str(plug_name)
        gnode = self.scene.gnodes[node_name]
        plug = gnode.out_plugs.get(plug_name)

        typeq = self.scene.typeLink(node_name, plug_name)
        try:
            #             color = self.scene.colorLink(typeq)
            color = self.scene.colType.colorLink(typeq)

        if not plug:
            return  # probably an input plug
        plug_pos = plug.mapToScene(plug.mapFromParent(plug.get_plug_point()))
        self._grabpos = self.mapFromScene(plug_pos)
        self._temp_link = Link(
            plug_pos,
            self.mapToScene(self.mapFromGlobal(QtGui.QCursor.pos())),
            True, False, color)
        self._temp_link.pen.setBrush(RED_2)
        self.scene.addItem(self._temp_link)

        self._grab_link = True
        self._grabbed_plug = (node_name, plug_name)

    def _move_grab_link(self, event):
        pos = self.mapToScene(event.pos())
        self._temp_link.update(self.mapToScene(self._grabpos), pos)

    def _release_grab_link(self, event, ret=False):
        max_square_dist = 100.
        # delete the temp link
        self.scene.removeItem(self._temp_link)
        del self._temp_link
        item = self.scene.itemAt(pos, Qt.QTransform())
        plug = None
            # look for its dest plug
            plug = None
            for source_dest, link in six.iteritems(self.scene.glinks):
                if link is item:
                    plug = source_dest[1]
                    break
            if plug is not None:
                # check the plug is not too far from the drop point
                gnode = self.scene.gnodes[plug[0]]
                gplug = gnode.in_plugs[plug[1]]
                plug_pos = gplug.mapToScene(
                    gplug.mapFromParent(gplug.get_plug_point()))
                pdiff = plug_pos - pos
                dist2 = pdiff.x() * pdiff.x() + pdiff.y() * pdiff.y()
                if dist2 > max_square_dist:
                    plug = None
            plug = str(item.name).split(':')
        if plug is not None:
            if self._grabbed_plug[0] not in ('', 'inputs'):
                src = '%s.%s' % self._grabbed_plug
            else:
                src = self._grabbed_plug[1]
            if plug[0] not in ('', 'outputs'):
                dst = '%s.%s' % tuple(plug)
            else:
                dst = plug[1]
            #             if (src != dst) and ("inputs."+src != dst) and not self.isInputYet(dst) :

            if (src != dst) and ("inputs." + src != dst):
                self.scene.pipeline.add_link('%s->%s' % (src, dst))
                self.scene.update_pipeline()

            if ret:
                self._grabbed_plug = None
                return '%s->%s' % (src, dst)
        self._grabbed_plug = None

    #     def isInputYet(self,dest):##################################################################### add by OM
    #         for listK in self.scene.glinks.keys():
    #             if ( eval(str(eval(str(listK))[1]))[0]+"."+ eval(str(eval(str(listK))[1]))[1]==dest or eval(str(eval(str(listK))[1]))[0]+"."+ eval(str(eval(str(listK))[1]))[1]=="outputs."+dest):
    #                 print("input '",dest, "' already used !!")
    #                 return True
    #         return False

    def _node_delete_clicked(self, name_node):
        self.current_node_name = name_node
        self.del_node()

    def _link_delete_clicked(self, src_node, src_plug, dst_node, dst_plug):
        src_node = str(src_node)
        src_plug = str(src_plug)
        dst_node = str(dst_node)
        dst_plug = str(dst_plug)

        #         print(src_node,",",src_plug,",",dst_node,",",dst_plug)

            # in logical view, links are not real links
            return
        if src_node in ('', 'inputs'):
            src = src_plug
            snode = self.scene.pipeline.pipeline_node
        else:
            src = '%s.%s' % (src_node, src_plug)
            snode = self.scene.pipeline.nodes[src_node]
        if dst_node in ('', 'outputs'):
            dst = dst_plug
            dnode = self.scene.pipeline.pipeline_node
        else:
            dst = '%s.%s' % (dst_node, dst_plug)
            dnode = self.scene.pipeline.nodes[dst_node]
        name = '%s->%s' % (src, dst)
        self._current_link = name  # (src_node, src_plug, dst_node, dst_plug)
        self._del_link()
        del self._current_link

    def _link_clicked(self, src_node, src_plug, dst_node, dst_plug):
            return
        else:
        else:
        self._current_link_def = (src_node, src_plug, dst_node, dst_plug)

        menu = QtGui.QMenu('Link: %s' % name)
        title = menu.addAction('Link: %s' % name)

        weak = False
        splug = snode.plugs[src_plug]
        for link in splug.links_to:
            if link[0] == dst_node and link[1] == dst_plug:
                weak = link[4]
                break
        weak_action = menu.addAction('Weak link')
        weak_action.setCheckable(True)
        weak_action.setChecked(bool(weak))
        weak_action.toggled.connect(self._change_weak_link)

        del_link = menu.addAction('Delete link')
        del_link.triggered.connect(self._del_link)

        del self._current_link_def

    def get_doc_browser(self, create=False):
        doc_browser = self.doc_browser
        pv = self
        proxy = False
        while isinstance(doc_browser, PipelineDeveloperView):
            # it's a proxy to a parent view
            pv = doc_browser
            doc_browser = pv.doc_browser
            proxy = True
        if doc_browser or not create:
            return doc_browser
        try:
            # use the newer Qt5 QtWebEngine
            from soma.qt_gui.qt_backend import QtWebEngine
            from soma.qt_gui.qt_backend.QtWebEngineWidgets \
                import QWebEngineView, QWebEnginePage
            use_webengine = True
        except ImportError as e:
            print('\n{}'.format(e))
            try:
                from soma.qt_gui.qt_backend import QtWebKit
                QWebEngineView = QtWebKit.QWebView
                QWebPage = QtWebKit.QWebPage
                QWebEnginePage = QWebPage
                use_webengine = False
            except ImportError as e:
                print('\n{}\n\nThe process documentation cannot be '
                     'displayed ...'.format(e))
        self._use_webengine = use_webengine

        class DocBrowser(QWebEngineView):
            def __init__(self, pview, *args, **kwargs):
                super(DocBrowser, self).__init__(*args, **kwargs)
                self.setAttribute(Qt.Qt.WA_DeleteOnClose)
                self.pview = pview

            def closeEvent(self, event):
                self.pview.doc_browser = None
                super(DocBrowser, self).closeEvent(event)

        doc_browser = DocBrowser(pv) # QWebEngineView()
        pv.doc_browser = doc_browser
        doc_browser.show()

        return doc_browser

    def _node_clicked(self, name, node):
        self.show_node_doc(node)
        if isinstance(node, Process):
            self.process_clicked.emit(name, node)
        else:
            self.node_clicked.emit(name, node)

    def get_node_html_doc(node):
        doc_path = getattr(node, '_doc_path', None)
        if doc_path and os.path.isabs(doc_path):
            return doc_path
        modname = node.__module__
        init_modname = modname
            mod = sys.modules[modname]
            mod_doc_path = getattr(mod, '_doc_path', None)
            if mod_doc_path:
                if doc_path:
                    return os.path.join(mod_doc_path, doc_path)
                node_type = 'process'
                if isinstance(node, Pipeline):
                    node_type = 'pipeline'
                path = os.path.join(
                    mod_doc_path, node_type,
                    '%s.html' % '.'.join((node.__module__,
                                          node.__class__.__name__)))
                if os.path.exists(path) or path.startswith('http://') \
                        or path.startswith('https://'):
                    return path
                # try using the 1st sub-module
                modsplit = init_modname.split('.')
                if len(modsplit) >= 3:
                    path = os.path.join(
                        mod_doc_path, modsplit[1], node_type,
                        '%s.html' % '.'.join((node.__module__,
                                              node.__class__.__name__)))
                    if os.path.exists(path) or path.startswith('http://') \
                            or path.startswith('https://'):
                        return path
            s = modname.rsplit('.', 1)
            if len(s) == 1:
                break
            modname = s[0]

    def show_doc(self, node_name=None):
        else:
        if isinstance(node, ProcessNode):
            node = node.process
        doc_browser = self.get_doc_browser(create=True)

    def show_node_doc(self, node):
        doc_browser = self.get_doc_browser()
        if doc_browser:
            doc_path = self.get_node_html_doc(node)
            if doc_path:
                if not doc_path.startswith('http://') \
                        and not doc_path.startswith('https://') \
                        and not doc_path.startswith('file://'):
                    doc_path = 'file://%s' % os.path.abspath(doc_path)
                doc_browser.setUrl(Qt.QUrl(doc_path))
            else:
                gethelp = getattr(node, 'get_help')
                msg = None
                if gethelp:
                    msg = node.get_help(returnhelp=True)
                if not msg:
                    msg = node.getattr(node, '__doc__', None)
                if msg:
                    doc_browser.setContent(Qt.QByteArray(msg.encode('utf-8')),
                                           'text/plain')

    def _node_clicked_ctrl(self, name, process):

        for source_dest, glink in six.iteritems(self.scene.glinks):
            glink.fonced_viewer(False)
            #             print("source-dest ",source_dest)
            if name not in str(source_dest):
                glink.fonced_viewer(True)
        #             else:
        #                 print(source_dest[0])
        for node_name, gnode in six.iteritems(self.scene.gnodes):
            #             print("    node_name",node_name)
            gnode.fonced_viewer(False)
            if name not in str(node_name):
                gnode.fonced_viewer(True)

    def _change_weak_link(self, weak):
        # src_node, src_plug, dst_node, dst_plug = self._current_link
        link_def = self._current_link
        self.scene.pipeline.remove_link(link_def)
        self.scene.pipeline.add_link(link_def, weak_link=weak)

    def _del_link(self):
        print('\nRemoving the link: ', self._current_link)
        src_node, src_plug, dst_node, dst_plug = self._current_link_def
        pipeline.remove_link(link_def)
        if (src_node in ('', 'inputs') and
          len(pipeline.pipeline_node.plugs[src_plug].links_to) == 0):
                # remove orphan pipeline plug
            pipeline.remove_trait(src_plug)
        elif (dst_node in ('', 'outputs') and
          len(pipeline.pipeline_node.plugs[dst_plug].links_from) == 0):
            # remove orphan pipeline plug
            pipeline.remove_trait(dst_plug)

    def _plug_right_clicked(self, name):

            if node_name in 'inputs':
                self.inputYet = True
            if node_name in 'outputs':
                self.outputYet = False

            return
            node = self.scene.pipeline.pipeline_node
        else:
            node = self.scene.pipeline.nodes[node_name]
        plug = node.plugs[plug_name]
        output = plug.output
        self._temp_node = node
        self._temp_plug = plug
        self._temp_plug_name = (node_name, plug_name)

        menu = QtGui.QMenu('Plug: %s' % name)
        title = menu.addAction('Plug: %s' % name)

            # not a main node: allow export
            if output:
            else:
            existing = False
                if link[0] == '':
                    existing = True
                    break
            export_action = menu.addAction('export plug')
            export_action.triggered.connect(lambda checked: self._export_plug())
            if existing:
                export_action.setEnabled(False)
            if isinstance(node, ProcessNode) \
                iter_action = menu.addAction('iterative plug')
                iter_action.setCheckable(True)
                iter_action.setChecked(
                    plug_name in node.process.iterative_parameters)
                iter_action.toggled[bool].connect(self._change_iterative_plug)

        else:
            del_plug = menu.addAction('Remove plug')
            del_plug.triggered.connect(self._remove_plug)
            edit_plug = menu.addAction('Rename / edit plug')
            edit_plug.triggered.connect(self._edit_plug)

        protect_action = menu.addAction('protected')
        protect_action.setCheckable(True)
        protect_action.setChecked(node.is_parameter_protected(plug_name))
        protect_action.toggled[bool].connect(self._protect_plug)
        complete_action = menu.addAction('completion enabled')
        complete_action.setCheckable(True)
        complete_action.setChecked(
            not node.get_trait(plug_name).forbid_completion)
        complete_action.toggled[bool].connect(self._enable_plug_completion)

        del self._temp_plug
        del self._temp_plug_name
        del self._temp_node

    class _PlugEdit(QtGui.QDialog):
        def __init__(self, show_weak=True, parent=None):
            super(PipelineDeveloperView._PlugEdit, self).__init__(parent)
            layout = QtGui.QVBoxLayout(self)
            hlay1 = QtGui.QHBoxLayout()
            layout.addLayout(hlay1)
            hlay1.addWidget(QtGui.QLabel('Plug name:'))
            hlay1.addWidget(self.name_line)
            hlay2 = QtGui.QHBoxLayout()
            layout.addLayout(hlay2)
            self.optional = QtGui.QCheckBox('Optional')
            hlay2.addWidget(self.optional)
            if show_weak:
                self.weak = QtGui.QCheckBox('Weak link')
                hlay2.addWidget(self.weak)
            hlay3 = QtGui.QHBoxLayout()
            layout.addLayout(hlay3)
            hlay3.addWidget(ok)
            hlay3.addWidget(cancel)

    def _export_plug(self):
        dial = self._PlugEdit()
        dial.name_line.setText(self._temp_plug_name[1])
        dial.optional.setChecked(self._temp_plug.optional)

        res = dial.exec_()
        if res:
            # for node_name, gnode in six.iteritems(self.scene.gnodes):
            #     print("list Nodes",node_name)
            try:
                self.scene.pipeline.export_parameter(
                    self._temp_plug_name[0], self._temp_plug_name[1],
                    pipeline_parameter=str(dial.name_line.text()),
                    is_optional=dial.optional.isChecked(),
                    weak_link=dial.weak.isChecked())
            #             print(str(dial.name_line.text()))
            #             self.scene.gnodes.changeHmin(15)
                print('exception while export plug:', e)
                pass

    def _change_iterative_plug(self, checked):
        node = self._temp_node
        node_name, name = self._temp_plug_name
        node.process.change_iterative_plug(name, checked)

    def _protect_plug(self, checked):
        node.protect_parameter(name, checked)

    def _enable_plug_completion(self, checked):
        node.get_trait(name).forbid_completion = not checked

    def _remove_plug(self):

        if self._temp_plug_name[0] in ('inputs', 'outputs'):
            # print 'remove plug:', self._temp_plug_name[1]
            #print('#' * 50)
            #print(self._temp_plug_name)
            #print(self._temp_plug)
            #for trait_name, trait in self.scene.pipeline.traits().items():
                #print(trait_name, trait)
                #if trait.handler is None:
                    #print('HANDLER IS NONE')
                #else:
                    #print('HANDLER:', trait.handler)
                    #if trait.has_items:
                        #print("HANDLER HAS ITEMS")

            self.scene.pipeline.remove_trait(self._temp_plug_name[1])

    def _edit_plug(self):
        dial = self._PlugEdit(show_weak=False)
        dial.name_line.setEnabled(False)  ## FIXME
        if res:
            plug = self._temp_plug
            plug.optional = dial.optional.isChecked()

            # print 'TODO.'

    def _prune_plugs(self):
        pnode = pipeline.pipeline_node
        to_del = []
        for plug_name, plug in six.iteritems(pnode.plugs):
            if plug.output and len(plug.links_from) == 0:
                to_del.append(plug_name)
            elif not plug.output and len(plug.links_to) == 0:
        for plug_name in to_del:
            pipeline.remove_trait(plug_name)

    def confirm_erase_pipeline(self):
        if len(self.scene.pipeline.nodes) <= 1:
        confirm = Qt.QMessageBox.warning(
            self,
            'New pipeline',
            'The current pipeline will be lost. Continue ?',
            Qt.QMessageBox.Ok | Qt.QMessageBox.Cancel,
            Qt.QMessageBox.Cancel)
        if confirm != Qt.QMessageBox.Ok:

    def new_pipeline(self):
        if not self.confirm_erase_pipeline():
            return
        w = Qt.QDialog(self)
        w.setModal(True)
        w.setWindowTitle('Pipeline name')
        l = Qt.QVBoxLayout()
        w.setLayout(l)
        le = Qt.QLineEdit()
        l.addWidget(le)
        l2 = Qt.QHBoxLayout()
        l.addLayout(l2)
        l2.addWidget(ok)
        l2.addWidget(cancel)
        ok.clicked.connect(w.accept)
        cancel.clicked.connect(w.reject)
        le.returnPressed.connect(w.accept)

        res = w.exec_()
        if res:
            class_kwargs = {
                '__module__': '__main__',
                'do_autoexport_nodes_parameters': False,
                'node_position': {},
                'node_dimension': {}
            }
            name = le.text()
            if type(name) is not str: # unicode ?
                name = name.encode()
            pipeline_class = type(name, (Pipeline,), class_kwargs)
            pipeline = pipeline_class()
            self.set_pipeline(pipeline)
            self._pipeline_filename = ''

    def load_pipeline(self, filename='', load_pipeline=True):
        class LoadProcessUi(Qt.QDialog):
            def __init__(self, parent=None, old_filename=''):
                super(LoadProcessUi, self).__init__(parent)
                self.old_filename = old_filename
                lay = Qt.QVBoxLayout()
                self.setLayout(lay)
                l2 = Qt.QHBoxLayout()
                lay.addLayout(l2)
                l2.addWidget(Qt.QLabel('Pipeline:'))
                self.proc_edit = PipelineDeveloperView.ProcessNameEdit()
                l2.addWidget(self.proc_edit)
                self.loadbt = Qt.QPushButton('...')
                l2.addWidget(self.loadbt)
                l3 = Qt.QHBoxLayout()
                lay.addLayout(l3)
                ok = Qt.QPushButton('OK')
                l3.addWidget(ok)
                cancel = Qt.QPushButton('Cancel')
                l3.addWidget(cancel)
                self.loadbt.clicked.connect(self.get_filename)
                self.proc_edit.returnPressed.connect(self.accept)

            def get_filename(self):
                filename = qt_backend.getOpenFileName(
                    None, 'Load the pipeline', self.old_filename,
                    'Compatible files (*.xml *.json *.py);; All (*)')
                if filename:
                    self.proc_edit.setText(filename)

            return

        if not filename:
            old_filename = getattr(self, '_pipeline_filename', '')
            dialog = LoadProcessUi(self, old_filename=old_filename)
            dialog.setWindowTitle('Load pipeline')
            dialog.setModal(True)
            dialog.resize(800, dialog.sizeHint().height())
            res = dialog.exec_()

            if res:
                filename = dialog.proc_edit.text()

        if filename:
            if not load_pipeline:
                return filename
            else:
                try:
                    if self.scene.pipeline:
                        # keep the same engine
                        engine = self.scene.pipeline.get_study_config().engine
                        pipeline = engine.get_process_instance(filename)
                    else:
                        pipeline = get_process_instance(filename)
                    print(e)
                    pipeline = None
                if pipeline is not None:
                    self.set_pipeline(pipeline)
                    self._pipeline_filename = filename
                    return filename

    def save_pipeline(self):
        '''
        Ask for a filename using the file dialog, and save the pipeline as a
        XML, JSON or python file.
        '''
        old_filename = getattr(self, '_pipeline_filename', '')
            None, 'Save the pipeline', old_filename,
            'Compatible files (*.xml *.json *.py);; All (*)')
            posdict = {}
            for key, value in six.iteritems(self.scene.pos):
                if hasattr(value, 'x'):
                    posdict[key] = (value.x(), value.y())
                else:
                    posdict[key] = (value[0], value[1])
            dimdict = {}
            for key, value in six.iteritems(self.scene.dim):
                if hasattr(value, 'boundingRect'):
                    dimdict[key] = (value.boundingRect().width(),
                                    value.boundingRect().height())
                else:
                    dimdict[key] = (value[0], value[1])

            pipeline.node_dimension = dimdict
            old_pos = pipeline.node_position
            old_dim = pipeline.node_dimension
            pipeline.node_position = posdict
            pipeline_tools.save_pipeline(pipeline, filename)
            self._pipeline_filename = six.text_type(filename)
            pipeline.node_position = old_pos
            pipeline.node_dimension = old_dim

    #def load_pipeline_parameters(self):
        #"""
        #Loading and setting pipeline parameters (inputs and outputs) from a Json file.
        #"""
        #pipeline = self.scene.pipeline
        #filename = qt_backend.getOpenFileName(
            #None, 'Load pipeline parameters', '',
            #'Compatible files (*.json)')

        #pipeline_tools.load_pipeline_parameters(filename, pipeline)

    #def save_pipeline_parameters(self):
        #"""
        #Saving pipeline parameters (inputs and outputs) to a Json file.
        #"""
        #filename = qt_backend.getSaveFileName(
            #None, 'Save pipeline parameters', '',

        #pipeline_tools.save_pipeline_parameters(filename, pipeline)

    def load_pipeline_parameters(self, root_path=''):
        """
        Loading and setting pipeline parameters (inputs and outputs) from a Json file.
        :return:
        """

        def hinted_tuple_hook(obj):
            
            if '__tuple__' in obj:
                return tuple(obj['items'])
            
            else:
                return obj

                                              None, 'Load the pipeline parameters', root_path,
                                              'Compatible files (*.json)')

            with io.open(filename, 'r', encoding='utf8') as fileJson:
                dic = json.load(fileJson)

            dic = json.loads(dic, object_hook=hinted_tuple_hook)

            if "pipeline_parameters" not in list(dic.keys()):
                raise KeyError('No "pipeline_parameters" key found in {0}.'.format(filename))

            for trait_name, trait_value in dic["pipeline_parameters"].items():
                
                if trait_name not in list(self.scene.pipeline.user_traits().keys()):
                    print('No "{0}" parameter in pipeline.'.format(trait_name))
                    
                try:
                    setattr(self.scene.pipeline, trait_name, trait_value)
                    
                    print("Error for the plug {0}".format(trait_name))

            self.scene.pipeline.update_nodes_and_plugs_activation()

    def save_pipeline_parameters(self):
        """
        Saving pipeline parameters (inputs and outputs) to a Json file.
        :return:
        """

        class MultiDimensionalArrayEncoder(json.JSONEncoder):

            def encode(self, obj):

                def hint_tuples(item):

                    if isinstance(item, tuple):
                        return {'__tuple__': True,
                                'items': [hint_tuples(e) for e in item]}

                    if isinstance(item, list):
                        return [hint_tuples(e) for e in item]

                    if isinstance(item, dict):
                        return dict((key, hint_tuples(value)) for key, value in
                                    item.items())

                    else:
                        return item

                return super(MultiDimensionalArrayEncoder, self).encode(
                    hint_tuples(obj))

            None, 'Save the pipeline parameters', '',
            'Compatible files (*.json)')

        if not filename:  # save widget was cancelled by the user
            return ''

        if os.path.splitext(filename)[1] == '':  # which means no extension
            filename += '.json'

        elif os.path.splitext(filename)[1] != '.json':
            msg = QMessageBox()
            msg.setIcon(QMessageBox.Warning)
            msg.setText('The parameters must be saved in the ".json" format, '
                        'not the "{0}" format'.format(
                os.path.splitext(filename)[1]))
            msg.setWindowTitle("Warning")
            msg.setStandardButtons(QMessageBox.Ok)
            msg.buttonClicked.connect(msg.close)
            msg.exec_()
            self.save_pipeline_parameters()
            return ''

        if os.path.exists(filename) and self.disable_overwrite:
            msg.setText('This file already exists, you do not have the '
                        'rights to overwrite it.')
            return ''

            from traits.api import Undefined
            # Generating the dictionary
            param_dic = {}

            for trait_name, trait in pipeline.user_traits().items():

                if trait_name in ["nodes_activation"]:
                    continue

                value = getattr(pipeline, trait_name)

                    value = ""

                param_dic[trait_name] = value

            # In the future, more information may be added to this dictionary
            dic = {}
            dic["pipeline_parameters"] = param_dic
            jsonstring = MultiDimensionalArrayEncoder().encode(dic)

            # Saving the dictionary in the Json file
            if sys.version_info[0] >= 3:
                with open(filename, 'w', encoding='utf8') as file:
                    json.dump(jsonstring, file)
            else:
                with open(filename, 'w') as file:

'''
GUI for pipeline file inputs/outputs checking

Classes
=======
:class:`PipelineFileWarningWidget`
----------------------------------
'''

import six
try:

class PipelineFileWarningWidget(Qt.QSplitter):
    """
    This class is a GUI for pipeline file inputs/outputs checking.

    It allows to check if there are missing inputs which may prevent the
    pipeline from running, or if there are already existing outputs which would
    be overwritten if the pipeline runs.

    It will show warnings accordingly.

    The widget is built from the output of
    :py:func:`capsul.pipeline_tools.nodes_with_missing_inputs` and
    :py:func:`capsul.pipeline_tools.nodes_with_existing_outputs`
    """

    def __init__(self, missing_inputs, overwritten_outputs, parent=None):
        super(PipelineFileWarningWidget, self).__init__(
            QtCore.Qt.Vertical, parent)
        """
        Builds the check widget.

        missing_inputs: mapping iterable (mandatory)
            a dict node: (list of pairs (param_name, file_name))
            as output of
            :py:func:`capsul.pipeline_tools.nodes_with_missing_inputs`
        overwritten_outputs: mapping iterable (mandatory)
            :py:func:`capsul.pipeline_tools.nodes_with_existing_outputs`
        parent: QWidget (optional)
        """
        splitter = self
        widget1 = QtGui.QWidget(splitter)
        layout1 = QtGui.QVBoxLayout(widget1)
        widget2 = QtGui.QWidget(splitter)
        layout2 = QtGui.QVBoxLayout(widget2)
        label = QtGui.QLabel()
        layout1.addWidget(label)

        text = '<h1>Pipeline file parameters problems</h1>\n'

        if len(missing_inputs) == 0:
            text += '<h2>Inputs: OK</h2>\n' \
                '<p>All input file are present.</p>\n'
            label.setText(text)
        else:
            text += '<h2>Inputs: missing files</h2>\n'

            table = QtGui.QTableWidget()
            layout1.addWidget(table)
            table.setColumnCount(3)
            sizes = [len(l) for node, l in six.iteritems(missing_inputs)]
            table.setRowCount(sum(sizes))
            table.setHorizontalHeaderLabels(
                ['node', 'parameter', 'filename'])
            row = 0
            for node_name, items in six.iteritems(missing_inputs):
                for param_name, file_name in items:
                    if not file_name or file_name is traits.Undefined:
                        file_name = '<temp. file>'
                    table.setItem(row, 0, QtGui.QTableWidgetItem(node_name))
                    table.setItem(row, 1,
                                  QtGui.QTableWidgetItem(param_name))
                    table.setItem(row, 2, QtGui.QTableWidgetItem(file_name))
                    row += 1
            table.setSortingEnabled(True)
            table.resizeColumnsToContents()

        label_out = QtGui.QLabel()
        layout2.addWidget(label_out)
        if len(overwritten_outputs) == 0:
            text = '<h2>Outputs: OK</h2>\n' \
                '<p>No output file will be overwritten.</p>\n'
            label_out.setText(text)
        else:
            text = '<h2>Outputs: overwritten files</h2>\n'

            layout2.addWidget(table)
            sizes = [len(l) for node, l in six.iteritems(overwritten_outputs)]
            row = 0
            for node_name, items in six.iteritems(overwritten_outputs):
                    row += 1

'''
A widget to visualize a pipeline as a simple workflow

Classes
=======
:class:`PipelineUserView`
'''

import sys
from soma.subprocess import check_call

class PipelineUserView(QtGui.QWidget):
    """ A widget to visualize a pipeline as a simple workflow.

    Uses Graphviz `dot` tool.
    """
    def __init__(self, pipeline):
        """ Initialize the WorkflowViewer class
        """
        super(PipelineUserView, self).__init__()

        # Class attributets

        # Initialize the widget
        layout = QtGui.QVBoxLayout(self)
        self.label = QtGui.QLabel()
        layout.addWidget(self.label)
        #self.setLayout(layout)

        self.update()

    def update(self):
        image = tempfile.NamedTemporaryFile(suffix=".png")
        dot = tempfile.NamedTemporaryFile(suffix=".png")
        self.write(dot)
        dot.flush()
        check_call(["dot", "-Tpng", "-o", image.name, dot.name])
        self.pixmap = QtGui.QPixmap(image.name)  # .scaledToHeight(600)
        self.label.setPixmap(self.pixmap)

    def write(self, out=sys.stdout):
        graph = self.pipeline.workflow_graph()
        out.write('digraph workflow {\n'.encode())
        ids = {}
        for n in graph._nodes:
            id = str(len(ids))
            ids[n] = id
            out.write(('  %s [label="%s"];\n' % (id, n)).encode())
        for n, v in graph._links:
            out.write(('  %s -> %s;\n' % (ids[n], ids[v])).encode())
        out.write('}\n'.encode())

'''
Result viewer

Classes
=======
:class:`ViewerWidget`
'''

import os

import six

# Qt import
try:
    _fromUtf8 = QtCore.QString.fromUtf8
except AttributeError:
    _fromUtf8 = lambda s: s

class ViewerWidget(QtGui.QWidget):
    """ View result class
    """

    def __init__(self, viewer_node_name, pipeline, study_config):
        """ Method to initialize a ViewerWidget class.

        viewer_node_name: str
            the name of the node containing the viewer process
        pipeline: str
            the full pipeline in order to get the viewer input trait values
            since the viewer node is unactivated
        """
        super(ViewerWidget, self).__init__()

        # Default parameters
        self.viewer_node_name = viewer_node_name

        # Build control
        button = QtGui.QToolButton(self)
        button.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding)
        button.setMinimumHeight(50)
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap(_fromUtf8(":/icones/view_result")),
                       QtGui.QIcon.Normal, QtGui.QIcon.Off)
        button.setIcon(icon)
        button.clicked.connect(self.onCreateViewerClicked)

    def onCreateViewerClicked(self):
        """ Event to create the viewer
        """
        # Get the viewer node and process
        viewer_node = self.pipeline.nodes[self.viewer_node_name]
        viewer_process = viewer_node.process

        # Propagate the parameters to the input viewer node
        # And check if the viewer is active (ie dependencies
        # are specified -> corresponding process have run)
        is_viewer_active = True
        for plug_name, plug in six.iteritems(viewer_node.plugs):

            if plug_name in ["nodes_activation", "selection_changed"]:
                continue

            # Since it is a viewer node we normally have only inputs
            for (source_node_name, source_plug_name, source_node,
                 source_plug, weak_link) in plug.links_from:

                # Get the source plug value and source trait
                source_plug_value = getattr(source_node.process,
                                            source_plug_name)
                source_trait = source_node.process.trait(source_plug_name)

                # Check if the viewer is active:
                # 1) the source_plug_value has been set
                if source_plug_value == source_trait.handler.default_value:
                    is_viewer_active = False
                    break
                # 2) if the plug is a file, the file exists
                str_description = trait_ids(source_trait)
                if (len(str_description) == 1 and
                   str_description[0] == "File" and
                   not os.path.isfile(source_plug_value)):

                    break

                # Update destination trait
                setattr(viewer_process, plug_name, source_plug_value)

            # Just stop the iterations if the status of the viewer
            # is alreadu known
            if not is_viewer_active:
                break

        # Execute the viewer process using the defined study configuration
        if is_viewer_active:
            soma.subprocess.Popen(viewer_process.get_commandline())
            # self.study_config.run(viewer_process)
        else:
            logging.error("The viewer is not active yet, maybe "
                          "because the processings steps have not run or are "
                          "not finished.")

# CAPSUL - CAPS - Copyright (C) CEA, 2013

import os
import sys
import six
from docutils.core import publish_parts

class LayoutHelperWriter(object):
    """ A basic class to create sphinx layout and associated index.
    """
    def __init__(self, module_names, root_module_name, rst_extension=".rst"):
        """ Initialize the LayoutHelperWriter class

        module_names: list of str (mandatory)
            list of modules defined in the project
        root_module_name: str (mandatory)
            the name of the python package
        rst_extension : string (optional)
            Extension for reST files, default '.rst'
        """
        self.module_names = module_names
        self.rst_extension = rst_extension
        self.root_module_name = root_module_name
        self.rst_section_levels = ['*', '=', '-', '~', '^']

    def generate_index_entry(self, module_name, indent=4):
        """ Make autodoc documentation of pilots

        module_name: string
            the name of the module we want to index
        ident: int
            the number of blank prefix

        Returns
        -------
        ad : string
            the reST formatted index description.
        """
        # Try to get the module description
        full_module_name = "{0}.{1}".format(self.root_module_name, module_name)
        try:
            __import__(full_module_name)
        except ImportError:
            exc_info = sys.exc_info()
            logging.error("".join(traceback.format_exception(*exc_info)))
            logging.error(
                "Can't load module {0}".format(full_module_name))
        module = sys.modules[full_module_name]
        description = module.__doc__

        # Then reST formatting
        spacer = " " * 4
        ad = spacer + "<div class='span6 box'>\n"
        ad += spacer + "<h2><a href='{0}/index.html'>\n".format(module_name)
        ad += spacer + "{0} module\n".format(module_name)
        ad += spacer + "</a></h2>\n"
        ad += spacer + "<blockquote>\n"
        if description is not None:
            ad += spacer + "{0}\n".format(("\n" + spacer).join(
                self.rst2html(description).splitlines()))
        ad += spacer + "</blockquote>\n"
        ad += spacer + "</div>\n"

        return ad

    def title_for(self, title):
        """ Create a title from a underscore-separated string.

            the string to format.

        Returns
        -------
        out: str
            the formatted string.
        """
        return title.replace("_", " ").capitalize()

    def rst2html(self, rst):
        """ Convert a rst formatted string to an html string.

        rst: str (mandatory)
            the rst formatted string.

        Returns
        -------
        out: str
            the html formatted string.
        """
        parts = publish_parts(rst, writer_name="html")
        return parts["body_pre_docinfo"] + parts["body"]

    def write_layout(self, outdir):
        """Generate the sphinx layout.

        outdir : string
            Directory name in which to store files
        """
        # Check if output dir exists
        if not os.path.exists(outdir):
            os.makedirs(outdir)

        # Get the layout template
        layout_file = os.path.join(
            os.path.dirname(__file__), "resources", "layout.html")

        # Top selection panel
        indices = [
            """<li><a href="{{{{ pathto('generated/{0}/index') }}}}">"""
            """{1}</a></li>""".format(x, self.title_for(x))
            for x in self.module_names]

        # Project description
        try:
            __import__(self.root_module_name)
                "Can't load module {0}".format(self.root_module_name))
        module = sys.modules[self.root_module_name]
        release_info = {}
        exec(compile(open(os.path.join(module.__path__[0], "info.py"), "rb").read(), os.path.join(module.__path__[0], "info.py"), 'exec'), release_info)

        # Carousel items
        carousel_items_path = os.path.join(
            module.__path__[0], os.pardir, "doc", "source", "_static",
            "carousel")
        carousel_items = [item for item in os.listdir(carousel_items_path)]
        images = []
        indicators = []
        for cnt, item in enumerate(carousel_items):
            if cnt == 0:
                indicators.append(
                    "<li data-target='#examples_carousel' data-slide-to='0' "
                    "class='active'></li>")
                images.append(
                    "<div class=\"active item\">"
                    "<a href=\"{{pathto('index')}}\">"
                    "<img src=\"{{ pathto('_static/carousel/%s', 1) }}\">"
                    "</div></a>" % item)
            else:
                    "<li data-target='#examples_carousel' data-slide-to='{0}' "
                    "</li>".format(cnt))
                    "<div class=\"item\"><a href=\"{{pathto('index')}}\">"
                    "</a></div>" % item)

        # Create correspondence mapping
        layout_info = {
            "NAME_LOWER": self.root_module_name,
            "NAME_UPPER": self.root_module_name.upper(),
            "INDEX": "\n".join(indices),
            "CAROUSEL_INDICATORS": "\n".join(indicators),
            "CAROUSEL_IMAGES": "\n".join(images),
            "DESCRIPTION": self.rst2html(release_info["LONG_DESCRIPTION"]),
            "LOGO": self.root_module_name,
        }

        # Get full output filename path
        path = os.path.join(outdir, "layout.html")

        # Start writing the index
        idx = open(path, "wt")
        w = idx.write

        # Edit the template
        with open(layout_file) as open_file:
            s = "".join(open_file.readlines())
            for key, value in six.iteritems(layout_info):
                s = s.replace("%({0})s".format(key), value)
            w(s)

        # Close the open file
        idx.close()

    def write_installation(self, outdir):
        """Generate the installation recommendations.

        """

            os.path.dirname(__file__), "resources", "installation.rst")

        # Generate title
        title = "Installing `{0}`".format(self.root_module_name.upper())
        title = [self.rst_section_levels[1] * len(title), title,
                 self.rst_section_levels[1] * len(title)]

            "TITLE": "\n".join(title),
        }

        path = os.path.join(outdir, "installation.rst")

            w(s)

    def write_index(self, outdir, froot="index", rst_extension=".rst"):
        """ Make a reST API index file from python modules

        outdir : string (mandatory)
            Directory to which to write generated index file
        froot : string (optional)
            root (filename without extension) of filename to write to
            Defaults to 'index'.  We add ``rst_extension``.
        """
        # Check output directory

        path = os.path.join(outdir, froot + rst_extension)

        # Header
        w(".. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n")
        w(".. raw:: html\n\n")
        w("    <div class='container-index'>\n\n")
        title = "Documentation of the {0} Pipelines\n".format(
            self.root_module_name.upper())
        w(title)
        w(self.rst_section_levels[1] * len(title) + "\n\n")

        # Modules
        w("    <!-- Block section -->\n\n")
        for cnt, module_name in enumerate(self.module_names):
            if cnt % 2 == 0:
                w("    <div class='row-fluid'>\n\n")
            w(self.generate_index_entry(module_name))
        w("\n    </div>\n")
        w("\n    </div>")

import os
import sys

def load_pilots(root, path, root_module_name):
    """ Load all the pilot functions.

    Path is recursively scanned for ``__init__.py`` files.
    Any function declared inside whose name start with ``pilot_`` will be
    loaded.

    root: str (mandatory)
        path to the use_cases module.
    path: str
        path to the module.
    root_module_name: str (mandatory)
        the name of the package.

    Returns
    -------
    pilots: dict
        a dict with module name as keys referencing to function module
        used for unitest.
    """
    # List the module path
    items = os.listdir(path)

    # Got through mdole items
    pilots = {}
    for item in items:
        # If a directory is found, try to load the potential module
        if os.path.isdir(os.path.join(path, item)):
            sub_pilots = load_pilots(
                root, os.path.join(path, item), root_module_name)
            pilots.update(sub_pilots)

    # Check if we are in a valid python module
    if not any([x in items for x in ["__init__.py"]]):
        return pilots

    # Go through all python files
    for fname in items:
        # Check if the file is a python file
        if fname.endswith(".py"):
            if fname == '__main__.py':
                # skip main
                continue
            # Construct the module name
            module_name = (
                [root_module_name] +
                path[len(os.path.normpath(root)) + 1:].split(os.path.sep) +
                [os.path.splitext(fname)[0]])
            module_name = ".".join([x for x in module_name if x])

            # Try to load the module from its string description
            try:
                __import__(module_name)
                module = sys.modules[module_name]

                # Try to find the pilots
                for function in dir(module):
                    if function.startswith("pilot_"):
                        pilots.setdefault(module_name, []).append(
                            getattr(module, function))

            # An api exists, but it cannot be imported
                logging.debug(
                    "Could not import {0}: {1}".format(module_name, e))
                raise

    return pilots

import os
import six

# Define logger
logger = logging.getLogger(__file__)

class PipelineHelpWriter(object):
    """ Class for automatic generation of pipeline API documentations
    in Sphinx-parsable reST format.
    """

    # Only separating first two levels
    rst_section_levels = ['*', '=', '-', '~', '^']

    def __init__(self, pipelines, rst_extension=".rst", short_names={}):
        """ Initialize package for parsing

        pipelines : list (mandatory)
            list of pipeline class
            extension for reST files, default '.rst'
        short_names : dict (optional)
            translation dict for module/pipeline file names
        """
        self.pipelines = sorted(pipelines)
        self.short_names = short_names

    def generate_api_doc(self, pipeline, schema):
        """ Make autodoc documentation for a pipeline python module

        pipeline : string
            python location of pipeline - e.g 'caps.fmri.PIPELINE'
        schema : string
            path to the pipeline representation image

        Returns
        -------
        ad : string
            contents of API doc
        title : string
            the fist line of the docstring
        """
        # Fiest get the pipeline instance from its string description
        pipeline_instance = get_process_instance(pipeline)

        # Get the header, ie. the first line of the docstring
        # Default title is ''
        header = pipeline_instance.__doc__
        title = ""
        if header:
            title = pipeline_instance.__doc__.splitlines()[0]

        # Add header to tell us that this documentation must not be edited
        ad = ".. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n"

        # Generate the page title: name of the pipeline
        ad += ":orphan:\n\n"

        # Set the current module
        currentmodule = ".".join(pipeline_instance.id.split(".")[:-1])
        ad += ".. currentmodule:: {0}\n\n".format(currentmodule)

        # Generate a bookmark (for cross references)
        pipeline_name = pipeline_instance.__class__.__name__
        label = pipeline + ":"
        ad += "\n.. _{0}\n\n".format(label)

        chap_title = pipeline
        ad += (chap_title + "\n" +
               self.rst_section_levels[1] * len(chap_title) + "\n\n")

        # Add a subtitle
        ad += (pipeline_name + "\n" +
               self.rst_section_levels[2] * len(pipeline_name) + "\n\n")

        # Then add the trait description
        # It will generate two sections: input and output
        ad += pipeline_instance.get_help(returnhelp=True, use_labels=True)

        # Add schema if generated
        if schema:
            schama_title = "Pipeline schema"
            ad += ("\n" + schama_title + "\n" +
                   "~" * len(schama_title) + "\n\n")
            ad += ".. image:: {0}\n".format(schema)
            ad += "    :height: 400px\n"
            ad += "    :align: center\n\n"

        return ad, title

    def write_api_docs(self, outdir=None, returnrst=False):
        """ Generate API reST files.

        outdir : string (optional, default None)
            directory name in which to store files.
            Automatic filenames are created for each module.
        returnrst: bool (optional, default False)
            if True return the rst string documentation,
            otherwise write it to disk.

        Notes
        -----
        Sets self.written_modules to list of written modules
        """
        if returnrst is False:
            if not isinstance(outdir, six.string_types):
                raise Exception("If 'returnrst' is False, need a valid output "
                                "directory.")
            if not os.path.exists(outdir):
                os.makedirs(outdir)
        else:
            rstdoc = {}

        # Generate reST API
        written_modules = []
        for pipeline in self.pipelines:

            # Information message
            logger.info("Processing pipeline '{0}'...".format(pipeline))

            pipeline_short = self.get_short_name(pipeline)
            # Check if an image representation of the pipeline exists
            if returnrst is False:
                schema = os.path.join(os.pardir, "schema",
                                      pipeline_short + ".png")
                if not os.path.isfile(os.path.join(outdir, schema)):
                    schema = None
            else:
                schema = None

            # Generate the rst string description
            api_str, title_str = self.generate_api_doc(pipeline, schema)
            if not api_str:
                continue

            # Write to file
                outfile = os.path.join(outdir,
                                       pipeline_short + self.rst_extension)
                fileobj = open(outfile, "wt")
                fileobj.write(api_str)
                fileobj.close()
            else:
                rstdoc[pipeline] = api_str

            # Update the list of written modules
            written_modules.append((title_str, pipeline))

        # Update the class attribute containing the list of written modules
        self.written_modules = written_modules

        if returnrst is True:
            return rstdoc

    def get_short_name(self, name):
        """
        Get a short file name prefix for module/process in the
        short_names dict. Used to build "reasonably short" path/file names.
        """
        short_name = self.short_names.get(name)
        if short_name:
            return short_name
        # look for a shorter name for the longest module prefix
        modules = name.split(".")
        for i in range(len(modules)-1, 0, -1):
            path = '.'.join(modules[:i])
            short_path = self.short_names.get(path)
            if short_path:
                return '.'.join([short_path] + modules[i+1:])
        # not found
        return name

    def write_index(self, outdir, froot="index", relative_to=None,
                    rst_extension=".rst"):
        """ Make a reST API index file from the list of written files

            directory to which to write generated index file
        relative_to : string
            path to which written filenames are relative.  This
            component of the written file path will be removed from
            outdir, in the generated index.  Default is None, meaning,
            leave path as it is.
        """
        # Check if some modules have been written
        if self.written_modules is None:
            raise ValueError('No modules written')

        # Get full index filename path

        # Path written into index is relative to rootpath
        if relative_to is not None:
            relpath = outdir.replace(relative_to + os.path.sep, "")
        else:
            relpath = outdir
        print('relpath:', relpath)

        # Information message
        logger.info("Writing index at location '{0}'...".format(
            os.path.abspath(path)))

        # Edit the index file

        # Generate a table with all the generated modules
        # module_name (link) + first docstring line

        # Table definition
        table = ["<!-- Block section -->"]
        table.append("<table border='1' class='docutils' style='width:100%'>")
        table.append("<colgroup><col width='25%'/><col width='75%'/>"
                     "</colgroup>")
        table.append("<tbody valign='top'>")

        # Add all modules
        for title_str, f in self.written_modules:
            pipeline_short = self.get_short_name(f)
            print('title_str:', title_str, ', f:', f)
            relative_pipeline = ".".join(f.split(".")[2:])
            print('relative_pipeline:', relative_pipeline)
            ref = os.path.join(relpath, pipeline_short + ".html")
            print('ref:', ref)
            table.append("<tr class='row-odd'>")
            table.append(
                "<td><a class='reference internal' href='{0}'>"
                "<em>{1}</em></a></td>\n".format(ref, relative_pipeline))
            table.append("<td>{0}</td>".format(title_str))
            table.append("</tr>")

        # Close divs
        table.append("</tbody>\n\n")
        table.append("</table>")

        # Format the table
        table_with_indent = [" " * 4 + line for line in table]
        w("\n".join(table_with_indent))

        # Close the file

    def write_main_index(self, outdir, module_name, root_module_name,
                         froot="index", rst_extension=".rst",
                         have_usecases=True):
        """ Make a reST API index file for the module

        module_name: str (mandatory)
            The name of module from which we want to generate an index.
            The python package name
        """

        logger.info("Writing module '{0}' index at location '{1}'...".format(
            module_name, os.path.abspath(path)))

        # Open the result index file

        # Stat writing

        w(":orphan:\n\n")

        # Generate a title
        chap_title = " ".join([x.capitalize() for x in module_name.split("_")])
        w(chap_title + "\n" +
          self.rst_section_levels[0] * len(chap_title) + "\n\n")

        # Generate a markup
        label = module_name
        w(".. _{0}:\n\n".format(label))

        # Page use cases
        # # Generate a title
        chap_title = ":mod:`{0}.{1}`: User Guide".format(
            root_module_name, module_name)
          self.rst_section_levels[1] * len(chap_title) + "\n\n")

        if have_usecases:
            # # Generate a markup
            label = module_name + "_ug"
            w(".. _{0}:\n\n".format(label))
            # # Some text description
            w("Some live examples containing snippets of codes.\n\n")
            # # Include user guide index
            w(".. include:: use_cases/index%s\n\n" % rst_extension)

        # API page
        chap_title = ":mod:`{0}.{1}`: API".format(
        # # Generate a markup
        label = module_name + "_api"
        # # Some text description
        w("The API of functions and classes, as given by the "
          "docstrings.")
            w(" For the *user guide* see the {0}_ug_ "
              "section for further details.\n\n".format(module_name))
        else:
            w("\n\n")
        # # Include pipeline and buildingblock indexes
        # ## Pipeline
        chap_title = "Pipelines"
          self.rst_section_levels[2] * len(chap_title) + "\n\n")
        w(".. include:: pipeline/index%s\n\n" % rst_extension)
        # ## Buildingblocks
        chap_title = "Buildingblocks"
        w(".. include:: process/index%s\n\n" % rst_extension)

        # Close file

import ast
import os
import six

class UseCasesHelperWriter(object):
    """ A basic class to convert the pilot codes to rst use cases
    """
    def __init__(self, pilots, rst_extension=".rst"):
        """ Initialize the UseCasesHelper class

        pilots: list of @function (mandatory)
            list of pilot functions.
            Extension for reST files, default '.rst'.
        """
        self.pilots = pilots

    def getsource(self, function):
        """ Method that returns the source code of a function

        function: @function (mandatory)
            a python function.

        Returns
        -------
        srccode: str
            the function source code.
        """
        return inspect.getsource(function)

    def generate_usecases_doc(self, src_code, module_name):

        src_code : string
            pilot source code.

        Returns
        -------
        ad : string
            the use case reST formatted documentation.
        """
        # First parse the pilot function code
        code_tree = ast.parse(src_code).body
        pilot_tree = code_tree[0].body

        lines = src_code.splitlines()
        nb_lines = len(lines)
        ad += ".. _example_{0} :\n\n".format(module_name)

        line_start_code = 0
        line_end_code = 0
        is_header = True
        full_code = "# The full use case code: {0}\n".format(module_name)
        for code_item in pilot_tree:
            if (isinstance(code_item, ast.Expr) and
                    isinstance(code_item.value, ast.Str)):
                # Find End code line
                code_value = lines[line_start_code:line_end_code]
                for line_index in range(line_end_code, nb_lines):
                    clean_line = lines[line_index].lstrip()
                    if clean_line[:3] in ["'''", '"""']:
                        break
                    else:
                        code_value.append(lines[line_index])

                # Add code value to reST documentation
                if line_start_code != 0:
                    full_code += "\n".join(code_value)
                    ad += "::\n\n"
                    ad += "\n".join(code_value)
                    ad += "\n\n"

                # Add expand box with full code in header
                if is_header:
                    code = ".. hidden-code-block:: python\n"
                    code += "    :starthidden: True\n\n"
                    code += "    $FULL_CODE"

                # Add comments to reST
                comment = inspect.cleandoc(code_item.value.s)
                # Insert full code after main title
                    comment = comment.splitlines()
                    comment.insert(3, code)
                    comment.insert(3, "")
                    ad += "\n".join(comment)
                    is_header = False
                else:
                    ad += comment
                ad += "\n\n"
                line_start_code = code_item.lineno

            else:
                line_end_code = code_item.lineno + 1

        # Add the end of the file
        if line_start_code != nb_lines:
            full_code += "\n".join(lines[line_start_code:])
            ad += "::\n\n"
            ad += "\n".join(lines[line_start_code:])
            ad += "\n\n"

        # Insert full use case code
        ad = ad.replace("$FULL_CODE", full_code)

        return ad

    def write_usecases_docs(self, outdir=None, returnrst=False):
        """ Generate Use Cases reST files.

            Directory name in which to store files.
            We create automatic filenames for each Use case.
        """
        else:
            rstdoc = {}

        # Write reST use cases documentation
        written_usecases = []
        for pilot in self.pilots:

            logger.info("Processing pilot '{0}' in module '{1}'...".format(
                pilot.__name__, pilot.__module__))

            # Generate reST
            uid = pilot.__module__ + "." + pilot.__name__
            code_str = self.getsource(pilot)
            use_case_str = self.generate_usecases_doc(code_str, uid)
            title_str = code_str.splitlines()[3].strip()
            if not use_case_str:
                continue

                outfile = os.path.join(outdir, uid + self.rst_extension)
                fileobj.write(use_case_str)
            else:
                rstdoc[uid] = use_case_str

            # Update the list of written use cases
            written_usecases.append((title_str, uid))

        # Update the class attribute containing the list of written use cases
        self.written_usecases = written_usecases

        """ Make a reST API index file from written files

        """
        if self.written_usecases is None:
        # Get full filename path
        else:

        for title_str, f in self.written_usecases:

            # generate the relative pipeline name
            relative_uid = ".".join(f.split(".")[2:])

            ref = os.path.join(relpath, f + ".html")
                "<em>{1}</em></a></td>\n".format(ref, relative_uid))

'''
Utility functions for configuration

Functions
=========
:func:`environment`
'''

import os
import re

def environment(sh_file=None, env={}):
    """ Function that return a dictionary containing the environment
    needed by a program (for instance FSL or FreeSurfer).

    In the configuration file, the variable are expected to be defined
    as 'VARIABLE_NAME=value'.

    sh_file: str (mandatory)
        the path to the sh script used to set up the environment.
    env: dict (optional, default empty)
        the default environment used to parse the configuration sh file.

    Returns
    -------
    environment: dict
        a dict containing the program configuration.
    """
    # Use sh commands and a string instead of a list since
    # we're using shell=True
    # Pass empty environment to get only the program variables
    command = ["bash", "-c", ". '{0}' ; /usr/bin/printenv".format(sh_file)]
    process = soma.subprocess.Popen(command, env=env,
                               stdout=soma.subprocess.PIPE, stderr=soma.subprocess.PIPE)
    stdout, stderr = process.communicate()
    if process.returncode != 0:
        raise Exception(
            "Could not parse sh_file: {0}. Maybe you should check if all "
            "the dependencies are installed".format(stderr))

    # Parse the output : each line should be of the form
    # 'VARIABLE_NAME=value'
    environment = {}
    for line in stdout.decode().split(os.linesep):
        if line.startswith("export"):
            line = line.replace("export ", "")
            line = line.replace("'", "")
        match = re.match(r"^(\w+)=(\S*)$", line)
        if match:
            name, value = match.groups()
            if name != "PWD":
                environment[name] = str(value)

    # Debug message
    logger.debug("Parsed FSL environment: {0}.".format(environment))

    return environment

'''
Memory caching. Probably mostly obsolete, this code is not much used now.

Classes
=======
:class:`UnMemorizedProcess`
:class:`MemorizedProcess`
:class:`CapsulResultEncoder`
:class:`Memory`

Functions
=========
:func:`get_process_signature`
:func:`has_attribute`
:func:`file_fingerprint`
'''

from __future__ import with_statement
import os
import hashlib
import time
import six
import sys

from capsul.process.process import Process, ProcessResult

# TRAITS import

###########################################################################
# Proxy process objects

class UnMemorizedProcess(object):
    """ This class replaces MemorizedProcess when there is no cache.
    It provides an identical API but does not write anything on disk.
    """
    def __init__(self, process, verbose=1):
        """ Initialize the UnMemorizedProcess class.

        process: capsul process
            the process instance to wrap.
        verbose: int
            if different from zero, print console messages.
        """
        self.verbose = verbose

        """ Call the process.

        .. note::
            matlab process input image headers are saved since matlab
            tools may modify image headers.

            should correspond to the declared process parameters.
        """
        # Set the process inputs early to get some argument checking
        for name, value in six.iteritems(kwargs):
            self.process.set_parameter(name, value)
        input_parameters = self._get_process_arguments()

        if self.verbose != 0:
            print("{0}\n[Process] Calling {1}...\n{2}".format(
                80 * "_", self.process.id,
                get_process_signature(self.process, input_parameters)))

        # Start a timer
        start_time = time.time()

        result = self.process()
        duration = time.time() - start_time

            msg = "{0:.1f}s, {1:.1f}min".format(duration, duration / 60.)
            print(max(0, (80 - len(msg))) * '_' + msg)

        Returns
        -------
            the process input parameters.
        """

        for name, trait in six.iteritems(self.process.user_traits()):

            # Get the trait value
            value = self.process.get_parameter(name)

            # Split input and output traits
            is_input = True
            if "output" in trait.__dict__ and trait.output:
                is_input = False

            # Skip undefined trait attributes and outputs
            if is_input and value is not Undefined:

                # Store the input parameter
                input_parameters[name] = value

        return input_parameters

        """ Define behavior for when a user attempts to access an attribute
        of a MemorizedProcess instance.

        First check the MemorizedProcess object and then the Process object.

        name: string
            the name of the parameter we want to access.
        """
        if name in self.__dict__:
            super(UnMemorizedProcess, self).__getattr__(name)
        elif name in self.process.user_traits():
            return self.process.get_parameter(name)
        elif name in dir(self.process):
            return getattr(self.process, name)
        else:
            raise AttributeError(
                "'UnMemorizedProcess' and 'Process' objects have no attribute "
                "'{0}'".format(name))

        """ Define behavior for when a user attempts to set an attribute

            the name of the parameter we want to set.
            the parameter value.
        """
        if ("process" in self.__dict__ and
                name in self.__dict__["process"].__dict__):
            super(self.__dict__["process_class"],
                  self.__dict__["process"]).__setattr__(name, value)
        else:
            super(UnMemorizedProcess, self).__setattr__(name, value)

    def __repr__(self):
        """ ProcessFunc class representation.
        """
        return "{0}({1})".format(self.__class__.__name__, self.process.id)

class MemorizedProcess(object):
    """ Callable object decorating a capsul process for caching its return
    values each time it is called.

    All values are cached on the filesystem, in a deep directory
    structure. Methods are provided to inspect the cache or clean it.
    """

    def __init__(self, process, cachedir, timestamp=None, verbose=1):
        """ Initialize the MemorizedProcess class.

        cachedir: string
            the directory in which the computation will be stored
        timestamp: float (optional)
            The reference time from which times in tracing messages
            are reported.
        callback: callable (optional)
            an optional callable called each time after the function
            is called.
        """
        # Check the a process is passed
        self.process_class = process.__class__
        if not (isinstance(self.process_class, object) and
                issubclass(self.process_class, Process)):
                "The 'process' argument should be a capsul process class, "
                "but '{0}' (type '{1}') was passed.".format(
                    self.process_class, type(self.process_class)))

        # Check the memory directory
        cachedir = os.path.abspath(cachedir)
        if not os.path.exists(cachedir) and os.path.isdir(cachedir):
            raise ValueError("'base_dir' should be an existing directory.")
        self.cachedir = cachedir

        # Define the cache time
        if timestamp is None:
            timestamp = time.time()
        self.timestamp = timestamp

        # Set the documentation of the class
        self.__doc__ = self.process.get_help(returnhelp=True)

        # Store if some messages have to be displayed

        """ Call wrapped process and cache result, or read cache if

        This function returns the wrapped function output and some metadata.

        .. note::

        """

        # Create the destination folder and a unique id for the current
        # process
        process_dir, process_hash, input_parameters = self._get_process_id()

        if not os.path.isdir(process_dir):

            # Create the destination memory folder
            os.makedirs(process_dir)

            # Try to execute the process and if an error occurred remove the
            # cache folder
            try:
                # Run
                result = self._call_process(process_dir, input_parameters)

                # Save the result files in the memory with the corresponding
                # mapping
                output_parameters = {}
                for name, trait in self.process.traits(output=True).items():
                    # Get the trait value
                    value = self.process.get_parameter(name)
                    output_parameters[name] = value
                file_mapping = []
                self._copy_files_to_memory(output_parameters, process_dir,
                                           file_mapping)
                map_fname = os.path.join(process_dir, "file_mapping.json")
                with open(map_fname, "w") as open_file:
                    open_file.write(json.dumps(file_mapping))

            except Exception as e:  # noqa: E722
                print('error in MemorizedProcess.__call__:', e)
                shutil.rmtree(process_dir)
                raise

        # Restore the process results from the cache folder
        else:
            # Restore the memorized files
            map_fname = os.path.join(process_dir, "file_mapping.json")
            with open(map_fname, "r") as json_data:
                file_mapping = json.load(json_data)

            # Go through all mapping files
            for workspace_file, memory_file in file_mapping:

                # Determine if the workspace directory is writeable
                if os.access(os.path.dirname(workspace_file), os.W_OK):
                    shutil.copy2(memory_file, workspace_file)
                else:
                    logger.debug("Can't restore file '{0}', access rights are "
                                 "not sufficients.".format(workspace_file))

            # Update the process output traits
            result = self._load_process_result(process_dir, input_parameters)

    def _copy_files_to_memory(self, python_object, process_dir, file_mapping):
        """ Copy file items inside the memory.

        process_dir: str
            the process memory path.
        file_mapping: list of 2-uplet
            store in this structure the mapping between the workspace and the
            memory (workspace_file, memory_file).
        """
                    self._copy_files_to_memory(val, process_dir, file_mapping)

        # Otherwise start the copy if the object is a file
        else:
                out = os.path.join(process_dir, fname)
                shutil.copy2(python_object, out)
                file_mapping.append((python_object, out))

    def _call_process(self, process_dir, input_parameters):
        """ Call a process.

        process_dir: string
            the directory where the cache has been written.
            the process input_parameters.

        Returns
        -------
        result: dict
            the process results.
        """
            print("{0}\n[Memory] Calling {1}...\n{2}".format(

        caching = getattr(study_config, 'use_smart_caching', None)
        # avoid recusrion
        study_config.use_smart_caching = False

        study_config.use_smart_caching = caching

        # Save the result in json format
        cache = {'parameters': dict((i, getattr(self.process, i)) 
                                    for i in self.process.user_traits()),
                 'result': result}
        json_data = json.dumps(cache, sort_keys=True,
                               check_circular=True, indent=4,
                               cls=CapsulResultEncoder)
        result_fname = os.path.join(process_dir, "result.json")
        with open(result_fname, "w") as open_file:
            open_file.write(json_data)

    def _load_process_result(self, process_dir, input_parameters):
        """ Load the result of a process.

        Returns
        -------
        result: ProcessResult
            the process cached results.
        """
        # Display an information message
            print("[Memory]: Loading {0}...".format(

        # Load the process result
        if not os.path.isfile(result_fname):
            raise KeyError(
                "Non-existing cache value (may have been cleared).\n"
                "File {0} does not exist.".format(result_fname))
        with open(result_fname, "r") as json_data:
            result_dict = json.load(json_data, cls=CapsulResultDecoder)

        ## Update the process output traits
        for name, value in six.iteritems(result_dict['parameters']):

        return result_dict['result']

    def _get_process_id(self, **kwargs):
        """ Return the directory in which are persisted the result of the
        process called with the given arguments.

        Returns
        -------
            the directory where the cache should be write.
        process_hash: string
            the process md5 hash.
        """
        # Get the process id
        process_hash, input_parameters = self._get_argument_hash()
        process_dir = os.path.join(self._get_process_dir(), process_hash)

        return process_dir, process_hash, input_parameters

    def _get_argument_hash(self):
        """ Get a hash of the process arguments.

        Some parameters are not considered during the hash computation:
            * if the parameter value is not defined
            * if the corresponding trait has an attribute 'nohash'

        Add the tool versions to check roughly if the running codes have
        changed.

        Returns
        -------
        """

                # Check specific flags before hash
                if has_attribute(trait, "nohash", attribute_value=True,
                                 recursive=True):
                    continue

        # Add the tool versions to check roughly if the running codes have
        # changed and add file path fingerprints
        process_parameters = input_parameters.copy()
        process_parameters = self._add_fingerprints(process_parameters)
        process_parameters["versions"] = self.process.versions

        # Generate the process hash
        hasher = hashlib.new("md5")
        hasher.update(json.dumps(process_parameters, sort_keys=True).encode())
        process_hash = hasher.hexdigest()

        return process_hash, input_parameters

    def _add_fingerprints(self, python_object):
        """ Add file path fingerprints.

        Returns
        -------
            the input object with fingerprint-file representation.
        """
        out = {}
            for key, val in six.iteritems(python_object):
                    out[key] = self._add_fingerprints(val)

            out = []
                    out.append(self._add_fingerprints(val))

        else:
                out = file_fingerprint(python_object)

        return out

    def _get_process_dir(self):
        """ Get the directory corresponding to the cache for the current
        process.

        Returns
        -------
        """
        # Build the memory path from the process id
        path = [self.cachedir]
        path.extend(self.process.id.split("."))
        process_dir = os.path.join(*path)

        # Guarantee the path exists on the disk
        if not os.path.exists(process_dir):

        return process_dir

        """
        return "{0}({1}, base_dir={2})".format(
            self.__class__.__name__, self.process.id, self.cachedir)

        """
            super(MemorizedProcess, self).__getattr__(name)
        else:
                "'MemorizedProcess' and 'Process' objects have no attribute "

        """
        else:
            super(MemorizedProcess, self).__setattr__(name, value)

def get_process_signature(process, input_parameters):
    """ Generate the process signature.

    process: Process
        a capsul process object
    input_parameters: dict
        the process input_parameters.

    Returns
    -------
    signature: string
        the process signature.
    """
    kwargs = ["{0}={1}".format(name, value)
              for name, value in six.iteritems(input_parameters)]
    return "{0}({1})".format(process.id, ", ".join(kwargs))

def has_attribute(trait, attribute_name, attribute_value=None,
                  recursive=True):
    """ Checks if a given trait has an attribute and optionally if it
    is set to particular value.

    trait: Trait
        the input trait object.
    attribute_name: string
        the trait attribute name that will be checked.
    attribute_value: object (optional)
        the trait attribute axpected value.
    recursive: bool (optional, default True)
        check for the attribute in the inner traits.

    Returns
    -------
    res: bool
        True if input given trait has an attribute and optionally if it
        is set to a particular value.
    """
    # Count the number of trait having the target attribute
    count = 0

    # Check the current trait
    if (attribute_name in trait.__dict__ and
        (trait.__dict__[attribute_name] == attribute_value or
         attribute_value is None)):
            count += 1

    # Check the inner traits
    if recursive:
        if len(trait.inner_traits) > 0:
            for inner_trait in trait.inner_traits:
                count += has_attribute(inner_trait, attribute_name,
                                       attribute_value, recursive)

    return count > 0

def file_fingerprint(a_file):
    """ Computes the file fingerprint.

    Do not consider the file content, just the fingerprint (ie. the mtime,
    the size and the file location).

    a_file: string
        the file to process.

    Returns
    -------
    fingerprint: tuple
        the file location, mtime and size.
    """
    fingerprint = {
        "name": a_file,
        "mtime": None,
        "size": None
    }
    if os.path.isfile(a_file):
        stat = os.stat(a_file)
        fingerprint["size"] = str(stat.st_size)
        fingerprint["mtime"] = str(stat.st_mtime)
    return fingerprint

class CapsulResultEncoder(json.JSONEncoder):
    """ Deal with ProcessResult in json.
    """
    def default(self, obj):
        try:
            import numpy
            # numpy is not here
            numpy = None

        # File special case
        if isinstance(obj, ProcessResult):
            result_dict = {}
            for name in ["runtime", "returncode", "inputs",
                         "outputs"]:
                result_dict[name] = tuple_json_encoder(getattr(obj, name))
            return result_dict

        # Undefined parameter special case
        if isinstance(obj, Undefined.__class__):
            return "<undefined_trait_value>"

        # InterfaceResult special case
        # avoid explicitly loading nipype: it takes much time...
        nipype = sys.modules.get('nipype.interfaces.base')
        if nipype:
            InterfaceResult = getattr(nipype, 'InterfaceResult')
        else:
            class InterfaceResult(object):
                pass

        if isinstance(obj, InterfaceResult):
            return "<skip_nipype_interface_result>"

        # Array special case
        if numpy is not None and isinstance(obj, numpy.ndarray):
            return obj.tolist()

        # Call the base class default method
        return json.JSONEncoder.default(self, obj)

def tuple_json_encoder(obj):
    """ Encode a tuple in order to save it in json format.

    obj: object
        a python object to encode.

    Returns
    -------
    encobj: object
        the encoded object.
    """
    if isinstance(obj, tuple):
        return {
            "__tuple__": True,
            "items": [tuple_json_encoder(item) for item in obj]
        }
    elif isinstance(obj, list):
        return [tuple_json_encoder(item) for item in obj]
    elif isinstance(obj, dict):
        return dict((tuple_json_encoder(key), tuple_json_encoder(value))
                    for key, value in six.iteritems(obj))
    else:
        return obj

class CapsulResultDecoder(json.JSONDecoder):
    """
    def __init__(self, *args, **kargs):
        json.JSONDecoder.__init__(self, object_hook=self.object_object, *args,
                                  **kargs)

    def object_object(self, obj):
        # Tuple special case
        if "__tuple__" in obj:
            return tuple(obj["items"])
        elif obj == "<undefined_trait_value>":
            return Undefined
        elif isinstance(obj, dict):
            for key, value in six.iteritems(obj):
                if value == "<undefined_trait_value>":
                    obj[key] = Undefined
            return obj
        # Default
        else:
            return obj

############################################################################
# Memory manager: provide some tracking about what is computed when, to
# be able to flush the disk

class Memory(object):
    """ Memory context to provide caching for processes.

    `cachedir`: string
        the location for the caching. If None is given, no caching is done.

    Methods
    -------
    cache
    clear
    """

    def __init__(self, cachedir):
        """ Initialize the Memory class.

        base_dir: string
            the directory name of the location for the caching.
        """
        # Build the capsul memory folder
        if cachedir is not None:
            cachedir = os.path.join(
                os.path.abspath(cachedir), "capsul_memory")
            if not os.path.exists(cachedir):
                os.makedirs(cachedir)
            elif not os.path.isdir(cachedir):
                raise ValueError("'base_dir' should be a directory")

        # Define class parameters
        self.timestamp = time.time()

    def cache(self, process, verbose=1):
        """ Create a proxy of the given process in order to only execute
        the process for input parameters not cached on disk.

            the capsul Process to be wrapped and cached.

        Returns
        -------
        proxy_process: MemorizedProcess object
            the returned object is a MemorizedProcess object, that behaves
            as a process object, but offers extra methods for cache lookup
            and management.

        Examples
        --------
        Create a temporary memory folder

        >>> from tempfile import mkdtemp
        >>> mem = Memory(mkdtemp())

        Here we create a callable that can be used to apply an
        fsl.Merge interface to files

        >>> from capsul.process import get_process_instance
        >>> nipype_fsl_merge = get_process_instance(
        ...    "nipype.interfaces.fsl.Merge")
        >>> fsl_merge = mem.cache(nipype_fsl_merge)

        Now we apply it to a list of files. We need to specify the
        list of input files and the dimension along which the files
        should be merged.

        >>> results = fsl_merge(in_files=['a.nii', 'b.nii'], dimension='t')

        We can retrieve the resulting file from the outputs:

        >>> results.outputs._merged_file
        """
        # If a proxy process is found get the encapsulated process
        if (isinstance(process, MemorizedProcess) or
                isinstance(process, UnMemorizedProcess)):

        # If the cachedir is None no caching is done
        if self.cachedir is None:
            return UnMemorizedProcess(process, verbose)
        # Otherwise a proxy process is created
        else:
            return MemorizedProcess(process, self.cachedir, self.timestamp,
                                    verbose)

    def clear(self, skips=None):
        """ Remove all the cache apart from those given to the method
        input.

        skips: list
            a list of path to keep during the cache deletion.
        """
        # Get all memory directories to remove
        to_remove_folders = []
        skips = skips or []
        for root, dirs, files in os.walk(self.cachedir):
            if "result.json" and files and dirs == [] and root not in skips:
                to_remove_folders.append(root)

        # Delete memory directories
        for folder in to_remove_folders:
            shutil.rmtree(folder)

        """ Memory class representation.
        """
        return "{0}(cachedir={1})".format(self.__class__.__name__,
                                          self.cachedir)

'''
Process instance factory

Functions
=========
:func:`is_process`
:func:`is_pipeline_node`
:func:`get_process_instance`
:func:`get_node_class`
:func:`get_node_instance`
'''

import sys
import re
import six
import os

# Caspul import
from capsul.process.nipype_process import nipype_factory
from capsul.process.xml import create_xml_process
from capsul.pipeline.xml import create_xml_pipeline
from capsul.pipeline.json_io import create_json_pipeline

process_xml_re = re.compile(r'<process.*</process>', re.DOTALL)

_interface = None
_nipype_loaded = False

def _get_interface_class():
    '''
    returns the nypype Interface type, or a custom type if it cannot be
    imported.

    We use this function on demand because importing nipype is long (it
    sometimes takes several seconds) and it's not always needed.

    We don't really import nipype, but use sys.modules to get it instead,
    because here we only use Interface to check if a given object is an
    instance of Interface.
    '''
    global _interface, _nipype_loaded
    if _interface is not None and _nipype_loaded:
        return _interface
    if not _nipype_loaded:
        # Nipype import
        if nipype is None:
            _interface = type("Interface", (object, ), {})
        else:
            _interface = getattr(nipype, 'Interface')
            _nipype_loaded = True
    return _interface

def is_process(item):
    """ Check if the input item is a process class or function with decorator
    or XML docstring which makes it seen as a process
    """
    Interface = _get_interface_class()
    if inspect.isclass(item) and item not in (Pipeline, Process) \
            and (issubclass(item, Process) or issubclass(item, Interface)):
    if not inspect.isfunction(item):
    if hasattr(item, 'capsul_xml'):
    if item.__doc__:
        match = process_xml_re.search(item.__doc__)
        if match:
    return False

def is_pipeline_node(item):
    """ Check if the input is an instance of pipeline Node
    """
    return item is not Node and isinstance(item, Node)

def get_process_instance(process_or_id, study_config=None, **kwargs):
    """ Return a Process instance given an identifier.

    Note that it is convenient to create a process from a StudyConfig instance:
    StudyConfig.get_process_instance()

    The identifier is either:

        * a derived Process class.
        * a derived Process class instance.
        * a Nipype Interface instance.
        * a Nipype Interface class.
        * a string description of the class `<module>.<class>`.
        * a string description of a function to warp `<module>.<function>`.
        * a string description of a module containing a single process
          `<module>`
        * a string description of a pipeline `<module>.<fname>.xml`.
        * an XML filename for a pipeline.
        * a JSON filename for a pipeline.
        * a Python (.py) filename with process name in it:
          `/path/process_source.py#ProcessName`.
        * a Python (.py) filename for a file containing a single process.

    Default values of the process instance are passed as additional parameters.

    .. note:

        If no process is found an ImportError is raised.

    .. note:

        If the 'process_or_id' parameter is not valid a ValueError is raised.

    .. note:

        If the function to warp does not contain a process description in its
        docstring ('<process>...</process>') a ValueError is raised.

    process_or_id: instance or class description (mandatory)
        a process/nipype interface instance/class or a string description.
    study_config: StudyConfig instance (optional)
        A Process instance belongs to a StudyConfig framework. If not specified
        the study_config can be set afterwards.
    kwargs:
        default values of the process instance parameters.

    Returns
    -------
    result: Process
        an initialized process instance.
    """
    # NOTE
    # here we make a bidouille to make study_config accessible from processes
    # constructors. It is used for instance in ProcessIteration.
    # This is not elegant, not thread-safe, and forbids to have one pipeline
    # build a second one in a different study_config context.
    # I don't have a better solution, however.
    import capsul.study_config.study_config as study_cmod
    set_study_config = (study_config is not None
                        and study_cmod._default_study_config
                            is not study_config)

    try:
        if set_study_config:
            old_default_study_config = study_cmod._default_study_config
            study_cmod._default_study_config = study_config
        return _get_process_instance(process_or_id, study_config=study_config,
                                     **kwargs)
    finally:
            study_cmod._default_study_config = old_default_study_config

def _execfile(filename):
    # This chunk of code cannot be put inline in python 2.6
    glob_dict = {}
    exec(compile(open(filename, "rb").read(), filename, 'exec'),
          glob_dict, glob_dict)
    return glob_dict

def _load_module(filename, modname=None):
    if not modname:
        modname = os.path.basename(filename).rsplit('.', 2)[0]
    spec = importlib.util.spec_from_file_location(modname, filename)
    mod = importlib.util.module_from_spec(spec)
    sys.modules[modname] = mod
    spec.loader.exec_module(mod)
    return mod
    if mod is not None:
        sys.modules[modname] = mod
    return mod

def _get_process_instance(process_or_id, study_config=None, **kwargs):

    def _find_single_process(module_dict, filename):
        ''' Scan objects in module_dict and find out if a single one of them is
        a process
        '''
        object_name = None
        for name, item in six.iteritems(module_dict):
            if is_process(item):
                if object_name is not None:
                    raise KeyError(
                        'file %s contains several processes. Please '
                        'specify which one should be used using '
                        'filename.py#ProcessName or '
                        'module.submodule.ProcessName' % filename)
                object_name = name
        return object_name

    result = None
    # If the function 'process_or_id' parameter is already a Process
    # instance.
    if isinstance(process_or_id, Process):
        result = process_or_id

    # If the function 'process_or_id' parameter is a Process class.
    elif (isinstance(process_or_id, type) and
          issubclass(process_or_id, Process)):
        result = process_or_id()

    # If the function 'process_or_id' parameter is already a Nipye
    # interface instance, wrap this structure in a Process class
    elif isinstance(process_or_id, Interface):
        result = nipype_factory(process_or_id)

    # If the function 'process_or_id' parameter is an Interface class.
          issubclass(process_or_id, Interface)):
        result = nipype_factory(process_or_id())

    # If the function 'process_or_id' parameter is a function.
    elif isinstance(process_or_id, types.FunctionType):
        xml = getattr(process_or_id, 'capsul_xml', None)
        if xml is None:
            # Check docstring
            if process_or_id.__doc__:
                match = process_xml_re.search(
                    process_or_id.__doc__)
                if match:
                    xml = match.group(0)
        if xml:
            result = create_xml_process(process_or_id.__module__,
                                        process_or_id.__name__, 
                                        process_or_id, xml)()
        else:
            raise ValueError('Cannot find XML description to make function {0} a process'.format(process_or_id))
        
    # If the function 'process_or_id' parameter is a class string
    # description
    elif isinstance(process_or_id, six.string_types) \
            and not process_or_id.startswith('<pipeline'):
        py_url = os.path.basename(process_or_id).split('#')
        as_xml = False
        as_py = False
        module_dict = None
        module = None
        if len(py_url) >= 2 and py_url[-2].endswith('.py') \
                or len(py_url) == 1 and py_url[0].endswith('.py'):
            # python file + process name: something.py#ProcessName
            # or just something.py if it contains only one process class
            if len(py_url) >= 2:
                filename = process_or_id[:-len(py_url[-1]) - 1]
                object_name = py_url[-1]
            else:
                filename = process_or_id
                object_name = None
            module = _load_module(filename)
            module_name = module.__name__
            module_dict = module.__dict__
            if object_name is None:
                object_name = _find_single_process(
                    module_dict, module_name)
                    module_name = process_or_id
                    as_py = True
            elif object_name in module_dict:
                as_py = True
        if object_name is None:
            elements = process_or_id.rsplit('.', 1)
            if len(elements) < 2:
                module_name, object_name = '__main__', elements[0]
            else:
                module_name, object_name = elements
            try:
                module = importlib.import_module(module_name)
                # update the Interface class since we may have loaded nipype
                # during import
                Interface = _get_interface_class()

                if object_name not in module.__dict__ \
                        or not is_process(getattr(module, object_name)):
                    # maybe a module with a single process in it
                    module = importlib.import_module(process_or_id)
                    module_dict = module.__dict__
                    # update the Interface class since we may have loaded
                    # nipype during import
                    Interface = _get_interface_class()
                    object_name = _find_single_process(
                        module_dict, module_name)
                    if object_name is not None:
                        module_name = process_or_id
                        as_py = True
                else:
                pass
        if not as_py:
            # maybe XML or JSON filename or URL
            for ext in ('.xml', '.json'):
                xml_url = process_or_id + ext
                if osp.exists(xml_url):
                    object_name = None
                    as_xml = True
                elif process_or_id.endswith(ext) and osp.exists(process_or_id):
                    xml_url = process_or_id
                else:
                    # maybe XML or JSON file with pipeline name in it
                    xml_url = module_name + ext
                    if not osp.exists(xml_url) and module_name.endswith(ext) \
                            and osp.exists(module_name):
                        xml_url = module_name
                    if not osp.exists(xml_url):
                        # try XML file in a module directory + class name
                        basename = None
                        module_name2 = None
                        if module_name in sys.modules:
                            basename = object_name
                            module_name2 = module_name
                            object_name = None # to allow unmatching class / xml
                            if basename.endswith(ext):
                                basename = basename[:-4]
                        else:
                            elements = module_name.rsplit('.', 1)
                            if len(elements) == 2:
                                module_name2, basename = elements
                        if module_name2 and basename:
                            try:
                                importlib.import_module(module_name2)
                                mod_dirname = osp.dirname(
                                    sys.modules[module_name2].__file__)
                                xml_url = osp.join(mod_dirname, basename + ext)
                                if not osp.exists(xml_url):
                                    # if basename includes .xml extension
                                    xml_url = osp.join(mod_dirname, basename)
                                as_xml = True
                            except ImportError as e:
                                raise ImportError('Cannot import %s: %s'
                                                  % (module_name, str(e)))
                if as_xml:
                    break

            if osp.exists(xml_url):
                loaders = {'.xml': create_xml_pipeline,
                           '.json': create_json_pipeline}
                result = loaders[ext](module_name, object_name, xml_url)()

        if result is None and not as_xml:
            if module_dict is not None:
                module_object = module_dict.get(object_name)
            else:
                module_object = getattr(module, object_name, None)
            if module_object is not None:
                if (isinstance(module_object, type) and
                    issubclass(module_object, Process)):
                    result = module_object()
                elif isinstance(module_object, Interface):
                    # If we have a Nipype interface, wrap this structure in a
                    # Process class
                    result = nipype_factory(result)
                elif (isinstance(module_object, type) and
                    issubclass(module_object, Interface)):
                    result = nipype_factory(module_object())
                elif isinstance(module_object, types.FunctionType):
                    xml = getattr(module_object, 'capsul_xml', None)
                    if xml is None:
                        # Check docstring
                        if module_object.__doc__:
                            match = process_xml_re.search(
                                module_object.__doc__)
                            if match:
                                xml = match.group(0)
                    if xml:
                        result = create_xml_process(module_name, object_name,
                                                    module_object, xml)()
            if result is None and module is not None:
                xml_file = osp.join(osp.dirname(module.__file__),
                                    object_name + '.xml')
                if osp.exists(xml_file):
                    result = create_xml_pipeline(module_name, None,
                                                 xml_file)()
    elif hasattr(process_or_id, 'read') \
            or (isinstance(process_or_id, bytes)
                and process_or_id.startswith(b'<pipeline')) \
            or (isinstance(process_or_id, str)
                and process_or_id.startswith('<pipeline')):
        # file-like object or xml string
        module_name = __name__
        # can be either XML or python
        #try:
        result = create_xml_pipeline(module_name, None, process_or_id)()
        #except Exception as e:
            #result = create_xml_pipeline(module_name, None, process_or_id)()

    if result is None:
        raise ValueError("Invalid process_or_id argument. "
                         "Got '{0}' and expect a Process instance/string "
                         "description or an Interface instance/string "
                         "description".format(process_or_id))

    # Set the instance default parameters
    for name, value in six.iteritems(kwargs):
        result.set_parameter(name, value)

    if study_config is not None:
        if result.study_config is not None \
                and result.study_config is not study_config:
            raise ValueError("StudyConfig mismatch in get_process_instance "
                             "for process %s" % result)
        result.set_study_config(study_config)
    return result

def get_node_class(node_type):
    """
    Get a custom node class from module + class string.
    The class name is optional if the module contains only one node class.
    It is OK to pass a Node subclass or a Node instance also.
    """
    if inspect.isclass(node_type):
        if issubclass(node_type, Node):
            return node_type.__name__, node_type  # already a Node class
        return Node
    if isinstance(node_type, Node):
        return node_type.__class__.__name__, node_type.__class__
    cls = None
    try:
        mod = importlib.import_module(node_type)
        for name, val in six.iteritems(mod.__dict__):
            if inspect.isclass(val) and val.__name__ != 'Node' \
                    and issubclass(val, Node):
                cls = val
                break
        else:
        name = node_type.split('.')[-1]
        modname = node_type[:-len(name) - 1]
        mod = importlib.import_module(modname)
        cls = getattr(mod, name)
    if cls is None:
    return name, cls

def get_node_instance(node_type, pipeline, conf_dict=None, name=None,
                      **kwargs):
    """
    Get a custom node instance from a module + class name (see
    :func:`get_node_class`) and a configuration dict or Controller.
    The configuration contains parameters needed to instantiate the node type.
    Each node class may specify its parameters via its class method
    `configure_node`.

    node_type: str or Node subclass or Node instance
        node type to be built. Either a class (Node subclass) or a Node
        instance (the node will be re-instantiated), or a string
        describing a module and class.
        pipeline in which the node will be inserted.
    conf_dict: dict or Controller
        configuration dict or Controller defining parameters needed to build
        the node. The controller should be obtained using the node class's
        `configure_node()` static method, then filled with the desired values.
        If not given the node is supposed to be built with no parameters, which
        will not work for every node type.
    kwargs:
        default values of the node instance parameters.
    """
    cls_and_name = get_node_class(node_type)
    if cls_and_name is None:
        raise ValueError("Could not find node class %s" % node_type)
    nname, cls = cls_and_name
    if not name:
        name = nname

    if isinstance(conf_dict, Controller):
        conf_controller = conf_dict
    elif conf_dict is not None:
        if hasattr(cls, 'configure_controller'):
            if conf_controller is None:
                raise ValueError("node type %s has a configuration controller "
                                 "problem (see %s.configure_controller()"
                                 % (node_type, node_type))
            conf_controller.import_from_dict(conf_dict)
        else:
            conf_controller = Controller()
    else:
        else:
    if hasattr(cls, 'build_node'):
        node = cls.build_node(pipeline, name, conf_controller)
    else:
        # probably bound to fail...
        node = cls(pipeline, name, [], [])

        setattr(node, name, value)

    return node

'''
Process and pipeline execution management

Functions
=========
:func:`run_process`
'''

import errno
import os
import six

from capsul.study_config.memory import Memory

# TRAIT import
from traits.api import Undefined, File, Directory

def run_process(output_dir, process_instance,
                generate_logging=False, verbose=0, configuration_dict=None,
                cachedir=None,
                **kwargs):
    """ Execute a capsul process in a specific directory.

    output_dir: str (mandatory)
        the folder where the process will write results.
    process_instance: Process (mandatory)
        the capsul process we want to execute.
    cachedir: str (optional, default None)
        save in the cache the current process execution.
        If None, no caching is done.
    generate_logging: bool (optional, default False)
        if True save the log stored in the process after its execution.
    verbose: int
        if different from zero, print console messages.
    configuration_dict: dict (optional)
        configuration dictionary

    Returns
    -------
    returncode: ProcessResult
        contains all execution information.
    output_log_file: str
        the path to the process execution log file.
    """
    # Message
    logger.info("Study Config: executing process '{0}'...".format(
        process_instance.id))

    study_config = process_instance.get_study_config()

    if configuration_dict is None:
        configuration_dict \
            = process_instance.check_requirements('global')

    # create directories for outputs
    if study_config.create_output_directories:
        for name, trait in process_instance.user_traits().items():
            if trait.output and isinstance(trait.handler, (File, Directory)):
                value = getattr(process_instance, name)
                if value is not Undefined and value:
                    base = os.path.dirname(value)
                    if base and not os.path.exists(base):
                        try:
                            os.makedirs(base)
                        except OSError as err:
                            if err.errno != errno.EEXIST:
                                raise
                            # We have a race condition?
                            pass

        configuration_dict = {}
    # clear activations for now.
    from capsul import engine
    engine.activated_modules = set()
    #print('activate config:', configuration_dict)
    engine.activate_configuration(configuration_dict)

    # Run
    if study_config.get_trait_value("use_smart_caching") in [None, False]:
        cachedir = None
    elif cachedir is None:
        cachedir = output_dir

    # Update the output directory folder if necessary
    if output_dir not in (None, Undefined) and output_dir:
        if study_config.process_output_directory:
            output_dir = os.path.join(output_dir, '%s-%s' % (study_config.process_counter, process_instance.name))
        # Guarantee that the output directory exists
        if not os.path.isdir(output_dir):
            os.makedirs(output_dir)
            if 'output_directory' in process_instance.user_traits():
                if (process_instance.output_directory is Undefined or
                        not(process_instance.output_directory)):
                    process_instance.output_directory = output_dir

    # Set the current directory directory if necessary
    if hasattr(process_instance, "_nipype_interface"):
        if "spm" in process_instance._nipype_interface_name:
            process_instance._nipype_interface.mlab.inputs.prescript += [
                "cd('{0}');".format(output_dir)]

    # Setup the process log file
    output_log_file = None
    if generate_logging and output_dir not in (None, Undefined):
        output_log_file = os.path.join(
            os.path.basename(output_dir),
            os.path.dirname(output_dir) + ".json")
        process_instance.log_file = output_log_file

    # Check extra parameters name
    for arg_name in kwargs:
        # If the extra parameter name does not match with a user
        # trait parameter name, raise a AttributeError
        if arg_name not in process_instance.user_traits():
                "execution of process {0} got an unexpected keyword "
                "argument '{1}'".format(process_instance, arg_name))

    # Information message
    if verbose:
        for name, trait in six.iteritems(process_instance.user_traits()):
            value = process_instance.get_parameter(name)
            if not trait.output and value is not Undefined:
        input_parameters = ["{0}={1}".format(n, v)
                            for n, v in six.iteritems(input_parameters)]
        call_with_inputs = "{0}({1})".format(process_instance.id, ", ".join(input_parameters))
        print("{0}\n[Process] Calling {1}...\n{2}".format(
            80 * "_", process_instance.id,
            call_with_inputs))
    if cachedir:
        # Create a memory object
        mem = Memory(cachedir)
        proxy_instance = mem.cache(process_instance, verbose=verbose)

        # Execute the proxy process
        returncode = proxy_instance(**kwargs)
    else:
            setattr(process_instance, k, v)
        missing = process_instance.get_missing_mandatory_parameters()
        if len(missing) != 0:
            raise ValueError('In process %s: missing mandatory parameters: %s'
                             % (process_instance.name, ', '.join(missing)))
        process_instance._before_run_process()
        returncode = process_instance._run_process()
        returncode = process_instance._after_run_process(returncode)

    # Save the process log
    if generate_logging:
        process_instance.save_log(returncode)

    # Increment the number of executed process count
    study_config.process_counter += 1

    return returncode, output_log_file

'''
Main :class:`StudyConfig` class for configuration of Capsul software, directories etc.

Classes
========
:class:`StudyConfig`
:class:`StudyConfigModule`

Functions
=========
:func:`default_study_config`
'''

import os
import sys
import six
import threading
if sys.version_info[:2] >= (2, 7):
    from collections import OrderedDict
else:
    from soma.sorted_dictionary import SortedDictionary as OrderedDict

from traits.api import File, Directory, Bool, String, Undefined, Int

from capsul.study_config.run import run_process

class StudyConfig(Controller):
    """ Class to store the study parameters and processing options.

    StudyConfig is deprecated and will probably be removed in Capsul 3.
    Please use :class:`~capsul.engine.CapsulEngine` instead and its
    construction function, :func:`~capsul.engine.capsul_engine` when possible.

    This in turn is used to evaluate a Process instance or a Pipeline.

    StudyConfig has modules (see BrainVISAConfig, AFNIConfig, FSLConfig,
    MatlabConfig, ANTSConfig, SmartCachingConfig, SomaWorkflowConfig,
    SPMConfig, FOMConfig, MRTRIXConfig).
    Modules are initialized in the constructor, so their list has to be setup
    before instantiating StudyConfig. A default modules list is used when no
    modules are specified: StudyConfig.default_modules

    StudyConfig configuration is loaded from a global file and then from a
    study specific file (based on study_name parameter). The global
    configuration file name is either in os.environ['CAPSUL_CONFIG'] or
    in "~/.config/capsul/config.json". The study specific configuration
    file name is either defined in the global configuration or in
    "~/.config/capsul/<study_name>/config.json".
    ::

      from capsul.api import StudyConfig

      study_config = StudyConfig(modules=['SPMConfig', 'FomConfig'])
      # or:
      study_config = StudyConfig(modules=StudyConfig.default_modules +
      ['FomConfig'])

    input_directory : str
        parameter to set the study input directory
    output_directory : str
        parameter to set the study output directory
    generate_logging : bool (default False)
        parameter to control the log generation
    create_output_directories : bool (default True)
        Create parent directories of all output File or Directory before
        running a process
    process_output_directory : bool (default False)
        Create a process specific output_directory by appending a
        subdirectory to output_directory. This subdirectory is named 
        '<count>-<name>' where <count> if self.process_counter and <name> 
        is the name of the process.

    Methods
    -------
    run
    reset_process_counter
    set_trait_value
    get_trait
    get_trait_value
    update_study_configuration
    set_study_configuration
    """

    default_modules = ['AFNIConfig', 'ANTSConfig', 'FSLConfig', 'MatlabConfig',
                       'MRTRIXConfig', 'SmartCachingConfig',
                       'SomaWorkflowConfig', 'SPMConfig']
    _user_config_directory = os.path.join("~", ".config", "capsul")

    study_name = String(
        None,
        desc="Name of the study to configure",
        # traits with transient=True will not be saved in configuration
        # see  http://code.enthought.com/projects/traits/docs/html/
        # traits_user_manual/advanced.html#pickling-hastraits-objects
        transient=True, groups=['study'])

    user_level = Int(
        0,
        desc="0: basic, 1: advanced, 2: expert... used to display or hide "
             "some advanced features or process parameters that would be "
             "confusing to a novice user",
        groups=['study'])

    input_directory = Directory(
        Undefined,
        desc="Parameter to set the study input directory",

    output_directory = Directory(
        desc="Parameter to set the study output directory",

    generate_logging = Bool(
        False,
        desc="Parameter to control the log generation",

    create_output_directories = Bool(
        True,
        desc="Create parent directories of all output File or Directory before running a process",

    process_output_directory = Bool(
        False,
        desc="Create a process specific output_directory by appending a "
             "subdirectory to output_directory. This subdirectory is named "
             "'<count>-<name>' where <count> if self.process_counter and "
             "<name> is the name of the process.",

    def __init__(self, study_name=None, init_config=None, modules=None,
                 engine=None, **override_config):
        """ Initialize the StudyConfig class

        study_name: Name of the study to configure. This name is used to
            identify specific configuration for a study.
        init_config: if not None, must contain a dictionary that will be used
            to configure this StudyConfig (instead of reading configuration
            from configuration files).
        modules: list of string (default self.default_modules).
            the names of configuration module classes that will be included
            in this study configuration.
        engine: CapsulEngine
            this parameter is temporary, it just helps to handle the transition
            to :class:`capsul.engine.CapsulEngine`. Don't use it in client code.
        override_config: dictionary
            The content of these keyword parameters will be set on the
            configuration after it has been initialized from configuration
            files (or from init_config).
        """

        super(StudyConfig, self).__init__()

        if study_name:
            self.study_name = study_name

        if engine is None:
            from capsul.engine import capsul_engine
            self.engine = capsul_engine()
            self.engine.study_config = weakref.proxy(self)
        else:
            self.engine = weakref.proxy(engine)

        # Read the configuration for the given study
        if init_config is None:
            config = self.read_configuration()
            config.update(override_config)
        else:
            self.global_config_file = None
            self.study_config_file = None
            if override_config:
                config = init_config.copy()
                config.update(override_config)
            else:
                config = init_config

        self.visible_groups = set(['study'])

        # Create modules
        if modules is None:
            # Make it possible for a study to define its own set of modules
            modules = config.pop('config_modules', self.default_modules)

        # 'modules_data' is a container for modules-specific internal data
        # each module is encouraged to prefix its variables there by its
        # module name
        self.modules_data = Controller()

        self.modules = {}
            self.load_module(module, config)

        # Set self attributes according to configuration values
        for k, v in six.iteritems(config):
            setattr(self, k, v)
        self.initialize_modules()
        self.run_lock = threading.RLock()
        self.run_interruption_request = False

    def initialize_modules(self):
        """
        Modules initialization, calls initialize_module on each config module.
        This is not done during module instantiation to allow interactions
        between modules (e.g. Matlab configuration can influence Nipype
        configuration). Modules dependencies are taken into account in
        initialization.
        """
        already_initialized = set()
        # Use a stack to allow to manage module dependencies
        stack = list(self.modules.keys())
        while stack:
            module_name = stack.pop(0)
            if module_name in already_initialized:
                continue
            module = self.modules.get(module_name)
            if not module:
                raise EnvironmentError('Required StudyConfig module %s is '
                                       'missing' % module_name)
            # Check if there are dependent modules that must be initialized
            # before the current one
            initialize_first = [m for m in module.dependencies
                                if m not in already_initialized]
            if initialize_first:
                stack = initialize_first + [module_name] + stack
                continue
            # Initialize a module
            module.initialize_module()
            module.initialize_callbacks()
            already_initialized.add(module_name)

        # Intern identifier

        # Parameter that is incremented at each process execution
        self.process_counter = 1

    # Methods

    def load_module(self, config_module_name, config):
        """
        Load an optional StudyConfig module.

        config_module_name: Name of the module to load (e.g. "FSLConfig").
        config: dictionary containing the configuration of the study.
        """
        if config_module_name not in self.modules:
            python_module = (
                "capsul.study_config.config_modules.{0}_config".format(
                    config_module_name[:-6].lower()))
            python_module = __import__(python_module,
                                       fromlist=[config_module_name])
            config_module_class = getattr(python_module, config_module_name)
            module = config_module_class(self, config)
            self.modules[config_module_name] = module
            # load dependencies
            for dep_module_name in module.dependencies:
                if dep_module_name not in self.modules:
                    self.load_module(dep_module_name, config)
            return module

    def run(self, process_or_pipeline, output_directory=None,
            execute_qc_nodes=True, verbose=0, configuration_dict=None,
            **kwargs):
        """Method to execute a process or a pipeline in a study configuration
         environment.

         Depending on the studies_config settings, it may be a sequential run,
         or a parallel run, which can involve remote execution (through soma-
         workflow).

         Only pipeline nodes can be filtered on the 'execute_qc_nodes'
         attribute.

         A valid output directory is expected to execute the process or the
         pepeline without soma-workflow.

        process_or_pipeline: Process or Pipeline instance (mandatory)
            the process or pipeline we want to execute
        output_directory: Directory name (optional)
            the output directory to use for process execution. This replaces
            self.output_directory but left it unchanged.
        execute_qc_nodes: bool (optional, default False)
            if True execute process nodes that are tagged as qualtity control
            process nodes.
        configuration_dict: dict (optional)
            configuration dictionary
        """

        # Use soma workflow to execute the pipeline or process in parallel
        # on the local machine. This has now moved to CapsulEngine.
        if self.get_trait_value("use_soma_workflow"):
            return self.engine.check_call(process_or_pipeline, **kwargs)

        # here we only deal with the (obsolete) local execution mode.

        needs_opengl = getattr(process_or_pipeline, 'needs_opengl', False)

        with self.run_lock:
            self.run_interruption_request = False

        # set parameters values
            setattr(process_or_pipeline, k, v)
        # output_directory cannot be in kwargs
        if output_directory not in (None, Undefined, '') \
                and 'output_directory' in process_or_pipeline.traits():
            process_or_pipeline.output_directory = output_directory

        missing = process_or_pipeline.get_missing_mandatory_parameters()
            ptype = 'process'
            if isinstance(process_or_pipeline, Pipeline):
                ptype = 'pipeline'
            raise ValueError('In %s %s: missing mandatory parameters: %s'
                             % (ptype, process_or_pipeline.name,
                                ', '.join(missing)))

        # Use the local machine to execute the pipeline or process
        if output_directory is None or output_directory is Undefined \
                or output_directory == '':
            if 'output_directory' in process_or_pipeline.traits():
                output_directory = getattr(process_or_pipeline,
                                           'output_directory')
                output_directory = self.output_directory
        # Not all processes need an output_directory defined on
        # StudyConfig
            # Check the output directory is valid
            if not isinstance(output_directory, six.string_types):
                    "'{0}' is not a valid directory. A valid output "
                    "directory is expected to run the process or "
                    "pipeline.".format(output_directory))
            try:
                if not os.path.isdir(output_directory):
                    os.makedirs(output_directory)
            except OSError:
                    "Can't create folder '{0}', please investigate.".format(
                        output_directory))

        # Temporary files can be generated for pipelines
        temporary_files = []
        result = None
        try:
            # Generate ordered execution list
            execution_list = []
                execution_list = \
                    process_or_pipeline.workflow_ordered_nodes()
                # Filter process nodes if necessary
                if not execute_qc_nodes:
                    execution_list = [node for node in execution_list
                                      if node.node_type
                                          == "processing_node"]
                for node in execution_list:
                    # check temporary outputs and allocate files
                    process_or_pipeline._check_temporary_files_for_node(
                        node, temporary_files)
            elif isinstance(process_or_pipeline, Process):
                execution_list.append(process_or_pipeline)
            else:
                raise Exception(
                    "Unknown instance type. Got {0}and expect Process or "
                    "Pipeline instances".format(
                        process_or_pipeline.__module__.name__))

            with self.run_lock:
                if self.run_interruption_request:
                    self.run_interruption_request = False
                    raise RuntimeError('Execution interruption requested')

            if not needs_opengl:
                # if any node needs opengl, we must initialize it first
                # before any node execution loads Qt Gui
                for process_node in execution_list:
                    if getattr(process_node, 'needs_opengl', False):
                        needs_opengl = True
                        break
            if needs_opengl:
                from soma.qt_gui import qt_backend

                if qt_backend.headless:
                    qt_backend.set_headless(True, True)

            # Execute each process node element
            for process_node in execution_list:
                # Execute the process instance contained in the node
                if isinstance(process_node, Node):
                    result, log_file = run_process(
                        output_directory,
                        process_node.process,
                        generate_logging=self.generate_logging,
                        verbose=verbose,
                        configuration_dict=configuration_dict)

                # Execute the process instance
                else:
                        process_node,

                with self.run_lock:
                    if self.run_interruption_request:
                        self.run_interruption_request = False
                        raise RuntimeError('Execution interruption requested')

        finally:
            # Destroy temporary files
            if temporary_files:
                # If temporary files have been created, we are sure that
                # process_or_pipeline is a pipeline with a method
                # _free_temporary_files.
                process_or_pipeline._free_temporary_files(temporary_files)

    def reset_process_counter(self):
        """ Method to reset the process counter to one.
        """

    def read_configuration(self):
        """Find the configuration for the current study (whose name is defined
        in self study_name) and returns a dictionary that is a merge between
        global options and study specific options.

        Global option are taken from environment variable CAPSUL_CONFIG if
        it is defined, otherwise from "~/.config/capsul/config.json" if it
        exists.

        The configuration for a study can be defined the global configuration
        if it contains a dictionary in it "studies_config" option and if there
        is a key corresponding to self.study_name in this dictionary. If the
        corresponding value is a string, it must be a valid json configuration
        file name (either absolute or relative to the global configuration
        file). Otherwise, the corresponding value must be a dictionary
        containing study specific configuration values.
        If no study configuration is found from global configuration, then
        a file named "~/.config/capsul/%s/config.json" (where %s is
        self.study_name) is used if it exists.
        """

        # First read global options
        global_config_file = os.environ.get("CAPSUL_CONFIG")
        if (isinstance(global_config_file, six.string_types) and
            os.path.isfile(global_config_file)):

            with open(global_config_file) as f:
                config = json.load(f)
            self.global_config_file = global_config_file
        else:
            global_config_file = \
                os.path.expanduser(os.path.join(self._user_config_directory,
                                                "config.json"))
            if os.path.isfile(global_config_file):
                with open(global_config_file) as f:
                    config = json.load(f)
                self.global_config_file = global_config_file
            else:
                config = {}
                self.global_config_file = None

        # Look for study specific configuration file
        study_config = \
            config.pop('studies_config', {}).get(self.study_name)
        if isinstance(study_config, six.string_types):
            if self.global_config_file:
                study_config = \
                    os.path.join(os.path.dirname(self.global_config_file),
                                 study_config)
            self.study_config_file = study_config
            with open(study_config) as f:
                study_config = json.load(f)
        elif study_config is None:
            study_config_file = \
                os.path.expanduser(
                    os.path.join(self._user_config_directory,
                                 "%s", "config.json") % str(self.study_name))
            if os.path.exists(study_config_file):
                with open(study_config_file) as f:
                    study_config = json.load(f)
                self.study_config_file = study_config_file
            else:
                study_config = {}
                self.study_config_file = None
        else:
                self.study_config_file = self.global_config_file

        # Merge study configuration file with global configuration
        config.update(study_config)

    def get_configuration_dict(self):
        """ Returns a json compatible dictionary containing current
        configuration.
        """
        config = self.export_to_dict(exclude_transient=True,
                                     exclude_undefined=True,
                                     exclude_none=True)

    def save_configuration(self, file):
        """ Save study configuration as json file.

        file: file or str (mandatory)
            either a writable opened file or the path to the output json file.
        """
        # Dump the study configuration elements
        config = self.get_configuration_dict()
        if isinstance(file, six.string_types):
            with open(file, "w") as f:
                json.dump(config, f, indent=4, separators=(",", ": "))
        else:
            json.dump(config, file, indent=4, separators=(",", ": "))

    def update_study_configuration(self, json_fname):
        """ Update the study configuration from a json file.

        json_fname: str (mandatory)
            the path to the output json file.
        """
        # Load the json file
        with open(json_fname, "r") as json_data:
            new_config = json.load(
                json_data, object_pairs_hook=OrderedDict)

        # Update the study configuration
        self.set_study_configuration(new_config)

    def add_trait(self, name, *trait):
        """ Add a new trait.

            the trait name.
        trait: traits.api (mandatory)
            a valid trait.
        """
        # Call the controller add_trait method
        super(StudyConfig, self).add_trait(name, *trait)

        # Get the trait instance and if it is a user trait load the traits
        # to get it in the traits accessor method that can select traits from
        # trait attributes
        trait_instance = self.trait(name)
        if self.is_user_trait(trait_instance):
            self.trait_get(name)

    def set_study_configuration(self, new_config):
        """ Method to set the new configuration of the study.

        If a study configuration element can't be updated properly,
        send an error message to the logger.

        new_config: ordered dict (mandatory)
            the structure that contain the default study configuration:
            see the class attributes to build this structure.
        """
        # Go through the configuration structure, respecting the traits
        # declaration order
        for trait_name in self.user_traits():
            try:
                trait_value = new_config[trait_name]
                # not specified in new_config
                continue
            # Try to update the 'trait_name' configuration element
            try:
                self.set_trait_value(trait_name, trait_value)
                    "Could not set value for config variable {0}: "
                    "{1}".format(trait_name, repr(trait_value)))

    def set_trait_value(self, trait_name, trait_value):
        """ Method to set the value of a parameter.

        trait_value: object (mandatory)
        """
        if trait_name in self.user_traits():
            setattr(self, trait_name, trait_value)

        """ Method to access the 'trait_name' study configuration element.

        Notes
        -----
        If the 'trait_name' element is not found, return None

            the trait name we want to access

        Returns
        -------
        trait: trait
            the trait we want to access
        """
            return self.trait(trait_name)
        else:

    def get_trait_value(self, trait_name):
        """ Method to access the value of the 'trait_name' study
        configuration element.

        .. note:

            If the 'trait_name' element is not found, return None

        Returns
        -------
        """
            return getattr(self, trait_name)
        else:

        """ Return a Process instance given an identifier.

        The identifier is either:

            * a derived Process class.
            * a derived Process class instance.
            * a Nipype Interface instance.
            * a Nipype Interface class.
            * a string description of the class `<module>.<class>`.
            * a string description of a function to warp `<module>.<function>`.
            * a string description of a pipeline `<module>.<fname>.xml`.
            * an XML filename for a pipeline

        Default values of the process instance are passed as additional
        parameters.

        .. note:

            If no process is found an ImportError is raised.

        .. note:

            If the 'process_or_id' parameter is not valid a ValueError is
            raised.

        .. note:

            If the function to warp does not contain a process description in
            its decorator or docstring ('<process>...</process>') a ValueError
            is raised.

        process_or_id: instance or class description (mandatory)
            a process/nipype interface instance/class or a string description.
        kwargs:
            default values of the process instance parameters.

        Returns
        -------
        result: Process
            an initialized process instance.

        """
        return get_process_instance(process_or_id, study_config=self,
                                    **kwargs)

    def get_iteration_pipeline(self, pipeline_name, node_name, process_or_id,
                               iterative_plugs=None, do_not_export=None,
                               make_optional=None, **kwargs):
        """ Create a pipeline with an iteration node iterating the given
        process.

        Returns
        -------
        """
        return self.engine.get_iteration_pipeline(
            pipeline_name, node_name, process_or_id,
            iterative_plugs=iterative_plugs, do_not_export=do_not_export,
            make_optional=make_optional, **kwargs)

_default_study_config = None

def default_study_config():
    """
    On the first call create a StudyConfig instance with default configuration
    (eventually reading configuration files). Then returns this instance on all
    subsequent calls.
    """
    global _default_study_config
    if _default_study_config is None:
        _default_study_config = StudyConfig()
    return _default_study_config

class StudyConfigModule(object):
    '''
    :class:`StudyConfig` module base class (abstract)
    '''
    @property
    def name(self):
        """The name of a module that can be used in configuration to select
        modules to load.
        """
        return self.__class__.__name__

    # List of modules that must be initialized before this one. It can be
    # overridden be derived module classes.
    dependencies = []

    def __init__(self, study_config, configuration):

    def initialize_module(self):
        """Method called to initialize selected study configuration modules
        on startup. This method does nothing but can be overridden by modules.
        """

    def initialize_callbacks(self):
        """Method called just after the first call to initialize_modules.
        """

    # Test the configuration time
    import timeit

    # Standard configuration
    tic = timeit.default_timer()
    study = StudyConfig()
    toc = timeit.default_timer()
    print("Standard configuration done in  {0} s.".format(toc - tic))

    # Empty configuration
    empty_config = OrderedDict([])
    study = StudyConfig(empty_config)
    print("Empty configuration done in  {0} s.".format(toc - tic))

'''

Classes
=======
:class:`AttributesConfig`
'''

from traits.api import Str, Undefined, List, Dict
from capsul.study_config.study_config import StudyConfigModule

class AttributesConfig(StudyConfigModule):
    '''Attributes-based completion configuration module for StudyConfig

    This module adds the following options (traits) in the
    :class:`~capsul.study_config.study_config.StudyConfig` object:

    attributes_schema_paths: list of str (filenames)
        attributes shchema module name
    attributes_schemas: dict(str, str)
        attributes shchemas names
    process_completion: str (default:'builtin')
        process completion model name
    path_completion: str
        path completion model name
    '''

        super(AttributesConfig, self).__init__(study_config, configuration)
        default_paths = ['capsul.attributes.completion_engine_factory']
        self.study_config.add_trait(
            'attributes_schema_paths',
            List(default_paths, Str(Undefined, output=False), output=False,
            desc='attributes shchema module name', groups=['attributes']))
            'attributes_schemas',
            Dict(str, str, output=False,
                desc='attributes shchemas names', groups=['attributes']))
            'process_completion',
            Str('builtin', output=False,
                desc='process completion model name', groups=['attributes']))
            'path_completion',
            Str(Undefined, output=False,
                desc='path completion model name', groups=['attributes'],
                optional=True))

        '''
        '''
        from capsul.engine import CapsulEngine

        if type(self.study_config.engine) is not CapsulEngine:
            # engine is a proxy, thus we are initialized from a real
            # CapsulEngine, which holds the reference values
            self.sync_from_engine()
        else:
            # otherwise engine is "owned" by StudyConfig
            if 'capsul.engine.module.attributes' \
                    not in self.study_config.engine._loaded_modules:
                self.study_config.engine.load_module(
                    'capsul.engine.module.attributes')
            self.sync_to_engine()

        self.study_config.on_trait_change(
            self.update_module,
            ['attributes_schemas', 'process_completion',
             'path_completion', 'attributes_schema_paths'])
        #  WARNING ref to self in callback
        self.study_config.engine.settings.module_notifiers.setdefault(
            'capsul.engine.module.attributes', []).append(
                self.sync_from_engine)

    def update_module(self, param=None, value=None):
        #if param == 'attributes_schema_paths':
            #factory = self.study_config.modules_data.attributes_factory
            #factory.module_path = self.study_config.attributes_schema_paths
        self.sync_to_engine(param, value)

    def sync_to_engine(self, param=None, value=None):
        with self.study_config.engine.settings as session:
            config = session.config('attributes', 'global')
            if config:
                if param is not None:
                    if value is Undefined:
                    setattr(config, param, value)
                else:
                    params = ['attributes_schemas', 'attributes_schema_paths',
                              'process_completion', 'path_completion']
                    for param in params:
                        value = getattr(self.study_config, param)
                        if value is Undefined:
                            value = None

    def sync_from_engine(self, param=None, value=None):
        if param is not None:
            if value is None:
                value = Undefined
            setattr(self.study_config, param, value)
        else:
            with self.study_config.engine.settings as session:
                config = session.config('attributes', 'global')
                if config:
                        value = getattr(config, param)
                        if value is None:
                            value = Undefined
                        setattr(self.study_config, param, value)

'''

Classes
=======
:class:`BrainVISAConfig`
'''

import os
from traits.api import Directory, Undefined
from soma import config as soma_config

class BrainVISAConfig(StudyConfigModule):
    '''
    Configuration module allowing to use `BrainVISA / Axon <http://brainvisa.info/axon/user_doc>`_ shared data in Capsul processes.

    shared_directory: str (filename)
        Study shared directory
     '''

        super(BrainVISAConfig, self).__init__(study_config, configuration)
        study_config.add_trait('shared_directory',Directory(
            Undefined,
            desc='Study shared directory', groups=['brainvisa']))

        study_config.shared_directory = soma_config.BRAINVISA_SHARE
        # the following would be good but if Axon is not present, it will
        # use an unavailable FOM and cause an error.
        #if 'FomConfig' in self.study_config.modules \
                #and self.study_config.shared_fom in (None, ''):
            #self.study_config.shared_fom = 'shared-brainvisa-1.0'

        '''
        '''

            # CapsulEngine, which should hold the reference values,
            # BUT values are actuallu defined from here...
            old_shared = self.study_config.shared_directory
                config = session.config('axon', 'global')
                if config and config.shared_directory is Undefined:
                    config.shared_directory = old_shared
        else:
            if 'capsul.engine.module.axon' \
                    'capsul.engine.module.axon')

        self.study_config.on_trait_change(self.sync_to_engine,
                                          ['shared_directory', 'user_level'])
            'capsul.engine.module.axon', []).append(self.sync_from_engine)

        if getattr(self, '_syncing', False):
            return
        self._syncing = True
        try:
                cif = self.study_config.engine.settings.config_id_field
                shared_dir = self.study_config.shared_directory
                if shared_dir is Undefined:
                    shared_dir = None
                if config is None:
                    session.new_config(
                        'axon', 'global',
                        {'shared_directory': shared_dir,
                        'user_level': self.study_config.user_level,
                          cif: 'axon'})
                else:
                    config.shared_directory = shared_dir
                    config.user_level = self.study_config.user_level
        finally:
            del self._syncing

            return
        try:
                config = session.config('axon', 'global', any=True)
                if config:
                    shared_dir = config.shared_directory
                    if shared_dir is None:
                        shared_dir = Undefined
                    self.study_config.shared_directory = shared_dir
                    self.study_config.user_level = config.user_level
        finally:

'''

Classes
=======
'''

import os
import six
from traits.api import Bool, Str, Undefined, List, Directory
from capsul.engine import CapsulEngine

class FomConfig(StudyConfigModule):
    '''FOM (File Organization Model) configuration module for StudyConfig

    .. note::
        :class:`~capsul.study_config.config_modules.fom_config.FomConfig`
        needs :class:`~capsul.study_config.config_modules.brainvisa_config.BrainVISAConfig`
        to be part of
        :class:`~capsul.study_config.study_config.StudyConfig` modules.

    input_fom: str
        input FOM
    output_fom: str
        output FOM
    shared_fom: str
        shared data FOM
    volumes_format: str
        Format used for volumes
    meshes_format: str
        Format used for meshes
    auto_fom: bool (default: True)
        Look in all FOMs when a process is not found. Note that auto_fom
        looks for the first FOM matching the process to get
        completion for, and does not handle ambiguities. Moreover
        it brings an overhead (typically 6-7 seconds) the first
        time it is used since it has to parse all available FOMs.
    fom_path: list of directories
        list of additional directories where to look for FOMs (in addition to
        the standard share/foms)
    use_fom: bool
        Use File Organization Models for file parameters completion'

    *Methods:*
    '''

    dependencies = ['BrainVISAConfig', 'SPMConfig', 'AttributesConfig']

        super(FomConfig, self).__init__(study_config, configuration)
        self.study_config.add_trait('input_fom', Str(Undefined, output=False,
            desc='input FOM', groups=['fom']))
        self.study_config.add_trait('output_fom', Str(Undefined, output=False,
            desc='output FOM', groups=['fom']))
        self.study_config.add_trait('shared_fom', Str(Undefined, output=False,
            desc='shared data FOM', groups=['fom']))
        self.study_config.add_trait('volumes_format',
                                    Str(Undefined, output=False,
            desc='Format used for volumes', groups=['fom']))
        self.study_config.add_trait('meshes_format',
            desc='Format used for meshes', groups=['fom']))
            'auto_fom',
            Bool(True, output=False,
                 desc='Look in all FOMs when a process is not found (in '
                 'addition to the standard share/foms). Note that auto_fom '
                 'looks for the first FOM matching the process to get '
                 'completion for, and does not handle ambiguities. Moreover '
                 'it brings an overhead (typically 6-7 seconds) the first '
                 'time it is used since it has to parse all available FOMs.',
                 groups=['fom']))
            'fom_path',
            List(Directory(output=False),
                 desc='list of additional directories where to look for FOMs',
        self.study_config.add_trait('use_fom', Bool(
            desc='Use File Organization Models for file parameters '
                'completion',
            groups=['fom']))

        # defaults
        self.study_config.input_fom = ""
        self.study_config.output_fom = ""
        self.study_config.shared_fom = ""
        self.study_config.modules_data.foms = {}
        self.study_config.modules_data.fom_atp = {}
        self.study_config.modules_data.fom_pta = {}

        if 'capsul.engine.module.fom' \
                not in self.study_config.engine._loaded_modules:
            self.study_config.engine.load_module('capsul.engine.module.fom')

        else:
            # 1st sync may have changed things

            self.sync_to_engine,
            ['input_directory', 'input_fom', 'meshes_format',
             'output_directory', 'output_fom', 'shared_fom', 'volumes_format',
             'auto_fom', 'fom_path', 'shared_directory'])
        self.study_config.engine.settings.module_notifiers[
            'capsul.engine.module.fom'].append(self.sync_from_engine)

        params = ['input_fom', 'meshes_format', 'output_fom', 'shared_fom',
                  'volumes_format', 'input_directory', 'output_directory',
                  'auto_fom', 'fom_path']

            config = session.config('fom', 'global')
            if config:
                if param in params:
                    params = [param]
                for p in params:
                    value = getattr(self.study_config, p)
                    setattr(config, p, value)
            else:
                values = {}
                    values[p] = value
                values[self.study_config.engine.settings.config_id_field] \
                    = 'fom'
                session.new_config('fom', 'global', values)

            params = [param]

            if not config:
                self.study_config.use_fom = False
                return
            for p in params:
                # print('sync_from_engine:', p, type(p))
                value = getattr(config, p)
                if value is None:
                    value = Undefined
                setattr(self.study_config, p, value)
            self.study_config.use_fom = True

    def load_fom(self, schema):
            if config:
                from capsul.engine.module.fom import load_fom
                return load_fom(self.study_config.engine, schema, config,
                                session, environment='global')

'''
FreeSurfer configuration module

Classes
=======
:class:`FreeSurferConfig`
'''

import os
import six

from traits.api import File, Bool, Undefined, Directory

class FreeSurferConfig(StudyConfigModule):
    """ Class to set up freesurfer configuration.

    Parse the 'SetUpFreeSurfer.sh' file and update dynamically the system
    environment.
    """
        """ Initialize the FreeSurferConfig class.

        study_config: StudyConfig object
            the study configuration we want to update in order to deal with 
            freesurfer functions.
        """
        super(FreeSurferConfig, self).__init__(study_config, configuration)
        self.study_config.add_trait("freesurfer_config", File(
            desc="Path to 'FreeSurferEnv.sh'", groups=['freesurfer']))
        self.study_config.add_trait('freesurfer_subjectsdir', Directory(
            desc='FreeSurfer subjects data directory', groups=['freesurfer']))
        self.study_config.add_trait("use_freesurfer", Bool(
            False,
            desc="If True, FreeSurfer configuration is set up on startup",
            groups=['freesurfer']))

        """ Set up FSL environment variables according to current
        """
        if 'capsul.engine.module.freesurfer' \
            self.study_config.engine.load_module(
                'capsul.engine.module.freesurfer')
        else:

            ['freesurfer_config', 'freesurfer_subjectsdir'])
            'capsul.engine.module.freesurfer'] = [self.sync_from_engine]

            # manage recursive calls
            return
        try:
            if 'FreeSurferConfig' in self.study_config.modules \
                    and 'capsul.engine.module.freesurfer' \
                        in self.study_config.engine._loaded_modules:
                with self.study_config.engine.settings as session:
                    cif = self.study_config.engine.settings.config_id_field
                    params = {
                        'freesurfer_config': 'setup',
                        'freesurfer_subjectsdir': 'subjects_dir',
                    }
                    config = session.config('freesurfer', 'global')
                    if config is None:
                        values = {cif: 'freesurfer'}
                        for param, ceparam in six.iteritems(params):
                            val = getattr(self.study_config, param)
                            if val is Undefined:
                                val = None
                            values[ceparam] = val
                        session.new_config('freesurfer', 'global', values)
                    else:
                                val = None
                            setattr(config, ceparam, val)
        finally:

            return
        try:
                    if config:
                        params = {
                            'freesurfer_config': 'setup',
                            'freesurfer_subjectsdir': 'subjects_dir',
                        }
                            val = getattr(config, ceparam)
                            if val in (None, ''):
                                val = Undefined
                            setattr(self.study_config, param, val)

                        if self.study_config.freesurfer_config:
                            self.study_config.use_freesurfer = True
                        else:
                            self.study_config.use_freesurfer = False
        finally:

'''
FSL configuration module

Classes
=======
:class:`FSLConfig`
'''

from traits.api import File, Bool, Undefined, String

from capsul.subprocess.fsl import check_fsl_configuration

class FSLConfig(StudyConfigModule):
    '''
    `FSL <https://fsl.fmrib.ox.ac.uk/fsl/fslwiki>`_ configuration module
    '''

        super(FSLConfig, self).__init__(study_config, configuration)
        self.study_config.add_trait('fsl_config', File(
            desc='Parameter to specify the fsl.sh path', groups=['fsl']))
        self.study_config.add_trait('fsl_prefix', String(Undefined,
            desc='Prefix to add to FSL commands', groups=['fsl']))
        self.study_config.add_trait('use_fsl', Bool(
            desc='Parameter to tell that we need to configure FSL',
            groups=['fsl']))

        try:
            self.study_config.engine.settings.module_notifiers.get(
                'capsul.engine.module.fsl', []).remove(self.sync_from_engine)
        except (ValueError, ReferenceError):
            pass

        """
        if 'capsul.engine.module.fsl' \
            self.study_config.engine.load_module('capsul.engine.module.fsl')
        else:
        # this test aims to raise an exception in case of incorrect setting,
        # complying to capsul 2.x behavior.
        if self.study_config.use_fsl is True:
            check_fsl_configuration(self.study_config)

            self.sync_to_engine, '[fsl_config, fsl_prefix, use_fsl]')
            'capsul.engine.module.fsl'] = [self.sync_from_engine]

            return
        try:
            if 'FSLConfig' in self.study_config.modules \
                    and 'capsul.engine.module.fsl' \
                    config = session.config('fsl', 'global')
                    fsl_conf = self.study_config.fsl_config
                    if fsl_conf is Undefined:
                        fsl_conf = None
                        fsl_dir = None
                    else:
                        fsl_dir = osp.dirname(fsl_conf)
                        if fsl_dir.endswith('/etc/fslconf'):
                            fsl_dir = osp.dirname(osp.dirname(fsl_dir))
                        elif fsl_dir.endswith('/etc'):
                            fsl_dir = osp.dirname(fsl_dir)
                        session.new_config(
                            'fsl', 'global',
                            {'directory': fsl_dir,
                             'config': fsl_conf,
                             'prefix': self.study_config.fsl_prefix
                                if self.study_config.fsl_prefix
                                    is not Undefined
                                else None,
                             cif: 'fsl'})
                    else:
                        tparam = {'fsl_config': 'config',
                                  'fsl_prefix': 'prefix'}
                        if param is not None:
                            if param not in ('use_fsl', ):
                                params = [param]
                            else:
                                params = []
                        else:
                            params = ['fsl_config', 'fsl_prefix']
                        defaults = {'prefix': None, 'config': None}
                        for p in params:
                            val = getattr(self.study_config, p)
                            ceparam = tparam[p]
                                val = defaults.get(ceparam, None)
                        if 'fsl_config' in params:
                            config.directory = fsl_dir
                    del config
                del session
        finally:

            return
        try:
                    if config:
                        directory = config.directory \
                            if config.directory not in (None, '') \
                            else Undefined

                        self.study_config.fsl_config = config.config \
                            if config.config is not None else Undefined
                        self.study_config.fsl_prefix = config.prefix \
                            if config.prefix is not None else Undefined
                        if self.study_config.fsl_config not in (
                                    '', None, Undefined)\
                                or self.study_config.fsl_prefix not in (
                                    '', None, Undefined):
                            self.study_config.use_fsl = True
                        else:
                            self.study_config.use_fsl = False
            pass
        finally:

'''
Matlab configuration module

Classes
=======
:class:`MatlabConfig`
'''

import os
from traits.api import File, Undefined, Bool

class MatlabConfig(StudyConfigModule):
    '''
    Matlab path configuration
    '''

        super(MatlabConfig, self).__init__(study_config, configuration)
        self.study_config.add_trait('matlab_exec', File(
            desc='Matlab command path',
            exists=True,
            groups=['matlab']))
        self.study_config.add_trait("use_matlab", Bool(
            desc="If True, Matlab configuration is set up on startup",

        """ Set up Matlab environment according to current configuration.
        """
        if not self.study_config.matlab_exec and self.study_config.use_matlab:
            # fix pathological config (appearing sometimes for an unknown reason)
            self.study_config.use_matlab = Undefined
        if 'capsul.engine.module.matlab' \
            self.study_config.engine.load_module('capsul.engine.module.matlab')

        else:

        # the following should be moved to CapsulEngine module
        if self.study_config.use_matlab is False:
            # Configuration is explicitly asking not to use Matlab
            return

        if self.study_config.use_matlab is Undefined:
            # If use_matlab is not defined, Matlab configuration will
            # be done if possible but there will be no error if it cannot be
            # done.
            force_configuration = False
        else:
            # If use_matlab is True configuration must be valid otherwise
            # an EnvironmentError is raised
            force_configuration = True

        if self.study_config.matlab_exec is Undefined:
            # matlab_exec is not set, it will not be possible to activate
            #Matlab
            self.study_config.use_matlab = False
            if force_configuration:
                raise EnvironmentError('matlab_exec must be defined in order '
                                       'to use Matlab')
            return

        if not os.path.exists(self.study_config.matlab_exec):
                raise EnvironmentError('"%s" does not exist. Matlab '
                                      'configuration is not valid.' % \
                                      self.study_config.matlab_exec)
            return

        self.study_config.on_trait_change(self.sync_to_engine, 'matlab_exec')
            'capsul.engine.module.matlab'] = [self.sync_from_engine]

            tparam = {'matlab_exec': 'executable'}
            ceparam = tparam.get(param)
        else:
            ceparam = 'executable'
            value = self.study_config.matlab_exec
            if value is Undefined:
                value = None
        if ceparam is not None:
                config = session.config('matlab', 'global', any=True)
                    if (ceparam !='executable'
                             or self.study_config.matlab_exec
                                not in (None, Undefined, '')):
                        cif = self.study_config.engine.settings.config_id_field
                        matlab_exec = self.study_config.matlab_exec
                        if matlab_exec is Undefined:
                            matlab_exec = None
                            'matlab', 'global',
                            {'executable': matlab_exec,
                            cif: 'matlab'})
                else:
                    setattr(config, ceparam, value)

        self.use_matlab = None  # avoid transcient inconsistencies
            tparam = {'executable': 'matlab_exec'}
            scparam = tparam.get(param)
            if scparam is not None:
                setattr(self.study_config, scparam, value)
        else:
            if self.study_config.use_matlab:
                # in case we reset matlab_exec to Undefined
                self.study_config.use_matlab = Undefined
                if config:
                    matlab_exec = config.executable
                    if matlab_exec is None:
                        matlab_exec = Undefined
                    self.study_config.matlab_exec = matlab_exec
        if self.study_config.matlab_exec not in (None, Undefined):
            self.study_config.use_matlab = True

'''
NiPype configuration module

Classes
=======
:class:`NipypeConfig`
'''

import os

from traits.api import Bool, Undefined, List, Directory

# NIPYPE import
try:
    import nipype.interfaces.matlab as matlab
    from nipype.interfaces import spm
    has_nipype = True
    has_nipype = False

import capsul.engine.module.nipype as ce_nipype

class NipypeConfig(StudyConfigModule):
    """ Nipype configuration.

    In order to use nipype spm, fsl and freesurfer interfaces, we need to
    configure the nipype module.
    """

        """ Initialize the NipypeConfig class.
        """
        super(NipypeConfig, self).__init__(study_config, configuration)
        study_config.add_trait("use_nipype", Bool(
            desc="If True, Nipype configuration is set up on startup.",
            groups=['nipype']))
        study_config.add_trait("add_to_default_matlab_path", List(
            Directory,
            default=[],
            desc="Paths that are added to Matlab default path.",
        self.has_nipype = has_nipype

        """ Set up Nipype environment variables according to current
        """
        if 'capsul.engine.module.nipype' \
                'capsul.engine.module.nipype')
        ce_nipype.ensure_config_exists(self.study_config.engine)
        if not self.has_nipype:
            self.study_config.use_nipype = False
            return
        if self.study_config.use_nipype is False:
            # Configuration is explicitly asking not to use Nipype
            return
        elif self.study_config.use_nipype is True:
            # If use_nipype is True configuration must be valid otherwise
        else:
            # If use_nipype is not defined, Nipype configuration will
            # done.

        # Configure matlab for nipype
        if self.study_config.use_matlab:
            matlab.MatlabCommand.set_default_matlab_cmd(
                self.study_config.matlab_exec + " -nodesktop -nosplash")

        # Configure spm for nipype
        if self.study_config.use_spm is True:

            # Standalone spm version
            if self.study_config.spm_standalone is True:
                spm.SPMCommand.set_mlab_paths(
                    matlab_cmd=self.study_config.spm_exec + " " \
                        + glob.glob(os.path.join(
                            self.study_config.spm_directory, 'mcr', 'v*'))[0] \
                        + " script",
                    use_mcr=True)
            # Matlab spm version
            else:
                matlab.MatlabCommand.set_default_paths(
                    [self.study_config.spm_directory] +
                    self.study_config.add_to_default_matlab_path)
                spm.SPMCommand.set_mlab_paths(matlab_cmd="", use_mcr=False)  

        self.study_config.use_nipype = True

        """ When the 'use_nipype' trait changes, configure nipype with the new
        setting.
        """
        self.study_config.on_trait_change(self.initialize_module, "use_nipype")

'''
Process execution cache configuration module

Classes
=======
:class:`SmartCachingConfig`
'''

from traits.api import Bool, Undefined

class SmartCachingConfig(StudyConfigModule):
    '''
    '''
        super(SmartCachingConfig, self).__init__(study_config, configuration)
        study_config.add_trait('use_smart_caching', Bool(
            False,
            desc='Use smart-caching during the execution',
            groups=['smartcaching']))
        # self.study_config.on_trait_change(self._use_smart_caching_changed, 'use_smart_caching')

'''
`Soma-Workflow <http://brainvisa.info/soma-workflow/>`_ configuration module

Classes
=======
:class:`ResourceController`
:class:`SomaWorkflowConfig`
'''

from traits.api import Bool, Str, Undefined, List, Dict, File
from soma.controller import Controller, ControllerTrait, OpenKeyController

class ResourceController(Controller):
    ''' Configuration options for one Soma-Workflow computing resource

    queue: str
        Jobs queue to be used on the computing resource for w
        orkflow submissions
    transfer_paths: list(str)
        list of paths where files have to be transferred by soma-workflow
    path_translations: dict(str, (str, str))
        Soma-workflow paths translations mapping:
        ``{local_path: (identifier, uuid)}``
    '''
        super(ResourceController, self).__init__()
        self.add_trait(
            'queue',
                desc='Jobs queue to be used on the computing resource for '
                'workflow submissions'))
            'transfer_paths', List(
                [],
                output=False,
                desc='list of paths where files have to be transferred '
                'by soma-workflow'))
            'path_translations',
            ControllerTrait(
                OpenKeyController(
                    value_trait=List(trait=Str(), value=('', ''),
                    minlen=2, maxlen=2)),
                desc='Soma-workflow paths translations mapping: '
                '{local_path: (identifier, uuid)}'))

class SomaWorkflowConfig(StudyConfigModule):
    ''' Configuration module for :somaworkflow:`Soma-Workflow <index.html>`

    Stores configuration options which are not part of Soma-Workflow own
    configuration, and used to run workflows from CAPSUL pipelines.

    The configuration module may also store connected Soma-Workflow
    :somaworkflow:`WorkflowController <client_API.html>` objects to allow
    monitoring and submitting workflows.

    use_soma_workflow: bool
        Use soma workflow for the execution
    somaworkflow_computing_resource: str
        Soma-workflow computing resource to be used to run processing
    somaworkflow_config_file: filename
        Soma-Workflow configuration file. Default: ``$HOME/.soma_workflow.cfg``
    somaworkflow_keep_failed_workflows: bool
        Keep failed workflows after pipeline execution through StudyConfig
    somaworkflow_keep_succeeded_workflows: bool
        Keep succeeded workflows after pipeline execution through StudyConfig
    somaworkflow_computing_resources_config: dict(str, ResourceController)
        Computing resource config dict, keys are resource ids. Values are
        :py:class:`ResourceController` instances

    Methods
    -------
    get_resource_id
    set_computing_resource_password
    get_workflow_controller
    connect_resource
    disconnect_resource
    '''
    
        super(SomaWorkflowConfig, self).__init__(study_config, configuration)
        study_config.add_trait('use_soma_workflow', Bool(
            False,
            desc='Use soma workflow for the execution',
            groups=['soma-workflow']))
        study_config.add_trait(
            'somaworkflow_computing_resource',
            Str(
                Undefined,
                desc='Soma-workflow computing resource to be used to run processing',
                groups=['soma-workflow']))
            'somaworkflow_config_file',
            File(Undefined, output=False, optional=True,
                 desc='Soma-Workflow configuration file. '
                 'Default: $HOME/.soma_workflow.cfg',
                 groups=['soma-workflow']))
            'somaworkflow_keep_failed_workflows',
            Bool(
                True,
                desc='Keep failed workflows after pipeline execution through '
                'StudyConfig',
            'somaworkflow_keep_succeeded_workflows',
            Bool(
                False,
                desc='Keep succeeded workflows after pipeline execution '
                'through StudyConfig',
            'somaworkflow_computing_resources_config',
                    value_trait=ControllerTrait(
                        ResourceController(),
                        output=False, allow_none=False,
                        desc='Computing resource config')),
                output=False, allow_none=False,
                desc='Computing resource config',
        self.study_config.modules_data.somaworkflow = {}

            self.initialize_module, 'use_soma_workflow')

    def get_resource_id(self, resource_id=None, set_it=False):
        ''' Get a computing resource name according to the (optional) input
        resource_id and Soma-Workflow configuration.

        For instance, ``None`` or ``"localhost"`` will be transformed to the
        local host id.

        resource_id: str (optional)
        set_it: bool (optional)
            if True, the computed resource id will be set as the current
            resource in the somaworkflow_computing_resource trait value.

        Returns
        -------
        computed resource id.
        '''
        if resource_id in (None, '', Undefined):
            resource_id = self.study_config.somaworkflow_computing_resource
        else:
            self.study_config.somaworkflow_computing_resource = resource_id
        if resource_id in (None, Undefined, 'localhost', ''):
            from soma_workflow import configuration as swcf
            resource_id = swcf.Configuration.get_local_resource_id()
        if set_it:
            if resource_id is None:
                import socket
                resource_id = socket.gethostname()
        return resource_id

    def set_computing_resource_password(self, resource_id, password=None,
                                        rsa_key_password=None):
        ''' Set credentials for a given computinf resource.

        Such credentials are stored in the config object, but will not be
        written when the config is saved in a file. They are thus non-
        persistent.

        password: str (optional)
        rsa_key_password: str (optional)
        '''
        resource_id = self.get_resource_id(resource_id)
        r = self.study_config.modules_data.somaworkflow.setdefault(
            resource_id, Controller())
        if password:
            r.password = password
        if rsa_key_password:
            r.rsa_key_password = rsa_key_password

    def get_workflow_controller(self, resource_id=None):
        ''' Get a connected
        :somaworkflow:`WorkflowController <client_API.html>` for the given
        resource
        '''

        wc = getattr(r, 'workflow_controller', None)
        return wc

    def connect_resource(self, resource_id=None, force_reconnect=False):
        ''' Connect a soma-workflow computing resource.

        Sets the current resource to the given resource_id (transformed by
        get_resource_id() if None or "localhost" is given, for instance).

            resource name, may be None or "localhost". If None, the current
            one (study_config.somaworkflow_computing_resource) is used, or
            the localhost if none is configured.
        force_reconnect: bool (optional)
            if True, if an existing workflow controller is already connected,
            it will be disconnected (deleted) and a new one will be connected.
            If False, an existing controller will be reused without
            reconnection.

        Returns
        -------
        :somaworkflow:`WorkflowController <client_API.html>` object
        '''
        import soma_workflow.client as swclient

        conf_file = self.study_config.somaworkflow_config_file
        if conf_file in (None, Undefined):
            conf_file \
                = swclient.configuration.Configuration.search_config_path()

        resource_id = self.get_resource_id(resource_id, True)

        if force_reconnect:
            self.disconnect_resource(resource_id)

        if not force_reconnect:
            wc = self.get_workflow_controller(resource_id)
            if wc is not None:
                return wc

        login = swclient.configuration.Configuration.get_logins(
            conf_file).get(resource_id)
        config = getattr(r, 'config', None)
            config = swclient.configuration.Configuration.load_from_file(
                resource_id=resource_id, config_file_path=conf_file)
            r.config = config
        password = getattr(r, 'password', None)
        rsa_key_pass = getattr(r, 'rsa_key_password', None)
        wc = swclient.WorkflowController(
            resource_id=resource_id,
            login=login,
            password=password,
            rsa_key_pass=rsa_key_pass,
            config=config)
        r.workflow_controller = wc
        return wc

    def disconnect_resource(self, resource_id=None):
        ''' Disconnect a connected
        :somaworkflow:`WorkflowController <client_API.html>` and removes it
        from the internal list
        '''
        wc = self.get_workflow_controller(resource_id)
        if wc:
            wc.disconnect()
            del self.study_config.modules_data.somaworkflow[
                self.study_config.somaworkflow_computing_resource
                ].workflow_controller

'''
SPM configuration module

Classes
=======
:class:`SPMConfig`
'''

from traits.api import Directory, File, Bool, Enum, Undefined, Str
from capsul.subprocess.spm import check_spm_configuration
import os

class SPMConfig(StudyConfigModule):
    """ SPM configuration.

    There is two ways to configure SPM:
        * the first one requires to configure matlab and then to set the spm
          directory.
        * the second one is based on a standalone version of spm and requires
          to set the spm executable directory.
    """

    dependencies = ["MatlabConfig"]

        super(SPMConfig, self).__init__(study_config, configuration)
        self.study_config.add_trait("spm_directory", Directory(
            desc="Directory containing SPM.",
            groups=['spm']))
        self.study_config.add_trait("spm_standalone", Bool(
            desc="If True, use the standalone version of SPM.",
        self.study_config.add_trait('spm_version', Str(
            Undefined, output=False,
        self.study_config.add_trait("spm_exec", File(
            desc="SPM standalone (MCR) command path (run_spm12.sh file).",
            'spm_mcr_directory',
            Directory(Undefined, output=False,
                      desc='Matlab MCR directory for standalone version',
                      groups=['spm']))
        self.study_config.add_trait("use_spm", Bool(
            desc="If True, SPM configuration is checked on module "
                 "initialization.",

        if 'capsul.engine.module.spm' \
            self.study_config.engine.load_module('capsul.engine.module.spm')

        else:
            check_spm_configuration(self.study_config)

                                          "[use_spm, spm_+]")
            'capsul.engine.module.spm'] = [self.sync_from_engine]

            return
        try:
            if 'SPMConfig' in self.study_config.modules \
                    and 'capsul.engine.module.spm' \
                    configs = list(session.configs('spm', 'global'))
                    configs = sorted(
                        configs,
                        key=lambda x: (
                            -int(x.version)
                                if x.version not in ('', None)
                                else 0)
                            * 1000 - int(bool(x.standalone)))
                    mlconfigs = list(session.configs('matlab', 'global'))
                    mlconfig = None
                    if len(mlconfigs) != 0:
                        mlconfig = mlconfigs[0]
                    if len(configs) != 0:
                        config = configs[0]
                        self.study_config.spm_directory = directory
                        if mlconfig and mlconfig.mcr_directory:
                            self.study_config.spm_mcr_directory \
                                = mlconfig.mcr_directory
                        self.study_config.spm_standalone \
                            = bool(config.standalone)
                        if config.version is not None:
                            self.study_config.spm_version = config.version
                        else:
                            self.study_config.spm_version = '0'
                        if directory not in (None, Undefined):
                            if config.version:
                                spm_exec = \
                                    osp.join(directory,
                                             'run_spm%s.sh' % config.version)
                                if os.path.exists(spm_exec):
                                    spm_exec = [spm_exec]
                                else:
                                    spm_exec = []
                            else:
                                spm_exec = glob.glob(osp.join(
                                    directory,
                                    'run_spm*.sh'))
                            if len(spm_exec) != 0:
                                self.study_config.spm_exec = spm_exec[0]
                            else:
                                self.study_config.spm_exec = Undefined
                        if self.study_config.spm_directory \
                                    not in (None, Undefined) \
                                and self.study_config.spm_exec \
                                and (not self.study_config.spm_standalone
                                     or self.study_config.spm_mcr_directory
                                        not in (None, Undefined)):
                            self.study_config.use_spm = True
                        else:
                            self.study_config.use_spm = False

        finally:

    def sync_to_engine(self):
            return
        try:
                    version = self.study_config.spm_version
                    if version in (None, Undefined):
                        version = '12'
                    standalone = (self.study_config.spm_standalone is True)
                    id = 'spm%s%s' % (version,
                                      '-standalone' if standalone else '')
                    query = '%s == "%s"' % (cif, id)
                    config = session.config('spm', 'global', selection=query)
                        # FIXME: do we keep only one config ? yes for now.
                        config_ids = [getattr(conf, cif)
                                      for conf in session.configs('spm',
                                                                  'global')
                                      if getattr(conf, cif) != id]
                        for config_id in config_ids:
                            session.remove_config('spm', 'global', config_id)
                        sconf = {
                            'directory':
                                self.study_config.spm_directory
                                    if self.study_config.spm_directory
                                        is not Undefined
                                    else None,
                            'version': version,
                            'standalone': standalone,
                            cif: id}
                        session.new_config('spm', 'global', sconf)
                    else:
                        tparam = {'spm_directory': 'directory',
                                  'spm_standalone': 'standalone',
                                  'spm_version': 'version'}
                        params = ['spm_directory',
                                  'spm_standalone',
                                  'spm_version']
                        defaults = {'directory': None, 'standalone': False,
                                    'version': '12'}
                        val = self.study_config.spm_exec
                        if val is Undefined:
                            val = None
                        else:
                            val = osp.dirname(self.study_config.spm_exec)
                        config.directory = val

                    if self.study_config.spm_mcr_directory \
                            and self.study_config.spm_standalone:
                        config = session.config('matlab', 'global',
                                                selection='%s == "matlab"'
                                                % cif)
                        mcr_dir = self.study_config.spm_mcr_directory
                        if mcr_dir is Undefined:
                            mcr_dir = None
                        if config is None:
                            session.new_config(
                                'matlab', 'global',
                                {'mcr_directory': mcr_dir,
                                 'config_id': 'matlab'})
                        else:
                            config.mcr_directory = mcr_dir
        finally:

'''
Specific subprocess-like functions to call FSL taking into account a 
potential configuration done in StudyConfig. If a StudyConfig is not
configured to use FSL, it may be automatically configured. Automatic
configuration had been tested in the two following cases:

- FSL was installed from the FMRIB site and, at least, FSLDIR 
  environment variable is set (fsl.sh can be sourced or not)
- FSL was installed from Neurodebian packages

Prefix areused  in Neurodebian install. For instance on Ubuntu 16.04 
of the StudyConfig instance.

Classes
=======
:class:`Popen`
--------------

Functions
=========
:func:`fsl_command_with_environment`
:func:`check_fsl_configuration`
:func:`check_configuration_values`
:func:`auto_configuration`
:func:`call`
------------
:func:`check_call`
:func:`check_output`

'''

import os

from capsul.engine.module import fsl as fsl_engine

def fsl_command_with_environment(study_config, command):
    '''
    call taking into account the study_config FSL configuration.
    '''
    else:
    fsl_prefix = getattr(study_config, 'fsl_prefix', '')
    if fsl_prefix is Undefined:
    if getattr(study_config, 'fsl_config', Undefined) is Undefined:
        cmd = ['%s%s%s' % (dir_prefix, 
    else:
        fsldir = osp.dirname(osp.dirname(osp.dirname(study_config.fsl_config)))
                'setenv FSLDIR "{0}";source {0}/etc/fslconf/fsl.csh;exec {0}/bin/{1}{2} '.format(fsldir, fsl_prefix, command[0]) + \
        else:
                'export FSLDIR="{0}";. {0}/etc/fslconf/fsl.sh;exec {0}/bin/{1}{2} '.format(fsldir, fsl_prefix, command[0]) + \
    return cmd

def check_fsl_configuration(study_config):
    '''
    Check thas study_config configuration is valid to call FSL commands.
    If not, try to automatically configure FSL. Finally raises an
    EnvironmentError if configuration is still wrong.
    '''

    if getattr(study_config, '_fsl_config_checked', False):
        # Check FLS configuration only once
        return
    if 'FSLConfig' not in study_config.modules:
        raise EnvironmentError('FSLConfig module is missing in StudyConfig.')
    if study_config.use_fsl is False:
        raise EnvironmentError('Configuration is set not to use FLS. Set use_fsl to True in order to use FSL.')
    # Configuration must be valid otherwise
    # try to update configuration and recheck is validity
    if check_configuration_values(study_config) is not None:
        auto_configuration(study_config)
        error_message = check_configuration_values(study_config)
        if error_message:
            raise EnvironmentError(error_message)
    study_config.use_fsl = True
    study_config._fsl_config_checked = True
    
def check_configuration_values(study_config):
    '''
    Check if the configuration is valid to run FLS and returns an error
    message if there is an error or None if everything is good.
    '''

    if study_config.fsl_config is Undefined:
        if not find_in_path('%sbet' % fsl_prefix):
    else:
            return 'FSL configuration must either use fsl_config or fsl_prefix but not both'
        if not osp.exists(study_config.fsl_config):
            return 'File "%s" does not exist' % study_config.fsl_config
        if not study_config.fsl_config.endswith('fsl.sh'):
            return 'File "%s" is not a path to fsl.sh script' % study_config.fsl_config

def auto_configuration(study_config):
    '''
    Try to automatically set the study_config configuration for FSL.
    '''
        fsl_config = '%s/etc/fslconf/fsl.sh' % fsl_dir
            study_config.fsl_config = fsl_config
        bet = find_in_path('fsl*-bet')
        if bet:
            study_config.fsl_prefix = osp.basename(bet)[:-3]

class Popen(soma.subprocess.Popen):
    '''
    '''
    def __init__(self, study_config, command, **kwargs):
        check_fsl_configuration(study_config)
        cmd = fsl_command_with_environment(study_config, command)
        super(Popen, self).__init__(cmd, **kwargs)
        
def call(study_config, command, **kwargs):
    '''
    '''
    check_fsl_configuration(study_config)
    cmd = fsl_command_with_environment(study_config, command)

def check_call(study_config, command, **kwargs):
    '''
    '''

def check_output(study_config, command, **kwargs):
    '''
    '''

'''
Classes
=======

Functions
=========
:func:`find_spm`
:func:`check_spm_configuration`
:func:`spm_command`
'''

def find_spm(spm_version='', matlab_exec='matlab', matlab_path=None):
    """ Function to return the root directory of SPM.

    spm_version: str (default='')
        if given, the version to use when running spm<version> in Matlab.
    matlab: str (default='matlab')
        if given, is the path to the Matlab executable.
    matlab_path: str (default None)
        if given, is a Matlab expression fed to addpath.

    Returns
    -------
    last_line: str
        the SPM root directory
    """
    # Script to execute with matlab in order to find SPM root dir
    script = ("spm%s;"
              "fprintf(1, '%%s', spm('dir'));"
              "exit();") % spm_version

    # Add matlab path if necessary
    if matlab_path:
        script = "addpath({0});".format(matlab_path) + script

    # Generate the matlab command
    command = [matlab_exec,
               "-nodisplay", "-nosplash", "-nojvm",
               "-r", script]

    # Try to execute the command
    try:
        last_line = soma.subprocess.check_output(command).decode('utf-8')
    except OSError:
        raise Exception("Could not find SPM.")

    # Do not consider weird data at the end of the line
    if '\x1b' in last_line:
        last_line = last_line[:last_line.index('\x1b')]

    # If the last line is empty, SPM not found
    if last_line == "":

    return last_line

def check_spm_configuration(study_config):
    '''
    Obsolete.

    Check thas study_config configuration is valid to call SPM commands.
    If not, try to automatically configure SPM. Finally raises an
    '''
    if getattr(study_config, '_spm_config_checked', False):
        # Check SPM configuration only once
        return
    if 'SPMConfig' not in study_config.modules:
        raise EnvironmentError('SPMConfig module is missing in StudyConfig.')
    if study_config.use_spm is False:
        raise EnvironmentError(
            'Configuration is set not to use SPM. Set use_spm to True in '
            'order to use SPM.')
    study_config.use_spm = True
    study_config._spm_config_checked = True

    '''
    Obsolete.

    Check if the configuration is valid to run SPM and returns an error
    '''

    if study_config.spm_directory is Undefined:
        return 'No SPM directory defined'
    if not osp.isdir(study_config.spm_directory):
        return "'%s' is not a valid SPM directory" % study_config.spm_directory
    if study_config.spm_standalone is Undefined:
        return 'SPM standalone usage is undefined'
    if study_config.spm_standalone:
        if study_config.spm_exec is Undefined:
            return 'spm_exec must be defined to use SPM standalone'
        if not osp.isdir(study_config.spm_mcr_directory):
            return '"%s" is not a valid mrc directory for SPM standalone' \
                % study_config.spm_mcr_directory
    else:
        if not study_config.use_matlab:
            return 'Matlab is disabled. Cannot use SPM via Matlab'

    '''
    Obsolete.

    Try to automatically set the study_config configuration for SPM.
    '''
    if study_config.spm_directory is not Undefined and \
            osp.exists(study_config.spm_directory):
        spm_sh = glob.glob(osp.join(study_config.spm_directory, 'run_spm*.sh'))
        if spm_sh:
            if study_config.spm_standalone is Undefined:
                study_config.spm_standalone = True
            elif not study_config.spm_standalone:
                raise EnvironmentError(
                    'Requested SPM non standalone but directory "%s" contains '
                    'SPM standalone' % study_config.spm_directory)

            spm_version = spm_sh[0][:-3]
            spm_version = spm_version[spm_version.rfind('run_spm')+7:]
            if study_config.spm_version is Undefined:
                study_config.spm_version = spm_version
            elif study_config.spm_version != spm_version:
                    'Requested SPM %s but directory "%s" contains SPM %s'
                    % (study_config.spm_version, study_config.spm_directory,
                       spm_version))
            mcr_dir = glob.glob(osp.join(study_config.spm_directory, 'mcr',
                                         'v*'))
            if mcr_dir:
                study_config.spm_mcr_directory = mcr_dir[0]
        return

    # Matlab
    # determine SPM version (currently only 8 or 12 are supported)
    if study_config.spm_directory is not Undefined \
            and osp.isdir(osp.join(study_config.spm_directory, 'toolbox',
                                   'OldNorm')):
        spm_version = '12'
    elif study_config.spm_directory is not Undefined \
            and osp.isdir(osp.join(study_config.spm_directory, 'templates')):
        spm_version = '8'
    else:
        spm_version = Undefined
    if spm_version is not Undefined:
        if study_config.spm_version is Undefined:
            study_config.spm_version = spm_version
        elif study_config.spm_version != spm_version:
            raise EnvironmentError(
                'Requested SPM %s but directory "%s" contains SPM %s'
                % (study_config.spm_version, study_config.spm_directory,
                   spm_version))

def spm_command(study_config, batch_file):
        cmd = [study_config.spm_exec,
               study_config.spm_mcr_directory,
               'script',
               batch_file]
        return cmd
    else:

    '''
    Equivalent to Python soma.subprocess.Popen for SPM batch
    '''

    def __init__(self, study_config, batch_file, **kwargs):
        check_spm_configuration(study_config)
        cmd = spm_command(study_config, batch_file)

def call(study_config, batch_file, **kwargs):
    '''
    Equivalent to Python soma.subprocess.call for SPM batch
    '''
    check_spm_configuration(study_config)
    cmd = spm_command(study_config, batch_file)

def check_call(study_config, batch_file, **kwargs):
    '''
    Equivalent to Python soma.subprocess.check_call for SPM batch
    '''

def check_output(study_config, batch_file, **kwargs):
    '''
    Equivalent to Python soma.subprocess.check_output for SPM batch
    '''

    from capsul.api import StudyConfig
    from capsul.soma.subprocess.spm import check_call as call_spm

    sc = StudyConfig(spm_directory='/home/yc176684/spm12-standalone-7219')
    batch = tempfile.NamedTemporaryFile(suffix='.m')
    batch.write("fprintf(1, '%s', spm('dir'));")
    batch.flush()
    call_spm(sc, batch.name)

'''
Utilities to find processes

Functions
=========
:func:`find_processes`
'''

import sys
import pkgutil
import re
from glob import glob

pipeline_xml_re = re.compile(r'<pipeline.*</pipeline>', re.DOTALL)
pipeline_json_re = re.compile(r'{.*"definition":', re.DOTALL)

def find_processes(module_name, ignore_import_error=True):
    ''' Find processes in a module and iterate over them
    '''
    importlib.import_module(module_name)
    module_names  = [module_name]
    for i, m, p in pkgutil.walk_packages(module.__path__, 
                                         prefix='%s.' % module_name):
        module_names.append(m)

    # avoid loading nipype if not needed, since it takes time
    nipype_loaded = False
    Interface = None

    for module_name in module_names:
        try:
            importlib.import_module(module_name)
            if not ignore_import_error:
                raise
            continue
        module = sys.modules[module_name]
        for name in dir(module):
            item = getattr(module, name)
            if not nipype_loaded:
                nipype = sys.modules.get('nipype.interfaces.base')
                if nipype is not None:
                    Interface = getattr(nipype, 'Interface')
                    nipype_loaded = True
                elif Interface is None:
                    # If nipype is not found create a dummy Interface class
                    Interface = type("Interface", (object, ), {})
            if (isinstance(item, type) and
                issubclass(item, Process)):
                yield '%s.%s' % (module_name, name)
            elif isinstance(item, Interface):
                # If we have a Nipype interface, wrap this structure in a Process
                # class
            elif isinstance(item, types.FunctionType):
                # Check docstring
                if getattr(item, 'capsul_xml', None) or (item.__doc__ and process_xml_re.search(item.__doc__)):
                    yield '%s.%s' % (module_name, name)
        module_dir = osp.dirname(module.__file__)
        for f in glob(osp.join(module_dir, '*.xml')):
            xml = open(osp.join(module_dir, f)).read()
            if pipeline_xml_re.search(xml):
                yield '%s.%s' % (module_name, osp.basename(f)[:-4])
        for f in glob(osp.join(module_dir, '*.json')):
            json = open(osp.join(module_dir, f)).read()
            if pipeline_json_re.search(json):

'''
Functions
=========
:func:`get_tool_version`
:func:`get_nipype_interfaces_versions`
--------------------------------------
'''

def get_tool_version(tool):
    """ Get the version of a python tool.

    Check if the python tool module has a '__version__' attribute and return
    this value. If this attribute is not found, return None.

    tool: str (mandatory)
        a tool name

    Returns
    -------
    version: str
        the tool version, None if no information found in the module.
    """
    # Initialize the version to None ie. not found
    version = None

    # Try to get the '__version__' module attribute.
    try:
        module = __import__(tool)
        version = module.__version__
    except Exception:
        pass

    logger.debug("Module '{0}' version is {1}".format(tool, version))

    return version

def get_nipype_interfaces_versions():
    """ Get the versions of the nipype interfaces.

    If nipype is not found, return None.
    If no interfaces are configured, returned an empty dictionary.

    Returns
    -------
    versions: dict
        a dictionary with interface names as keys and corresponding
        versions as values.
    """
    # Initialize the versions to an empty dict
    versions = {}

    # Try to load the nipype interfaces module
    try:
        nipype_module = __import__("nipype.interfaces")

        # List all the interface
        sub_modules = [
            "{0}".format(i)
            for i in dir(nipype_module)
            if (not i.startswith("_") and not i[0].isupper())]

        # For each interface, try to get its version and fill the
        # output structure
        for module in sub_modules:
            try:
                version = eval("nipype_module.{0}."
                               "Info.version()".format(module))
                if version:
                    versions[module] = version
                pass
        versions = None

    logger.debug("Nipype interfaceversions are {0}".format(versions))

    return versions