Task #3042: Reintegrated task branch with the trunk.

.gitattributes

@@ -697,7 +697,6 @@ CEP/DP3/AOFlagger/src/strategy/control/actionblock.cpp -text
 CEP/DP3/AOFlagger/src/strategy/control/actionfactory.cpp -text
 CEP/DP3/AOFlagger/src/strategy/control/strategyreader.cpp -text
 CEP/DP3/AOFlagger/src/strategy/control/strategywriter.cpp -text
-CEP/DP3/AOFlagger/src/strategy/imagesets/fitsimageset.cpp -text
 CEP/DP3/AOFlagger/src/strategy/imagesets/imageset.cpp -text
 CEP/DP3/AOFlagger/src/strategy/imagesets/msimageset.cpp -text
 CEP/DP3/AOFlagger/src/strategy/imagesets/parmimageset.cpp -text

CEP/Pipeline/framework/lofarpipe/support/baserecipe.py

@@ -8,20 +8,16 @@
 from ConfigParser import NoOptionError, NoSectionError
 from ConfigParser import SafeConfigParser as ConfigParser
 from threading import Event
-from functools import partial
 
 import os
 import sys
-import inspect
 import logging
 import errno
 
 import lofarpipe.support.utilities as utilities
-import lofarpipe.support.lofaringredient as ingredient
-from lofarpipe.support.lofarexceptions import PipelineException
+from lofarpipe.support.lofarexceptions import PipelineException, PipelineRecipeFailed
 from lofarpipe.cuisine.WSRTrecipe import WSRTrecipe
 from lofarpipe.support.lofaringredient import RecipeIngredients, LOFARinput, LOFARoutput
-from lofarpipe.support.remotecommand import run_remote_command
 from lofarpipe.support.group_data import store_data_map
 
 class BaseRecipe(RecipeIngredients, WSRTrecipe):
@@ -42,6 +38,11 @@ class BaseRecipe(RecipeIngredients, WSRTrecipe):
         super(BaseRecipe, self).__init__()
         self.error = Event()
         self.error.clear()
+        # Environment variables we like to pass on to the node script.
+        self.environment = dict(
+            (k,v) for (k,v) in os.environ.iteritems() 
+                if k.endswith('PATH') or k.endswith('ROOT')
+        )
 
     @property
     def __file__(self):

CEP/Pipeline/framework/lofarpipe/support/lofarnode.py

@@ -45,6 +45,7 @@ class LOFARnode(object):
         self.loghost = loghost
         self.logport = int(logport)
         self.outputs = {}
+        self.environment = os.environ
 
     def run_with_logging(self, *args):
         """

CEP/Pipeline/framework/lofarpipe/support/remotecommand.py

@@ -204,8 +204,8 @@ class ComputeJob(object):
                 self.host,
                 self.command,
                 {
-                    "PYTHONPATH": config.get('deploy', 'engine_ppath'),
-                    "LD_LIBRARY_PATH": config.get('deploy', 'engine_lpath')
+                    "PYTHONPATH": os.environ.get('PYTHONPATH'),
+                    "LD_LIBRARY_PATH": os.environ.get('LD_LIBRARY_PATH')
                 },
                 arguments = [id, jobhost, jobport]
             )

CEP/Pipeline/recipes/sip/master/dppp.py

@@ -38,11 +38,6 @@ class dppp(BaseRecipe, RemoteCommandRecipeMixIn):
             '--executable',
             help="The full path to the relevant DPPP executable"
         ),
-        'initscript': ingredient.FileField(
-            '--initscript',
-            help="The full path to an (Bourne) shell script which will "
-                 "intialise the environment (ie, ``lofarinit.sh``)"
-        ),
         'suffix': ingredient.StringField(
             '--suffix',
             default=".dppp",
@@ -210,7 +205,7 @@ class dppp(BaseRecipe, RemoteCommandRecipeMixIn):
                         sourcedb,
                         self.inputs['parset'],
                         self.inputs['executable'],
-                        self.inputs['initscript'],
+                        self.environment,
                         self.inputs['demix_always'],
                         self.inputs['demix_if_needed'],
                         self.inputs['data_start_time'],

CEP/Pipeline/recipes/sip/master/gainoutliercorrection.py

@@ -36,11 +36,6 @@ class gainoutliercorrection(BaseRecipe, RemoteCommandRecipeMixIn):
             help="Full path to the `parmexportcal` executable, not settings this"
             " results in edit_parmdb behaviour"
         ),
-        'initscript' : ingredient.FileField(
-            '--initscript',
-            help="The full path to an (Bourne) shell script which will "
-                 "intialise the environment (i.e., ``lofarinit.sh``)"
-        ),
         'suffix': ingredient.StringField(
             '--suffix',
             help="Suffix of the table name of the instrument model",
@@ -119,7 +114,7 @@ class gainoutliercorrection(BaseRecipe, RemoteCommandRecipeMixIn):
                         infile,
                         outfile,
                         self.inputs['executable'],
-                        self.inputs['initscript'],
+                        self.environment,
                         self.inputs['sigma']
                      ]
                 )

CEP/Pipeline/recipes/sip/master/imager_awimager.py

@@ -33,11 +33,6 @@ class imager_awimager(BaseRecipe, RemoteCommandRecipeMixIn):
             '--executable',
             help="The full path to the  awimager executable"
         ),
-        'initscript': ingredient.FileField(
-            '--initscript',
-            help='''The full path to an (Bourne) shell script which will\
-             intialise the environment (ie, ``lofarinit.sh``)'''
-        ),
         'parset': ingredient.FileField(
             '-p', '--parset',
             help="The full path to a awimager configuration parset."
@@ -103,7 +98,7 @@ class imager_awimager(BaseRecipe, RemoteCommandRecipeMixIn):
 
             #construct and save the output name
             arguments = [self.inputs['executable'],
-                         self.inputs['initscript'],
+                         self.environment,
                          self.inputs['parset'],
                           self.inputs['working_directory'],
                          self.inputs['output_image'],

CEP/Pipeline/recipes/sip/master/imager_create_dbs.py

@@ -36,11 +36,6 @@ class imager_create_dbs(BaseRecipe, RemoteCommandRecipeMixIn):
         'working_directory': ingredient.StringField(
             '-w', '--working-directory',
             help="Working directory used on nodes. Results location"
-        ),
-         'initscript': ingredient.FileField(
-            '--initscript',
-            help='''The full path to an (Bourne) shell script which will\
-             initialise the environment (ie, ``lofarinit.sh``)'''
         ),
         'sourcedb_suffix': ingredient.StringField(
             '--sourcedb-suffix',
@@ -195,7 +190,7 @@ class imager_create_dbs(BaseRecipe, RemoteCommandRecipeMixIn):
                          self.inputs["parmdb_executable"],
                          slice_paths,
                          self.inputs["parmdb_suffix"],
-                         self.inputs["initscript"],
+                         self.environment,
                          self.inputs["working_directory"],
                          self.inputs["makesourcedb_path"],
                          self.inputs["source_list_path"]]

CEP/Pipeline/recipes/sip/master/imager_finalize.py

@@ -19,11 +19,6 @@ class imager_finalize(BaseRecipe, RemoteCommandRecipeMixIn):
     This recipe does not have positional commandline arguments 
     """
     inputs = {
-        'initscript': ingredient.FileField(
-            '--initscript',
-            help='''The full path to an (Bourne) shell script which will\
-             intialise the environment (ie, ``lofarinit.sh``)'''
-        ),
         'awimager_output_map': ingredient.FileField(
             '--awimager-output-mapfile',
             help="""Mapfile containing (host, path) pairs of created sky

CEP/Pipeline/recipes/sip/master/imager_prepare.py

@@ -48,11 +48,6 @@ class imager_prepare(BaseRecipe, RemoteCommandRecipeMixIn):
             '--ndppp-exec',
             help="The full path to the ndppp executable"
         ),
-        'initscript': ingredient.FileField(
-            '--initscript',
-            help='''The full path to a shell script which will\
-             intialise the environment (ie, ``lofarinit.sh``)'''
-        ),
         'parset': ingredient.FileField(
             '-p', '--parset',
             help="The full path to a prepare parset"
@@ -176,7 +171,7 @@ class imager_prepare(BaseRecipe, RemoteCommandRecipeMixIn):
             paths_to_image_mapfiles.append((host,
                                             inputs_for_image_mapfile_path))
 
-            arguments = [self.inputs['initscript'],
+            arguments = [self.environment,
                          self.inputs['parset'],
                          self.inputs['working_directory'],
                          self.inputs['processed_ms_dir'],

CEP/Pipeline/recipes/sip/master/imager_source_finding.py

@@ -24,10 +24,6 @@ class imager_source_finding(BaseRecipe, RemoteCommandRecipeMixIn):
     in.    
     """
     inputs = {
-        'initscript': ingredient.FileField(
-            '--initscript',
-            help="Initscript to source (ie, lofarinit.sh)"
-        ),
         'bdsm_parset_file_run1': ingredient.FileField(
             '--bdsm-parset-file-run1',
             help="Path to bdsm parameter set for the first sourcefinding run"
@@ -103,7 +99,7 @@ class imager_source_finding(BaseRecipe, RemoteCommandRecipeMixIn):
                              self.inputs["working_directory"],
                              "bdsm_output.img"),
                          self.inputs['sourcedb_target_path'],
-                         self.inputs['initscript'],
+                         self.environment,
                          self.inputs['working_directory'],
                          self.inputs['makesourcedb_path']
                         ]

CEP/Pipeline/recipes/sip/master/new_bbs.py

@@ -51,11 +51,6 @@ class new_bbs(BaseRecipe):
             dest="kernel_exec",
             help="BBS Kernel executable"
         ),
-        'initscript': ingredient.FileField(
-            '--initscript',
-            dest="initscript",
-            help="Initscript to source (ie, lofarinit.sh)"
-        ),
         'parset': ingredient.FileField(
             '-p', '--parset',
             dest="parset",
@@ -262,11 +257,6 @@ class new_bbs(BaseRecipe):
             #                                          with our own threads.
             # --------------------------------------------------------------
             command = "python %s" % (self.__file__.replace('master', 'nodes'))
-            env = {
-                "LOFARROOT": utilities.read_initscript(self.logger, self.inputs['initscript'])["LOFARROOT"],
-                "PYTHONPATH": self.config.get('deploy', 'engine_ppath'),
-                "LD_LIBRARY_PATH": self.config.get('deploy', 'engine_lpath')
-            }
             jobpool = {}
             bbs_kernels = []
             with job_server(self.logger, jobpool, self.error) as (jobhost, jobport):
@@ -277,7 +267,6 @@ class new_bbs(BaseRecipe):
                         host, command,
                         arguments=[
                             self.inputs['kernel_exec'],
-                            self.inputs['initscript'],
                             files,
                             self.inputs['db_key'],
                             self.inputs['db_name'],
@@ -288,9 +277,7 @@ class new_bbs(BaseRecipe):
                     bbs_kernels.append(
                         threading.Thread(
                             target=self._run_bbs_kernel,
-                            args=(host, command, env, job_id,
-                                jobhost, str(jobport)
-                            )
+                            args=(host, command, job_id, jobhost, str(jobport))
                         )
                     )
                 self.logger.info("Starting %d threads" % len(bbs_kernels))
@@ -314,7 +301,7 @@ class new_bbs(BaseRecipe):
         self.outputs['mapfile'] = self.inputs['data_mapfile']
         return 0
 
-    def _run_bbs_kernel(self, host, command, env, *arguments):
+    def _run_bbs_kernel(self, host, command, *arguments):
         """
         Run command with arguments on the specified host using ssh. Return its
         return code.
@@ -328,7 +315,7 @@ class new_bbs(BaseRecipe):
                 self.logger,
                 host,
                 command,
-                env,
+                self.environment,
                 arguments=arguments
             )
         except Exception, e:
@@ -346,7 +333,6 @@ class new_bbs(BaseRecipe):
         Run BBS Global Control and wait for it to finish. Return its return
         code.
         """
-        env = utilities.read_initscript(self.logger, self.inputs['initscript'])
         self.logger.info("Running BBS GlobalControl")
         working_dir = tempfile.mkdtemp()
         with CatchLog4CPlus(
@@ -364,7 +350,7 @@ class new_bbs(BaseRecipe):
                         ],
                         self.logger,
                         cwd=working_dir,
-                        env=env
+                        env=self.environment
                     )
                     # _monitor_process() needs a convenient kill() method.
                     bbs_control_process.kill = lambda : os.kill(bbs_control_process.pid, signal.SIGKILL)

CEP/Pipeline/recipes/sip/nodes/dppp.py

@@ -16,7 +16,6 @@ import sys
 from lofarpipe.support.pipelinelogging import CatchLog4CPlus
 from lofarpipe.support.pipelinelogging import log_time
 from lofarpipe.support.parset import patched_parset
-from lofarpipe.support.utilities import read_initscript
 from lofarpipe.support.utilities import create_directory
 from lofarpipe.support.utilities import catch_segfaults
 from lofarpipe.support.lofarnode import LOFARnodeTCP
@@ -26,7 +25,7 @@ class dppp(LOFARnodeTCP):
 
     def run(
         self, infile, outfile, parmdb, sourcedb,
-        parsetfile, executable, initscript, demix_always, demix_if_needed,
+        parsetfile, executable, environment, demix_always, demix_if_needed,
         start_time, end_time, nthreads, clobber
     ):
         # Debugging info
@@ -36,7 +35,7 @@ class dppp(LOFARnodeTCP):
         self.logger.debug("sourcedb        = %s" % sourcedb)
         self.logger.debug("parsetfile      = %s" % parsetfile)
         self.logger.debug("executable      = %s" % executable)
-        self.logger.debug("initscript      = %s" % initscript)
+        self.logger.debug("environment     = %s" % environment)
         self.logger.debug("demix_always    = %s" % demix_always)
         self.logger.debug("demix_if_needed = %s" % demix_if_needed)
         self.logger.debug("start_time      = %s" % start_time)
@@ -44,6 +43,8 @@ class dppp(LOFARnodeTCP):
         self.logger.debug("nthreads        = %s" % nthreads)
         self.logger.debug("clobber         = %s" % clobber)
 
+        self.environment.update(environment)
+        
         if not nthreads:
             nthreads = 1
         if not outfile:
@@ -92,9 +93,8 @@ class dppp(LOFARnodeTCP):
                 )
                 shutil.copytree(infile, tmpfile)
 
-            # Initialise environment. Limit number of threads used.
-            env = read_initscript(self.logger, initscript)
-            env['OMP_NUM_THREADS'] = str(nthreads)
+            # Limit number of threads used.
+            self.environment['OMP_NUM_THREADS'] = str(nthreads)
             self.logger.debug("Using %s threads for NDPPP" % nthreads)
 
             # Put arguments we need to pass to some private methods in a dict
@@ -129,7 +129,7 @@ class dppp(LOFARnodeTCP):
                     ) as logger:
                         # Catch NDPPP segfaults (a regular occurance), and retry
                         catch_segfaults(
-                            cmd, working_dir, env, logger, 
+                            cmd, working_dir, self.environment, logger, 
                             cleanup = lambda : shutil.rmtree(tmpfile, ignore_errors=True)
                         )
                         # Replace outfile with the updated working copy

CEP/Pipeline/recipes/sip/nodes/gainoutliercorrection.py

@@ -16,7 +16,7 @@ import errno
 from lofarpipe.support.lofarnode import LOFARnodeTCP
 from lofarpipe.support.pipelinelogging import CatchLog4CPlus
 from lofarpipe.support.pipelinelogging import log_time
-from lofarpipe.support.utilities import read_initscript, create_directory, delete_directory
+from lofarpipe.support.utilities import create_directory, delete_directory
 from lofarpipe.support.utilities import catch_segfaults
 from lofarpipe.support.lofarexceptions import PipelineRecipeFailed
 
@@ -24,8 +24,10 @@ from lofarpipe.recipes.helpers.WritableParmDB import WritableParmDB, list_statio
 from lofarpipe.recipes.helpers.ComplexArray import ComplexArray, RealImagArray, AmplPhaseArray
 
 class GainOutlierCorrection(LOFARnodeTCP):
-    def run(self, infile, outfile, executable, initscript, sigma):
+    def run(self, infile, outfile, executable, environment, sigma):
 
+        self.environment.update(environment)
+        
         # Time execution of this job
         with log_time(self.logger):
             if os.path.exists(infile):
@@ -49,9 +51,6 @@ class GainOutlierCorrection(LOFARnodeTCP):
             self.logger.error("Executable %s not found" % executable)
             return 1
 
-        # Initialize environment
-        env = read_initscript(self.logger, initscript)
-
         try:
             temp_dir = tempfile.mkdtemp()
             with CatchLog4CPlus(
@@ -62,7 +61,7 @@ class GainOutlierCorrection(LOFARnodeTCP):
                 catch_segfaults(
                     [executable, '-in', infile, '-out', outfile],
                     temp_dir,
-                    env,
+                    self.environment,
                     logger
                 )
         except Exception, excp:

CEP/Pipeline/recipes/sip/nodes/imager_awimager.py

@@ -24,7 +24,6 @@ from lofarpipe.support.pipelinelogging import CatchLog4CPlus
 from lofarpipe.support.pipelinelogging import log_time
 from lofarpipe.support.utilities import patch_parset
 from lofarpipe.support.utilities import get_parset
-from lofarpipe.support.utilities import read_initscript
 from lofarpipe.support.utilities import catch_segfaults
 from lofarpipe.support.lofarexceptions import PipelineException
 import pyrap.tables as pt                   #@UnresolvedImport
@@ -36,27 +35,12 @@ import lofar.parmdb                          #@UnresolvedImport
 import numpy as np
 
 class imager_awimager(LOFARnodeTCP):
-    """
-    The imager_awimager creates from a measurement set and a sourcedb an image.
-    The sources in the database are use to create a mask which can be used in
-    the awimager to improve the clean function.
-    
-    1. Calculate / retrieve a number of parameters from the measurement set
-       Cell_size, npix, wmax and W projection planes
-    2. Create the needed directorys for the output image
-    3. Create a mask (casa image),
-    4. extend the parset with calculate parameters and the mask location
-    5. Run the Awimager with the extended parset
-    6. Return the created image to signal a successfull run
-    
-    | **Member functions:**
-    """
-    def run(self, executable, init_script, parset, working_directory,
+    def run(self, executable, environment, parset, working_directory,
             output_image, concatenated_measurement_set, sourcedb_path,
              mask_patch_size):
         """       
         :param executable: Path to awimager executable
-        :param init_script: initscript for catch_segfaults (executable runner)
+        :param environment: environment for catch_segfaults (executable runner)
         :param parset: parameters for the awimager, 
         :param working_directory: directory the place temporary files
         :param output_image: location and filesname to story the output images
@@ -71,6 +55,8 @@ class imager_awimager(LOFARnodeTCP):
         """
         self.logger.info("Start imager_awimager node run:")
         log4_cplus_name = "imager_awimager"
+        self.environment.update(environment)
+        
         with log_time(self.logger):
             # ****************************************************************
             # 1. Calculate awimager parameters that depend on measurement set
@@ -89,7 +75,7 @@ class imager_awimager(LOFARnodeTCP):
             # ****************************************************************
             # 3. Create the mask
             mask_file_path = self._create_mask(npix, cell_size, output_image,
-                         concatenated_measurement_set, init_script, executable,
+                         concatenated_measurement_set, executable,
                          working_directory, log4_cplus_name, sourcedb_path,
                           mask_patch_size, image_path_head)
 
@@ -120,13 +106,12 @@ class imager_awimager(LOFARnodeTCP):
             # 5. Run the awimager with the updated parameterset
             cmd = [executable, calculated_parset_path]
             try:
-                environment = read_initscript(self.logger, init_script)
                 with CatchLog4CPlus(working_directory,
                         self.logger.name + "." +
                         os.path.basename(log4_cplus_name),
                         os.path.basename(executable)
                 ) as logger:
-                    catch_segfaults(cmd, working_directory, environment,
+                    catch_segfaults(cmd, working_directory, self.environment,
                                             logger)
 
             # Thrown by catch_segfault
@@ -308,7 +293,7 @@ class imager_awimager(LOFARnodeTCP):
         return fov, station_diameter
 
     def _create_mask(self, npix, cell_size, output_image,
-                     concatenated_measurement_set, init_script, executable,
+                     concatenated_measurement_set, executable,
                      working_directory, log4_cplus_name, sourcedb_path,
                      mask_patch_size, image_path_directory):
         """
@@ -355,12 +340,11 @@ class imager_awimager(LOFARnodeTCP):
         cmd = [executable, mask_parset_path]
         self.logger.info(" ".join(cmd))
         try:
-            environment = read_initscript(self.logger, init_script)
             with CatchLog4CPlus(working_directory,
                     self.logger.name + "." + os.path.basename(log4_cplus_name),
                     os.path.basename(executable)
             ) as logger:
-                catch_segfaults(cmd, working_directory, environment,
+                catch_segfaults(cmd, working_directory, self.environment,
                                         logger)
         # Thrown by catch_segfault
         except CalledProcessError, exception:

CEP/Pipeline/recipes/sip/nodes/imager_create_dbs.py

@@ -15,7 +15,6 @@ import os
 from lofarpipe.support.lofarnode import LOFARnodeTCP
 from lofarpipe.support.pipelinelogging import log_process_output
 from lofarpipe.support.pipelinelogging import CatchLog4CPlus
-from lofarpipe.support.utilities import read_initscript
 from lofarpipe.support.utilities import catch_segfaults
 
 import monetdb.sql as db
@@ -58,10 +57,11 @@ class imager_create_dbs(LOFARnodeTCP):
     def run(self, concatenated_measurement_set, sourcedb_target_path,
             monet_db_hostname, monet_db_port, monet_db_name, monet_db_user,
             monet_db_password, assoc_theta, parmdb_executable, slice_paths,
-            parmdb_suffix, init_script, working_directory, makesourcedb_path,
+            parmdb_suffix, environment, working_directory, makesourcedb_path,
             source_list_path_extern):
 
         self.logger.info("Starting imager_create_dbs Node")
+        self.environment.update(environment)
 
         #*******************************************************************
         # 1. get a sourcelist: from gsm or from file
@@ -73,8 +73,8 @@ class imager_create_dbs(LOFARnodeTCP):
         #*******************************************************************
         # 2convert it to a sourcedb (casa table)
         if self._create_source_db(source_list, sourcedb_target_path,
-                                  init_script, working_directory,
-                                  makesourcedb_path, append) == None:
+                                  working_directory, makesourcedb_path, 
+                                  append) == None:
             self.logger.error("failed creating sourcedb")
             return 1
 
@@ -122,7 +122,7 @@ class imager_create_dbs(LOFARnodeTCP):
 
         return source_list, append
 
-    def _create_source_db(self, source_list, sourcedb_target_path, init_script,
+    def _create_source_db(self, source_list, sourcedb_target_path,
                           working_directory, executable, append=False):
         """
         _create_source_db consumes a sourcelist text file and produces a 
@@ -146,13 +146,12 @@ class imager_create_dbs(LOFARnodeTCP):
                               # db
 
         try:
-            environment = read_initscript(self.logger, init_script)
             with CatchLog4CPlus(working_directory,
                  self.logger.name + "." + os.path.basename("makesourcedb"),
                  os.path.basename(executable)
             ) as logger:
-                catch_segfaults(cmd, working_directory, environment,
-                                            logger, cleanup=None)
+                catch_segfaults(cmd, working_directory, self.environment,
+                                            logger, cleanup = None)
 
         except subprocess.CalledProcessError, called_proc_error:
             self.logger.error("Execution of external failed:")

CEP/Pipeline/recipes/sip/nodes/imager_prepare.py

@@ -13,7 +13,6 @@ import subprocess
 from lofarpipe.support.pipelinelogging import CatchLog4CPlus
 from lofarpipe.support.pipelinelogging import log_time
 from lofarpipe.support.utilities import patch_parset
-from lofarpipe.support.utilities import read_initscript
 from lofarpipe.support.utilities import catch_segfaults
 from lofarpipe.support.lofarnode import  LOFARnodeTCP
 from lofarpipe.support.utilities import create_directory
@@ -41,7 +40,7 @@ class imager_prepare(LOFARnodeTCP):
  
     **Members:** 
     """
-    def run(self, init_script, parset, working_dir, processed_ms_dir,
+    def run(self, environment, parset, working_dir, processed_ms_dir,
              ndppp_executable, output_measurement_set,
             time_slices_per_image, subbands_per_group, raw_ms_mapfile,
             asciistat_executable, statplot_executable, msselect_executable,
@@ -49,6 +48,7 @@ class imager_prepare(LOFARnodeTCP):
         """
         Entry point for the node recipe
         """
+        self.environment.update(environment)
         with log_time(self.logger):
             input_map = load_data_map(raw_ms_mapfile)
 
@@ -80,9 +80,9 @@ class imager_prepare(LOFARnodeTCP):
             #******************************************************************
             # 2. run dppp: collect frequencies into larger group
             time_slices = \
-                self._run_dppp(working_dir, time_slice_dir,
-                     time_slices_per_image, input_map, subbands_per_group,
-                     processed_ms_dir, parset, ndppp_executable, init_script)
+                self._run_dppp(working_dir, time_slice_dir, 
+                    time_slices_per_image, input_map, subbands_per_group, 
+                    processed_ms_dir, parset, ndppp_executable)
 
             self.logger.debug("Produced time slices: {0}".format(time_slices))
             #***********************************************************
@@ -198,7 +198,7 @@ class imager_prepare(LOFARnodeTCP):
 
     def _run_dppp(self, working_dir, time_slice_dir_path, slices_per_image,
                   input_map, subbands_per_image, collected_ms_dir_name, parset,
-                  ndppp, init_script):
+                  ndppp):
         """
         Run NDPPP:  
         Create dir for grouped measurements, assure clean workspace
@@ -254,10 +254,8 @@ class imager_prepare(LOFARnodeTCP):
             cmd = [ndppp, nddd_parset_path]
 
             try:
-                environment = read_initscript(self.logger, init_script)
-
                 # Actual dppp call to externals (allows mucking)
-                self._dppp_call(working_dir, ndppp, cmd, environment)
+                self._dppp_call(working_dir, ndppp, cmd, self.environment)
 
             except subprocess.CalledProcessError, exception:
                 self.logger.error(str(exception))

CEP/Pipeline/recipes/sip/nodes/imager_source_finding.py

@@ -7,7 +7,6 @@ from lofar.parameterset import parameterset
 from lofarpipe.support.lofarnode import LOFARnodeTCP
 import lofar.bdsm as bdsm#@UnresolvedImport
 
-from lofarpipe.support.utilities import read_initscript
 from lofarpipe.support.pipelinelogging import CatchLog4CPlus
 from lofarpipe.support.utilities import catch_segfaults
 
@@ -35,7 +34,7 @@ class imager_source_finding(LOFARnodeTCP):
     """
     def run(self, input_image, bdsm_parameter_run1_path,
             bdsm_parameter_run2x_path, catalog_output_path, image_output_path,
-            sourcedb_target_path, init_script, working_directory,
+            sourcedb_target_path, environment, working_directory,
             create_sourcdb_exec):
         """
         :param input_image: image to look for sources in
@@ -47,7 +46,7 @@ class imager_source_finding(LOFARnodeTCP):
                substracted
         :param sourcedb_target_path: Path to store the sourcedb created from 
             containing all the found sources
-        :param init_script: Initscript for runwithlog4cplus
+        :param environment: environment for runwithlog4cplus
         :param working_directory: Working dir
         :param create_sourcdb_exec: Path to create sourcedb executable 
         
@@ -55,6 +54,7 @@ class imager_source_finding(LOFARnodeTCP):
         
         """
         self.logger.info("Starting imager_source_finding")
+        self.environment.update(environment)
         # default frequency is None (read from image), save for later cycles.
         # output of pybdsm forgets freq of source image
         frequency = None
@@ -149,7 +149,7 @@ class imager_source_finding(LOFARnodeTCP):
         # *********************************************************************
         # 6. Convert sourcelist to sourcedb
         self._create_source_db(catalog_output_path, sourcedb_target_path,
-                init_script, working_directory, create_sourcdb_exec, False)
+            working_directory, create_sourcdb_exec, False)
         # Assign the outputs
         self.outputs["catalog_output_path"] = catalog_output_path
         self.outputs["source_db"] = sourcedb_target_path
@@ -213,7 +213,7 @@ class imager_source_finding(LOFARnodeTCP):
                                                 catalog_output_path))
 
 
-    def _create_source_db(self, source_list, sourcedb_target_path, init_script,
+    def _create_source_db(self, source_list, sourcedb_target_path,
                           working_directory, create_sourcdb_exec, append=False):
         """
         Convert a sourcelist to a sourcedb:
@@ -238,13 +238,12 @@ class imager_source_finding(LOFARnodeTCP):
         self.logger.info(' '.join(cmd))
 
         try:
-            environment = read_initscript(self.logger, init_script)
             with CatchLog4CPlus(working_directory,
                  self.logger.name + "." + os.path.basename("makesourcedb"),
                  os.path.basename(create_sourcdb_exec)
             ) as logger:
-                catch_segfaults(cmd, working_directory, environment,
-                                            logger, cleanup=None)
+                catch_segfaults(cmd, working_directory, self.environment,
+                                            logger, cleanup = None)
 
         except Exception, exception:
             self.logger.error("Execution of external failed:")

CEP/Pipeline/recipes/sip/nodes/new_bbs.py

@@ -14,7 +14,6 @@ import shutil
 
 from lofarpipe.support.pipelinelogging import CatchLog4CPlus
 from lofarpipe.support.lofarnode import LOFARnodeTCP
-from lofarpipe.support.utilities import read_initscript
 from lofarpipe.support.utilities import get_mountpoint
 from lofarpipe.support.utilities import log_time
 from lofarpipe.support.pipelinelogging import log_process_output
@@ -25,17 +24,12 @@ from lofar.parameterset import parameterset
 class new_bbs(LOFARnodeTCP):
     #                      Handles running a single BBS kernel on a compute node
     # --------------------------------------------------------------------------
-    def run(
-        self, executable, initscript, infiles,
-        db_key, db_name, db_user, db_host
-    ):
+    def run(self, executable, infiles, db_key, db_name, db_user, db_host):
         # executable : path to KernelControl executable
-        # initscript : path to lofarinit.sh
         # infiles    : tuple of MS, instrument- and sky-model files
         # db_*       : database connection parameters
         # ----------------------------------------------------------------------
         self.logger.debug("executable = %s" % executable)
-        self.logger.debug("initscript = %s" % initscript)
         self.logger.debug("infiles = %s" % str(infiles))
         self.logger.debug("db_key = %s" % db_key)
         self.logger.debug("db_name = %s" % db_name)
@@ -79,7 +73,6 @@ class new_bbs(LOFARnodeTCP):
             #               Catch & log output from the kernel logger and stdout
             # ------------------------------------------------------------------
             working_dir = mkdtemp()
-            env = read_initscript(self.logger, initscript)
             try:
                 self.logger.info("******** {0}".format(open(parset_file).read()))
                 cmd = [executable, parset_file, "0"]

CEP/Pipeline/recipes/sip/tasks.cfg.in

@@ -9,7 +9,6 @@ mapfile = %(runtime_directory)s/jobs/%(job_name)s/mapfiles/data.mapfile
 [ndppp]
 recipe = dppp
 executable = %(lofarroot)s/bin/NDPPP
-initscript = %(lofarroot)s/lofarinit.sh
 dry_run = False
 mapfile = %(runtime_directory)s/jobs/%(job_name)s/mapfiles/dppp.mapfile
 parset = %(runtime_directory)s/jobs/%(job_name)s/parsets/NDPPP.parset
@@ -21,7 +20,6 @@ clobber = False
 
 [bbs]
 recipe = bbs
-initscript = %(lofarroot)s/lofarinit.sh
 control_exec = %(lofarroot)s/bin/GlobalControl
 kernel_exec = %(lofarroot)s/bin/KernelControl
 parset = %(runtime_directory)s/jobs/%(job_name)s/parsets/bbs.parset
@@ -72,7 +70,6 @@ recipe = flag_baseline
 
 [demixing]
 recipe = demixing
-initscript = %(lofarroot)s/lofarinit.sh
 demix_parset_dir = %(lofarroot)s/share/pipeline/demixing
 db_host = ldb001
 skymodel = %(lofarroot)s/share/pipeline/skymodels/Ateam_LBA_CC.skymodel
@@ -84,7 +81,6 @@ nproc=1
 recipe = new_bbs
 control_exec = %(lofarroot)s/bin/GlobalControl
 kernel_exec = %(lofarroot)s/bin/KernelControl
-initscript = %(lofarroot)s/lofarinit.sh
 parset = %(runtime_directory)s/jobs/%(job_name)s/parsets/BBS.parset
 gvds = %(runtime_directory)s/jobs/%(job_name)s/vds/%(job_name)s.gvds
 db_key = %(job_name)s
@@ -98,13 +94,11 @@ data_mapfile = %(runtime_directory)s/jobs/%(job_name)s/mapfiles/bbs.mapfile
 [gainoutliercorrection]
 recipe = gainoutliercorrection
 executable = %(lofarroot)s/bin/parmexportcal
-initscript = %(lofarroot)s/lofarinit.sh
 mapfile = %(runtime_directory)s/jobs/%(job_name)s/mapfiles/instrument.mapfile
 
 [parmexportcal]
 recipe = parmexportcal
 executable = %(lofarroot)s/bin/parmexportcal
-initscript = %(lofarroot)s/lofarinit.sh
 mapfile = %(runtime_directory)s/jobs/%(job_name)s/mapfiles/instrument.mapfile
 
 [rficonsole]
@@ -116,7 +110,6 @@ recipe = get_metadata
 
 [imager_prepare]
 recipe = imager_prepare
-initscript = %(lofarroot)s/lofarinit.sh
 ndppp_exec = %(lofarroot)s/bin/NDPPP
 asciistat_executable = %(lofarroot)s/bin/asciistats.py
 statplot_executable = %(lofarroot)s/bin/statsplot.py
@@ -125,12 +118,10 @@ rficonsole_executable = %(lofarroot)s/bin/rficonsole
 
 [imager_awimager]
 recipe = imager_awimager
-initscript = %(lofarroot)s/lofarinit.sh
 executable = %(lofarroot)s/bin/awimager
 
 [imager_create_dbs]
 recipe = imager_create_dbs
-initscript = %(lofarroot)s/lofarinit.sh
 parmdb_executable = %(lofarroot)s/bin/parmdbm
 makesourcedb_path = %(lofarroot)s/bin/makesourcedb
 
@@ -140,12 +131,10 @@ bbs_executable = %(lofarroot)s/bin/bbs-reducer
   
 [imager_source_finding]
 recipe = imager_source_finding
-initscript = %(lofarroot)s/lofarinit.sh
 makesourcedb_path = %(lofarroot)s/bin/makesourcedb
 
 [imager_finalize]
 recipe = imager_finalize
-initscript = %(lofarroot)s/lofarinit.sh
 fillrootimagegroup_exec = %(lofarroot)s/bin/fillRootImageGroup
 
 [copier]