Bug 1251: Final update before release of version LOFAR20.1

.gitattributes

@@ -184,6 +184,7 @@ CEP/BB/MWImager/test/tmwimager.in_ms1.vds -text
 CEP/BB/MWImager/test/tmwimager.in_ms2.vds -text
 CEP/BB/MWImager/test/tmwimager.in_vd -text
 CEP/BB/ParmDB/test/tmakesourcedb.in_2 -text
+CEP/BB/Pipeline/library/makevds.bash -text
 CEP/BB/SourceDB/bootstrap -text
 /CMakeLists.txt -text
 Firmware/tools/bootstrap -text

CEP/BB/Pipeline/library/DPPP.py

@@ -37,12 +37,13 @@ class DPPP(WSRTrecipe):
     compress UV-data in frequency and time."""
     def __init__(self):
         WSRTrecipe.__init__(self)
-        self.inputs['parset-file']    = 'dppp.parset'
-        self.inputs['cluster-name']   = 'lioff'
-        self.inputs['observation']    = ''
-        self.inputs['output-dir']     = None
-        self.inputs['vds-dir']        = None
-        self.inputs['dryrun']         = False
+        self.inputs['parset-file']  = 'dppp.parset'
+        self.inputs['cluster-name'] = 'lioff'
+        self.inputs['observation']  = ''
+        self.inputs['output-dir']   = None
+        self.inputs['vds-dir']      = None
+        self.inputs['logfile']      = 'dppp.log'
+        self.inputs['dryrun']       = False
         self.helptext = """
         This function runs the distributed DPPP
         Usage: DPPP [OPTION...]
@@ -57,6 +58,8 @@ class DPPP(WSRTrecipe):
                              (default: '/data/${USER}/<observation>')
         --vds-dir            directory where the VDS-files reside;
                              (default: '/users/${USER}/data/<observation>')
+        --logfile            root name of logfile of each subprocess
+                             (default 'dppp.log')
         --dryrun             do a dry run
                              (default: no)
         """
@@ -88,7 +91,7 @@ class DPPP(WSRTrecipe):
         opts += ['-nomasterhost']
         opts += ['-dsn', dataset]
         opts += ['-cdn', clusterdesc]
-        opts += ['-logfile', 'dppp.log']
+        opts += ['-logfile', self.inputs['logfile']]
         opts += ['-dry' if self.inputs['dryrun'] else '-nodry']
         # program started by 'startdistproc'
         opts += [os.path.join(sysconfig.lofar_root(),
@@ -104,7 +107,7 @@ class DPPP(WSRTrecipe):
             self.print_error('startdistproc failed!')
             return 1
 
-        # Combine the VDS-files generated by dppp_node.py into one GDS file
+        # Combine the VDS-files generated by dppp_node.py into one GDS-file.
         self.print_message('Generating gds file from vds files')
         opts = [dataset]
         opts += glob.glob(os.path.join(vds_dir, '*.vds'))

CEP/BB/Pipeline/library/MWImager.py

@@ -54,10 +54,10 @@ class MWImager(WSRTrecipe):
                              (no default)
         --output-dir         directory where images will be stored
                              (default: same directory as MS)
-        --logfile            root name of logfile of each subprocess
-                             (default 'mwimager.log')
         --vds-dir            directory where the VDS-files reside;
                              (default: '/users/${USER}/data/<observation>')
+        --logfile            root name of logfile of each subprocess
+                             (default 'mwimager.log')
         --dryrun             do a dry run
                              (default: no)
         """

CEP/BB/Pipeline/library/MakeVDS.py

@@ -39,22 +39,25 @@ class MakeVDSError(Exception):
     
 
 class MakeVDS(WSRTrecipe):
-    """Generate a VDS files for all MS-files that belong to the specified
-    observation.
-    The argument `cluster-name' is used to locate a cluster description file,
-    which (among other information) contains a list of mount points to use
-    when searching for MS-files. Use the argument `directory' if the MS-files
-    are located in a directory below the mount points.
-    After the VDS files have been generated a GDS file (which is like a
-    concatenation of all VDS files) is generated and stored in the current
-    directory."""
+    """
+    Generate VDS files for all MS-files that belong to the specified
+    observation. The argument `cluster-name' is used to locate a cluster
+    description file, which (among other information) contains a list of mount
+    points to use when searching for MS-files. Use the argument `input-dir' if
+    the MS-files are located in a directory below the mount points. The
+    location where the VDS files will be stored can be specified using the
+    argument `vds-dir'; `/users/${USER}/data/<observation>' by default. When
+    the VDS files have been generated, a GDS file (which is like a
+    concatenation of all VDS files) is generated in the same directory.
+    """
     
     def __init__(self):
         WSRTrecipe.__init__(self)
-        self.inputs['cluster-name'] = 'lioff'
-        self.inputs['observation']  = ''
-        self.inputs['directory']    = None
-        self.inputs['vds-dir']      = None
+        self.inputs['cluster-name']    = 'lioff'
+        self.inputs['observation']     = ''
+        self.inputs['input-dir']       = None
+        self.inputs['vds-dir']         = None
+        self.inputs['dryrun']          = False
         self.helptext = """
         This function generates vds files for the MS files that comprise the
         given observation.
@@ -64,12 +67,14 @@ class MakeVDS(WSRTrecipe):
                              (default: 'lioff')
         --observation        name of the observation (e.g. L2007_03463)
                              (no default)
-        --directory          directory, relative to the mount point, where
+        --input-dir          directory, relative to the mount point, where
                              the input data is stored (e.g. /lifs001/pipeline);
                              if None, use directory <mount-point>/<observation>
         --vds-dir            directory for the output VDS-files;
                              if None, use directory
                              '/users/${USER}/data/<observation>'
+        --dryrun             do a dry run
+                             (default: no)
         """
     
     ## Code to generate results ----------------------------------------
@@ -80,7 +85,7 @@ class MakeVDS(WSRTrecipe):
         clusterdesc = sysconfig.clusterdesc_file(self.inputs['cluster-name'])
         ms_files = Observation(self.inputs['cluster-name'],
                                self.inputs['observation'],
-                               self.inputs['directory']).ms_files()
+                               self.inputs['input-dir']).ms_files()
         vds_dir = self.inputs['vds-dir'] \
                   if self.inputs['vds-dir'] \
                   else os.path.join('/users', os.environ['USER'], 'data',
@@ -98,22 +103,26 @@ class MakeVDS(WSRTrecipe):
         if not os.path.exists(vds_dir):
             os.makedirs(vds_dir)
             self.print_debug('Created directory ' + vds_dir)
-            
+
+        # Generate the VDS-files.
         fail = 0
         for (ms, vds) in zip(ms_files, vds_files):
             self.print_message('Processing file ' + ms)
-            fail += self.cook_system('makevds', [clusterdesc, ms, vds])
+            if not self.inputs['dryrun']:
+                fail += self.cook_system('makevds', [clusterdesc, ms, vds])
 
         if fail:
             self.print_error(str(fail) + ' makevds process(es) failed!')
             return 1
 
+        # Combine the VDS-files into one GDS-file.
         self.print_message('Generating gds file from vds files')
         opts = [os.path.join(vds_dir, self.inputs['observation'] + '.gds')]
         opts.extend(vds_files)
-        if self.cook_system('combinevds', opts):
-            self.print_error('combinevds failed!')
-            return 1
+        if not self.inputs['dryrun']:
+            if self.cook_system('combinevds', opts):
+                self.print_error('combinevds failed!')
+                return 1
 
         return 0
 

CEP/BB/Pipeline/library/Makefile.am

+# This will install .py files into $(bindir) with their execute bit set.
+bin_SCRIPTS	= dppp_node.py \
+		  DPPP.py \
+		  MakeVDS.py \
+		  MWImager.py \
+		  StandardImagingPipeline.py
+
+# This will install .py, .pyc, and .pyo files into the site-packages directory.
+pypkgdir	= $(pythondir)/lofar/pipeline
+pypkg_PYTHON	= __init__.py \
+		  Observation.py \
+		  sysconfig.py
+
+include $(top_srcdir)/Makefile.common

CEP/BB/Pipeline/library/StandardImagingPipeline.py

@@ -24,12 +24,10 @@
 
 """Script to run the Standard Imaging Pipeline"""
 
-from WSRTrecipe import WSRTrecipe
-from ingredient import WSRTingredient
-#from dppp import DPPP
-#from mwimager import MWImager
-from parset import Parset
-import sysconfig
+from lofar.pipeline.WSRTrecipe import WSRTrecipe
+from lofar.pipeline.ingredient import WSRTingredient
+from lofar.pipeline.parset import Parset
+from lofar.pipeline import sysconfig
 
 import os
 import sys
@@ -45,7 +43,7 @@ class StandardImagingPipeline(WSRTrecipe):
         self.inputs['make-vds-files']  = True
         self.inputs['input-dir']       = None
         self.inputs['output-dir']      = None
-        self.inputs['vds-output-dir']  = None
+        self.inputs['vds-dir']         = None
         self.inputs['dryrun']          = False
         self.helptext = """
         This is the recipe for the LOFAR standard imagaging pipeline.
@@ -60,14 +58,14 @@ class StandardImagingPipeline(WSRTrecipe):
         --observation        name of the observation to be processed
                              (no default)
         --make-vds-files     create VDS files
-                             (default: no)
+                             (default: yes)
         --input-dir          directory for the input MS-files;
                              only needed when VDS files are missing
                              (optional; no default)
         --output-dir         directory for the output MS-files;
                              only needed when VDS files are missing
                              (optional; default: '/data/${USER}/<obs>')
-        --vds-output-dir     directory for the output VDS-files;
+        --vds-dir            directory for the output VDS-files;
                              only needed until IDPPP creates these files
                              (optional; default: '/users/${USER}/data/<obs>)
         --dryrun             do a dry run
@@ -85,17 +83,18 @@ class StandardImagingPipeline(WSRTrecipe):
             self.inputs['output-dir']  = '/data/' + os.environ['USER'] + \
                                          '/' + obs
 
-        if self.inputs['vds-output-dir'] is None:
-            self.inputs['vds-output-dir'] = '/users/' + os.environ['USER'] + \
-                                            '/data/' + obs
+        if self.inputs['vds-dir'] is None:
+            self.inputs['vds-dir'] = '/users/' + os.environ['USER'] + \
+                                     '/data/' + obs
             
         # Create VDS files for the MS-files in the observation, if requested.
         if self.inputs['make-vds-files']:
             inputs = WSRTingredient()
             outputs = WSRTingredient()
             inputs['cluster-name'] = self.inputs['cluster-name']
-            inputs['observation']  = self.inputs['observation']
-            inputs['directory']    = self.inputs['input-dir']
+            inputs['observation'] = self.inputs['observation']
+            inputs['input-dir'] = self.inputs['input-dir']
+            inputs['dryrun'] = self.inputs['dryrun']
             sts = self.cook_recipe('MakeVDS', inputs, outputs)
             if sts:
                 print "MakeVDS returned with status", sts
@@ -106,7 +105,7 @@ class StandardImagingPipeline(WSRTrecipe):
         inputs['cluster-name'] = self.inputs['cluster-name']
         inputs['observation'] = self.inputs['observation']
         inputs['output-dir'] = self.inputs['output-dir']
-        inputs['vds-output-dir'] = self.inputs['vds-output-dir']
+        inputs['vds-dir'] = self.inputs['vds-dir']
         inputs['dryrun'] = self.inputs['dryrun']
         outputs = WSRTingredient()
         sts = self.cook_recipe('DPPP', inputs, outputs)
@@ -122,6 +121,9 @@ class StandardImagingPipeline(WSRTrecipe):
         inputs['cluster-name'] = self.inputs['cluster-name']
         inputs['observation'] = self.inputs['observation']
         inputs['output-dir'] = self.inputs['output-dir']
+        inputs['vds-dir'] = self.inputs['vds-dir']
+        inputs['dryrun'] = self.inputs['dryrun']
+        
         outputs = WSRTingredient()
         sts = self.cook_recipe('MWImager', inputs, outputs)
         if sts:

CEP/BB/Pipeline/library/__init__.py

+#!/usr/bin/env python
+#
+#  __init__.py: module initialization file
+#
+#  Copyright (C) 2002-2008
+#  ASTRON (Netherlands Foundation for Research in Astronomy)
+#  P.O.Box 2, 7990 AA Dwingeloo, The Netherlands, seg@astron.nl
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+#  $Id$
+
+"""Module initialization file"""

CEP/BB/Pipeline/library/dppp_node.py

 #!/usr/bin/env python
+#
+#  dppp_node.py: script to run DPPP on a single node
+#
+#  Copyright (C) 2002-2008
+#  ASTRON (Netherlands Foundation for Research in Astronomy)
+#  P.O.Box 2, 7990 AA Dwingeloo, The Netherlands, seg@astron.nl
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+#  $Id$
+
+"""Script to run DPPP on a single node"""
 
 import os, sys, logging, socket
 
@@ -64,10 +88,14 @@ if msin == msout:
 vds = vdsdir + '/' + os.path.basename(msout) + '.vds'
 logging.info('Output VDS: %s', vds)
 
-# Create working dir if it doesn't exist and change to it.
+# Create working dir if it doesn't exist and change to it. Working directory
+# `wd' may already have been created by another instance of dppp_node.py. So
+# wrap os.makedirs() in a try-except block.
 if wd:
-    if not os.path.exists(wd):
+    try:
         os.makedirs(wd)
+    except OSError, e:
+        logging.info(e)
     os.chdir(wd)
 wd = os.getcwd()
 logging.info('Current working directory: %s', wd)

CEP/BB/Pipeline/library/export_image.g

+#!/usr/bin/env glish
+
+include 'viewer.g'
+
+dp:=dv.newdisplaypanel();
+
+for(i in 3:len(argv))
+{
+    path:=split(argv[i],'/');
+    imfile:=path[len(path)];
+
+    print paste(argv[i], "->", spaste(imfile, ".ps"));
+
+    dd:=dv.loaddata(argv[i],'raster');
+    dp.register(dd);
+
+    dd.setoptions([titletext=[value=imfile]]);
+    dd.setoptions([axislabelswitch=[value=T],xgridtype=[value='Full grid'],ygridtype=[value='Full grid']]);
+    #dd.setoptions([colormode=[value=colormap]]); <-- seems to screw things up, cannot do 'Adjust' in the GUI after this.
+    dd.setoptions([colormap=[value='Hot Metal 2']]);
+    dd.setoptions([wedge=[value=T]]);
+
+    man:=dp.canvasprintmanager();
+    man.writeps(spaste(imfile, ".ps"), 'a4', F, 300);
+
+    dp.unregister(dd);
+}
+
+dp.done();
+exit;

CEP/BB/Pipeline/library/makevds.bash

+#!/bin/bash
+
+source /users/loose/mwimager/LOFAR/installed/gnu_opt/lofarinit.sh
+
+for (( i=0; i<3; i++ ))
+do
+  for (( j=0; j<12; j++ ))
+  do
+    ms=$(printf "/lifs%03d/pipeline/L2007_03463_SB%d.MS" \
+       $(expr $j + 1) $(expr 12 "*" $i + $j))
+    echo "Processing  $(basename $ms)"
+    makevds $HOME/lioff.clusterdesc $ms $(basename $ms).vds
+  done
+done
+