Task #3464: Added option to setupsourcedb to generate the skydb as one big blob

CEP/Pipeline/recipes/sip/bin/msss_calibrator_pipeline.py

@@ -196,7 +196,8 @@ class msss_calibrator_pipeline(control):
                 'share', 'pipeline', 'skymodels', 'Ateam_LBA_CC.skymodel'
             ),
             mapfile=os.path.join(mapfile_dir, 'dppp.sourcedb.mapfile'),
-            suffix='.dppp.sourcedb'
+            suffix='.dppp.sourcedb',
+            type='blob'
         )['mapfile']
 
         # Create a parameter-subset for DPPP and write it to file.

CEP/Pipeline/recipes/sip/bin/msss_target_pipeline.py

@@ -271,7 +271,8 @@ class msss_target_pipeline(control):
                 'share', 'pipeline', 'skymodels', 'Ateam_LBA_CC.skymodel'
             ),
             mapfile=os.path.join(mapfile_dir, 'dppp.sourcedb.mapfile'),
-            suffix='.dppp.sourcedb'
+            suffix='.dppp.sourcedb',
+            type='blob'
         )['mapfile']
 
         # Create a parameter-subset for DPPP and write it to file.

CEP/Pipeline/recipes/sip/master/setupsourcedb.py

@@ -37,6 +37,11 @@ class setupsourcedb(BaseRecipe, RemoteCommandRecipeMixIn):
             help="Input sky catalogue",
             default=''
         ),
+        'type': ingredient.StringField(
+            '--type',
+            help="Output type (casa or blob)",
+            default="casa"
+        ),
         'mapfile': ingredient.StringField(
             '--mapfile',
             help="Full path of mapfile to produce; it will contain "
@@ -109,7 +114,8 @@ class setupsourcedb(BaseRecipe, RemoteCommandRecipeMixIn):
                     arguments=[
                         self.inputs['executable'],
                         self.inputs['skymodel'],
-                        outfile
+                        outfile,
+                        self.inputs['type']
                     ]
                 )
             )

CEP/Pipeline/recipes/sip/nodes/setupsourcedb.py

@@ -5,7 +5,7 @@
 #                                                                loose@astron.nl
 # ------------------------------------------------------------------------------
 
-from subprocess import Popen, CalledProcessError, PIPE, STDOUT
+from subprocess import CalledProcessError
 import errno
 import os
 import tempfile
@@ -19,7 +19,7 @@ from lofarpipe.support.utilities import catch_segfaults
 
 
 class setupsourcedb(LOFARnodeTCP):
-    def run(self, executable, catalogue, skydb):
+    def run(self, executable, catalogue, skydb, dbtype):
         with log_time(self.logger):
             # Create output directory if it does not yet exist.
             skydb_dir = os.path.dirname(skydb)
@@ -38,9 +38,10 @@ class setupsourcedb(LOFARnodeTCP):
             scratch_dir = tempfile.mkdtemp()
             try:
                 cmd = [executable,
+                       "in=%s" % catalogue,
+                       "out=%s" % skydb,
+                       "outtype=%s" % dbtype,
                        "format=<",
-                       "in=%s" % (catalogue),
-                       "out=%s" % (skydb),
                        "append=false"
                       ]
                 with CatchLog4CPlus(
@@ -49,10 +50,10 @@ class setupsourcedb(LOFARnodeTCP):
                     os.path.basename(executable)
                 ) as logger:
                     catch_segfaults(cmd, scratch_dir, None, logger)
-            except CalledProcessError, e:
+            except CalledProcessError, err:
                 # For CalledProcessError isn't properly propagated by IPython
                 # Temporary workaround...
-                self.logger.error(str(e))
+                self.logger.error(str(err))
                 return 1
             finally:
                 shutil.rmtree(scratch_dir)