Task #3054

Added the option to specify the file name patterns on the command line

CEP/MS/src/expandcalps

@@ -7,9 +7,34 @@ if __name__ == "__main__":
     if len(sys.argv) < 3:
         print ''
         print 'Insufficient arguments; run as:'
-        print '    expandcalps parsetin parsetout'
+        print '    expandcalps parsetin parsetout [mspattern, [pdbname]]'
+        print ''
+        print ' If an mspattern is given, it must be the name of a directory'
+        print ' containing the MeasurementSets or a glob-pattern matching the'
+        print ' MeasurementSets to process. If a directory name is given, the'
+        print ' *.MS will be added to make it a glob pattern.'
+        print ' If no or an empty mspattern is given, it means that the msin'
+        print ' parameter in the input parset defines the glob-pattern for'
+        print ' the MeasurementSets to process.'
+        print ''
+        print ' If a pdbname is given, it must define the names of the output'
+        print ' ParmDBs using placeholders like <BN>.'
+        print ' If no or an empty pdbname is given, the pdbout parameter in the'
+        print ' input parset defines the output names'
         print ''
         sys.exit(1)
+    # See if the MS pattern is given.
+    inname = 'msin'
+    if len(sys.argv) > 3  and  len(sys.argv[3]) > 0:
+        # Add *.MS if no glob pattern given. Turn it into a list.
+        inname = [lmt.addfileglob (sys.argv[3], '*.MS')]
+    # See if the ParmDB name is given.
+    outname = 'pdbout'
+    if len(sys.argv) > 4  and  len(sys.argv[4]) > 0:
+        outname = [sys.argv[4]]
+    # Expand the parset.
+    print inname, outname
+    exit
     lmt.expandps (sys.argv[1], sys.argv[2],
-                  {'in': [['msin', 'Input_Correlated']],
-                   'out': [['pbout', 'Output_InstrumentModel']]})
+                  {'in': [(inname, 'Input_Correlated')],
+                   'out': [(outname, 'Output_InstrumentModel')]})

CEP/MS/src/expandimageps

@@ -7,7 +7,7 @@ if __name__ == "__main__":
     if len(sys.argv) < 3:
         print ''
         print 'Insufficient arguments; run as:'
-        print '    expandimageps parsetin parsetout [nsubbands] [nodeindex]'
+        print '    expandimageps parsetin parsetout [nsubbands [nodeindex]'
         print '        nsubbands    number of subbands in a single image'
         print '                     default = 10'
         print '        nodeindex    subband whose node to use to run the imaging pipeline on'
@@ -20,6 +20,18 @@ if __name__ == "__main__":
         nsubbands = int(sys.argv[3])
     if len(sys.argv) > 4:
         nodeindex = int(sys.argv[4])
+    # See if the MS pattern is given.
+    msname = 'msin'
+    if len(sys.argv) > 3  and  len(sys.argv[3]) > 0:
+        msname = [sys.argv[3]]
+    # See if the ParmDB pattern is given.
+    pdbname = 'pdbin'
+    if len(sys.argv) > 4  and  len(sys.argv[4]) > 0:
+        pdbname = [sys.argv[4]]
+    # See if the MS output name is given.
+    outname = 'msout'
+    if len(sys.argv) > 5  and  len(sys.argv[45]) > 0:
+        outname = [sys.argv[5]]
     lmt.expandps (sys.argv[1], sys.argv[2],
                   {'in': [['msin', 'Input_Correlated']],
                    'out': [['imgout', 'Output_SkyImage']]},

CEP/MS/src/expandtargetps

@@ -7,10 +7,23 @@ if __name__ == "__main__":
     if len(sys.argv) < 3:
         print ''
         print 'Insufficient arguments; run as:'
-        print '    expandtargetps parsetin parsetout'
+        print '    expandtargetps parsetin parsetout [mspattern [pdbpattern [outname]]]'
         print ''
         sys.exit(1)
+    # See if the MS pattern is given.
+    msname = 'msin'
+    if len(sys.argv) > 3  and  len(sys.argv[3]) > 0:
+        msname = [sys.argv[3]]
+    # See if the ParmDB pattern is given.
+    pdbname = 'pdbin'
+    if len(sys.argv) > 4  and  len(sys.argv[4]) > 0:
+        pdbname = [sys.argv[4]]
+    # See if the MS output name is given.
+    outname = 'msout'
+    if len(sys.argv) > 5  and  len(sys.argv[5]) > 0:
+        outname = [sys.argv[5]]
+    # Expand the parset.
     lmt.expandps (sys.argv[1], sys.argv[2],
-                  {'in': [['msin', 'Input_Correlated'],
-                          ['pbin', 'Input_InstrumentModel']],
-                   'out': [['msout', 'Output_Correlated']]})
+                  {'in': [[msname, 'Input_Correlated'],
+                          [pdbname, 'Input_InstrumentModel']],
+                   'out': [[outname, 'Output_Correlated']]})

CEP/MS/src/mstools.py

@@ -142,6 +142,28 @@ def movemss (srcPattern, dstPattern, userName, bandsPerBeam=80, dryrun=False):
                     os.system (cmd)
     print nInPlace, "source files are already on the correct destination mode"
 
+def addfileglob (filename, pattern):
+    """ If needed, add the glob pattern to the filename
+
+    If the basename of the filename does not contain glob characters
+    (*, ?, [], or {}), the glob pattern is added.
+
+    """
+    hasglob = False
+    if filename[-1] == '/':
+        filename = filename[:-1]
+    else:
+        import os
+        bname = os.path.basename(filename)
+        hasglob = False
+        for c in '*?[{':
+            if c in bname:
+                hasglob = True
+                break
+    if hasglob:
+        return filename
+    return filename + '/' + pattern
+
 def expandps (parsetin, parsetout, keymap, nsubbands=0, nodeindex=0):
     """ Expand dataset names in a parset file
 
@@ -166,15 +188,21 @@ def expandps (parsetin, parsetout, keymap, nsubbands=0, nodeindex=0):
         The parameter in the input parset defines one or more filename glob
         patterns. Usually one pattern will be used, but multiple are needed
         for e.g. the imaging pipeline (a pattern per observation slice).
+        Instead of a parameter name, it is also possible to directly give a
+        list of glob patterns directly. Thus passing a string means a parameter
+        name, while a list means glob patterns.
       | 'out' maps to a list of pairs. Each pair defines the names of the
         parameter in the input and output parset. The parameter value in the
-        input parset can contain the following cexecms-like place holders:
+        input parset must define the location of the output. It can contain
+        the following cexecms-like placeholders:
         | <DN>  is the directory name of the input dataset
         | <BN>  is the basename of the input dataset
         | <BN.> is the basename till the first dot (thus without the extension)
         | <.BN> is the basename after the first dot (thus the extension)
         | <SEQ> is a 3 digit sequence number (000, 001, ...) useful for the
                 imaging pipeline.
+        Instead of an input parameter name, it is possible to directly give
+        the output location. by passing it as a list containing one element.
      nsubbands
       If > 0, the number of subbands in a subband group
       (i.e., the number of subbands to combine in an image).
@@ -238,10 +266,10 @@ def expandps (parsetin, parsetout, keymap, nsubbands=0, nodeindex=0):
     # Open parset and get all keywords.
     ps = lofar.parameterset.parameterset (parsetin)
     pskeys = ps.keys()
-    # Write nsubbands parameter if given; otherwise set to 1.
-    if nsubbands > 0:
-        ps.add ('nsubbands', str(nsubbands));
-    else:
+    # See if nsubbands parameter is given; otherwise set to 1.
+    havesubbands = true
+    if nsubbands <= 0:
+        havesubbands = false
         nsubbands = 1
         # Check and initialize.
     if nodeindex < 0  or  nodeindex >= nsubbands:
@@ -255,11 +283,15 @@ def expandps (parsetin, parsetout, keymap, nsubbands=0, nodeindex=0):
     inkeys = keymap["in"]
     nrproc = 1
     for (keyin,keyout) in inkeys:
-        # Find keyin in the parset
-        if keyin not in pskeys:
-            raise KeyError, "keyword " + keyin + " not found in parset " + parsetin
-        # Get the file name patterns/
-        patterns = ps.getStringVector(keyin)
+        # If a string, find keyin in the parset.
+        # Otherwise it defines the glob patterns.
+        if isinstance(keyin, str):
+            if keyin not in pskeys:
+                raise KeyError, "keyword " + keyin + " not found in parset " + parsetin
+            # Get the file name pattern
+            patterns = ps.getStringVector(keyin)
+        else:
+            patterns = keyin
         locs  = []
         names = []
         for patt in patterns:
@@ -285,6 +317,11 @@ def expandps (parsetin, parsetout, keymap, nsubbands=0, nodeindex=0):
         ps.add (newkey + '.filenames', str(names));
         ps.remove (keyin)
 
+    # Write nsubbands if needed.
+    if havesubbands:
+        ps.add ('subbands_per_image', str(nsubbands));
+        ps.add ('slices_per_image', str(nslice));
+
     # Process output keywords if they are present.
     if 'out' in keymap:
         if len(filenames) == 0:
@@ -292,9 +329,14 @@ def expandps (parsetin, parsetout, keymap, nsubbands=0, nodeindex=0):
         inkeys = keymap["out"]
         nrproc += 1
         for (keyin,keyout) in inkeys:
-            if keyin not in pskeys:
-                raise KeyError, "keyword " + keyin + " not found in parset " + parsetin
-            name = ps.getString(keyin)
+            if isinstance(keyin, str):
+                if keyin not in pskeys:
+                    raise KeyError, "keyword " + keyin + " not found in parset " + parsetin
+                name = ps.getString(keyin)
+            else:
+                if len(keyin) != 1:
+                    raise KeyError, "Output key " + keyin + " is not a string, thus should be a sequence of length 1"
+                name = keyin[0];
             locs  = []
             names = []
             # Create output for all input names replacing tags like <BN>.