First commit

README.md

+SMTool: the LOFAR Sky Model Tool
+================================
+
+Authors:
+* David Rafferty, from contributed scripts from
+* Björn Adebahr
+* Francesco de Gasperin
+* Reinout van Weeren
+
+Contents:
+* __doc/__: documentation
+* __tests/__: contains test sky models and scripts useful for validation
+* __lsmtool/__: contains the main LSMTool scripts
+* __lsmtool/operations/__: contains the modules for operations
+* __parsets/__: some example parsets
+
+

lsmtool/_logging.py

+# -*- coding: utf-8 -*-
+#
+# Defines the custom logger
+
+import logging
+
+def add_coloring_to_emit_ansi(fn):
+    """
+    colorize the logging output
+    """
+    # add methods we need to the class
+    def new(*args):
+        levelno = args[1].levelno
+        if(levelno>=50):
+            color = '\x1b[31m' # red
+        elif(levelno>=40):
+            color = '\x1b[31m' # red
+        elif(levelno>=30):
+            color = '\x1b[33m' # yellow
+        elif(levelno>=20):
+            color = '\x1b[32m' # green
+        elif(levelno>=10):
+            color = '\x1b[35m' # pink
+        else:
+            color = '\x1b[0m' # normal
+        args[1].msg = color + args[1].msg +  '\x1b[0m'  # normal
+        return fn(*args)
+    return new
+
+# set the logging colors
+logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit)
+# set the logging format and default level (info)
+logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
+
+def setLevel(level):
+    """
+    Change verbosity
+    """
+    if level == 'warning':
+        logging.root.setLevel(logging.WARNING)
+    elif level == 'info':
+        logging.root.setLevel(logging.INFO)
+    elif level == 'debug':
+        logging.root.setLevel(logging.DEBUG)
+

lsmtool/_version.py

+# -*- coding: utf-8 -*-
+#
+#This module stores the version and changelog
+
+# Version number
+__version__ = '1.0.0'
+
+# Change log
+def changelog():
+    """
+    LSMTool Changelog.
+    -----------------------------------------------
+    2014/05/25 - Version 1.0.0 (initial release)
+    """
+    pass

lsmtool/operations/__init__.py

+import os
+import glob
+
+__all__ = [ os.path.basename(f)[:-3] for f in glob.glob(os.path.dirname(__file__)+"/*.py") if not f.endswith('__init__.py') and not f.split('/')[-1].startswith('_')]
+for x in __all__: __import__(x, locals(), globals())

lsmtool/operations/_cluster.py

+# -*- coding: utf-8 -*-
+#
+# Defines cluster functions used by the group operation
+#
+# Copyright (C) 2013 - Reinout van Weeren
+# Copyright (C) 2013 - Francesco de Gasperin
+# Modified by David Rafferty as required for integration into LSMTool
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+import os, sys
+import numpy as np
+import pylab as pl
+import itertools
+try:
+    import pyrap.tables as pt
+    import lofar.stationresponse as lsr
+except ImportError:
+    pass
+
+
+class Patch():
+    def __init__(self, name, ra, dec, flux):
+        self.name = name
+        self.ra = ra
+        self.dec = dec
+        self.flux = flux
+
+class Cluster():
+    def __init__(self, name, patch):
+        self.name = name
+        self.init_patch = patch
+        self.patches = []
+        self.update_cluster_coords()
+
+    def add_patch(self, patch):
+        """
+        Add a patch to this cluster
+        """
+        self.patches.append(patch)
+
+    def total_cluster_flux(self):
+        """
+        Return total cluster flux
+        """
+        cluster_flux = self.init_patch.flux
+        for patch in self.patches:
+            cluster_flux += patch.flux
+
+        return cluster_flux
+
+    def update_cluster_coords(self):
+        """
+        update the self.centroid_ra, self.centroid_dec, self.mid_ra, self.mid_dec
+        """
+        self.centroid_ra = self.init_patch.ra*self.init_patch.flux
+        self.centroid_dec = self.init_patch.dec*self.init_patch.flux
+        min_ra = np.inf
+        max_ra = -np.inf
+        min_dec = np.inf
+        max_dec = -np.inf
+
+        for patch in self.patches:
+            self.centroid_ra += patch.ra*patch.flux
+            self.centroid_dec += patch.dec*patch.flux
+            if patch.ra < min_ra: min_ra = patch.ra
+            if patch.ra > max_ra: max_ra = patch.ra
+            if patch.dec < min_dec: min_dec = patch.dec
+            if patch.dec > max_dec: max_dec = patch.dec
+
+        self.centroid_ra /= self.total_cluster_flux()
+        self.centroid_dec /= self.total_cluster_flux()
+        self.mid_ra = min_ra + (max_ra - min_ra)/2.
+        self.mid_dec = min_dec + (max_dec - min_dec)/2.
+
+
+def ratohms_string(ra):
+    rah, ram, ras = ratohms(ra)
+    return str(rah) + ':' + str(ram) + ':' +str(round(ras,2))
+
+
+def dectodms_string(dec):
+    decd, decm, decs = dectodms(dec)
+    return str(decd) + '.' + str(decm) + '.' +str(round(decs,2))
+
+
+def compute_patch_center(data, beam_ms = None):
+    """
+    Return the patches names, central (weighted) RA and DEC and total flux
+    """
+    patch_names = np.unique(data['Name'])
+    patches = []
+
+    # get the average time of the obs, OK for 1st order correction
+    if beam_ms != None:
+        t = pt.table(beam_ms, ack=False)
+        tt = t.query('ANTENNA1==0 AND ANTENNA2==1', columns='TIME')
+        time = tt.getcol("TIME")
+        time = min(time) + ( max(time) - min(time) ) / 2.
+        t.close()
+        sr = lsr.stationresponse(beam_ms, False, True)
+
+    for patch_name in patch_names:
+        comp_ids = np.where(data['Name'] == patch_name)[0]
+
+        patch_ra    = 0.
+        patch_dec   = 0.
+        weights_ra  = 0.
+        weights_dec = 0.
+        patch_flux  = 0.
+
+        for comp_id in comp_ids:
+
+            comp_ra   = data['RA'][comp_id]
+            comp_dec  = data['Dec'][comp_id]
+            comp_flux = np.float(data['I'][comp_id])
+
+            # beam correction
+            if beam_ms != None:
+                sr.setDirection(comp_ra*np.pi/180.,comp_dec*np.pi/180.)
+                # use station 0 to compute the beam and get mid channel
+                beam = sr.evaluateStation(time,0)
+                r = abs(beam[int(len(beam)/2.)])
+                beam = ( r[0][0] + r[1][1] ) / 2.
+                #print "Beam:", beam,
+                comp_flux *= beam
+
+            # calculate the average weighted patch center, and patch flux
+            patch_flux  += comp_flux
+            patch_ra    += comp_ra * comp_flux
+            patch_dec   += comp_dec * comp_flux
+            weights_ra  += comp_flux
+            weights_dec += comp_flux
+
+        patches.append(Patch(patch_name, patch_ra/weights_ra, patch_dec/weights_dec, patch_flux))
+
+    return patches
+
+
+def angsep2(ra1, dec1, ra2, dec2):
+    """Returns angular separation between two coordinates (all in degrees)"""
+    from astropy.coordinates import FK5
+    import astropy.units as u
+
+    coord1 = FK5(ra1, dec1, unit=(u.degree, u.degree))
+    coord2 = FK5(ra2, dec2, unit=(u.degree, u.degree))
+
+    return coord1.separation(coord2)
+
+
+def create_clusters(patches_orig, Q):
+    """
+    Clusterize all the patches of the skymodel iteratively around the brightest patches
+    """
+    # sort the patches by brightest first
+    idx = np.argsort([patch.flux for patch in patches_orig])[::-1] # -1 to reverse sort
+    patches = list(np.array(patches_orig)[idx])
+
+    # initialize clusters with the brightest patches
+    clusters = []
+    for i, patch in enumerate(patches[0:Q]):
+        clusters.append(Cluster('CLUSTER_'+str(i), patch))
+
+    # Iterate until no changes in which patch belongs to which cluster
+    count = 1
+    patches_seq_old = []
+    while True:
+        print 'Iteration', count
+
+        for cluster in clusters:
+            cluster.update_cluster_coords()
+            # reset patches
+            cluster.patches = []
+
+        # select patches after the first Q
+        for patch in patches[Q:]:
+            # finding closest cluster for each patch
+            angulardistance = np.inf
+            for cluster in clusters:
+                adis = abs(angsep2(cluster.centroid_ra, cluster.centroid_dec, patch.ra, patch.dec))
+                if adis < angulardistance:
+                    angulardistance = adis
+                    closest_cluster = cluster
+            # add this patch to the closest cluster
+            closest_cluster.add_patch(patch)
+
+        patches_seq = []
+        for cluster in clusters:
+            patches_seq.extend(cluster.patches)
+
+        count += 1
+        if patches_seq == patches_seq_old: break
+        patches_seq_old = patches_seq
+
+    # Make output patch column
+    patchNames = [''] * len(patches)
+    patchNames_orig = [p.name for p in patches_orig]
+    for c in clusters:
+        for p in c.patches:
+            patchNames[patchNames_orig.index(p.name)] = c.name
+
+    return np.array(patchNames)
+

lsmtool/operations/_filter.py

+# -*- coding: utf-8 -*-
+#
+# Defines filter functions used by the remove and select operations
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+import logging
+from .. import tableio
+
+
+def filter(LSM, filterExpression, exclusive=False, aggregate=False, weight=False,
+    beamMS=None, useRegEx=False):
+    """
+    Filters the sky model, keeping all sources that meet the given expression.
+
+    After filtering, the sky model contains only those sources for which the
+    given filter expression is true.
+
+    Parameters
+    ----------
+    filterExpression : str or dict
+        A string specifying the filter expression in the form:
+            '<property> <operator> <value> [<units>]'
+        (e.g., 'I <= 10.5 Jy'). These elements can also be given as a
+        dictionary in the form:
+            {'filterProp':property, 'filterOper':operator,
+                'filterVal':value, 'filterUnits':units}
+        or as a list:
+            [property, operator, value, value]
+        The property to filter on must be a valid column name or the filename
+        of a mask image.
+
+        Supported operators are:
+            - !=
+            - <=
+            - >=
+            - >
+            - <
+            - = (or '==')
+        Units are optional and must be specified as required by astropy.units.
+    exclusive : bool, optional
+        If False, sources that meet the filter expression are kept. If True,
+        sources that do not meet the filter expression are kept.
+    aggregate : bool, optional
+        If True, values are aggregated by patch before filtering. There,
+        filtering will be done by patch.
+    weight : bool, optional
+        If True, aggregated values will be calculated when appropriate using
+        the Stokes I fluxes of sources in each patch as weights
+    beamMS : string, optional
+        Measurement set from which the primary beam will be estimated. Either
+        beamMS or beamFWHM and position must be specified for ApparentFlux
+        filters.
+    useRegEx : bool, optional
+        If True, string matching will use regular expression matching. If
+        False, string matching uses Unix filename matching.
+
+    Examples
+    --------
+    Filter on column 'I' (Stokes I flux). This filter will select all sources
+    with Stokes I flux greater than 1.5 Jy::
+
+        >>> s = SkyModel('sky.model')
+        >>> s.filter('I > 1.5 Jy')
+        INFO: Filtered out 1102 sources.
+
+    If the sky model has patches and the filter is desired per patch, use
+    ``aggregate = True``::
+
+        >>> s.filter('I > 1.5 Jy', aggregate=True)
+
+    Filter on source names, keeping those that match "src*_1?"::
+
+        >>> s.filter('Name == src*_1?')
+
+    Use a CASA clean mask image to keep sources that lie in masked regions::
+
+        >>> s.filter('clean_mask.mask == True')
+
+    """
+    import numpy as np
+    import math as m
+    import os
+
+    if filterExpression is None:
+        logging.error('Please specify a filter expression.')
+        return 1
+
+    if type(filterExpression) is list:
+        filterProp, filterOperStr, filterVal, filterUnits = filterExpression
+        filterOper, f = convertOperStr(filterOperStr)
+
+    elif type(filterExpression) is dict:
+        if ('filterProp' in filterExpression.keys() and
+            'filterOper' in filterExpression.keys() and
+            'filterVal' in filterExpression.keys()):
+            filterProp = filterExpression['filterProp']
+            filterOperStr = filterExpression['filterOper']
+            filterOper, f = convertOperStr(filterOperStr)
+            filterVal = filterExpression['filterVal']
+        else:
+            logging.error("Please specify filter dictionary as "
+                "{'filterProp':property, 'filterOper':operator, "
+                "'filterVal':value, 'filterUnits':units}")
+            return 1
+        if 'filterUnits' in filterExpression.keys():
+            filterUnits = filterExpression['filterUnits']
+        else:
+            filterUnits = None
+
+    elif type(filterExpression) is str:
+        # Parse the filter expression
+        filterProp, filterOper, filterVal, filterUnits = parseFilter(filterExpression)
+    else:
+        return 1
+
+    # Get the column values to filter on
+    if LSM._verifyColName(filterProp) in LSM.table.colnames:
+        filterProp = LSM._verifyColName(filterProp)
+        colVals = LSM.getColValues(filterProp, units=filterUnits,
+            aggregate=aggregate, weight=weight)
+
+        if beamMS is not None and filterProp == 'I':
+            RADeg = LSM.getColValues('RA')
+            DecDeg = LSM.getColValues('Dec')
+            colVals = applyBeam(beamMS, colVals, RADeg, DecDeg)
+    else:
+        # Assume filterProp is a mask filename and try to load mask
+        if os.path.exists(fileName):
+            RARad = LSM.getColValues('RA', units='radian')
+            DecRad = LSM.getColValues('Dec', units='radian')
+            colVals = getMaskValues(mask, RARad, DecRad)
+            if colVals is None:
+                return 1
+        else:
+            return 1
+
+    # Do the filtering
+    if colVals is None:
+        return 1
+    filt = getFilterIndices(colVals, filterOper, filterVal, useRegEx=useRegEx)
+    if exclusive:
+        filt = [i for i in range(len(colVals)) if i not in filt]
+    if len(filt) == 0:
+        logging.error('Filter would result in an empty sky model.')
+        return 1
+    if len(filt) == len(colVals):
+        logging.info('Filtered out 0 sources.')
+        return 0
+
+    if LSM._hasPatches and aggregate:
+        sourcesToKeep = LSM.getColValues('Patch', aggregate=True)[filt]
+        def filterByName(tab, key_colnames):
+            if tab['Patch'][0] in sourcesToKeep:
+                return True
+            else:
+                return False
+        nPatchesOrig = len(LSM.table.groups)
+        LSM.table = LSM.table.groups.filter(filterByName) # filter
+        LSM.table = LSM.table.group_by('Patch') # regroup
+        nPatchesNew = len(LSM.table.groups)
+        if exclusive:
+            logging.info('Removed {0} patches.'.format(nPatchesOrig-nPatchesNew))
+        else:
+            logging.info('Kept {0} patches.'.format(nPatchesNew))
+    else:
+        nRowsOrig = len(LSM.table)
+        LSM.table = LSM.table[filt]
+        nRowsNew = len(LSM.table)
+        if LSM._hasPatches:
+            LSM.table = LSM.table.group_by('Patch') # regroup
+        if exclusive:
+            logging.info('Removed {0} sources.'.format(nRowsOrig-nRowsNew))
+        else:
+            logging.info('Kept {0} sources.'.format(nRowsNew))
+
+    return LSM
+
+
+def parseFilter(filterExpression):
+    """
+    Takes a filter expression and returns tuple of
+    (property, operation, val, units), all as strings
+    """
+    # Get operator function
+    filterOper, filterOperStr = convertOperStr(filterExpression)
+    if filterOper is None:
+        return (None, None, None, None)
+
+    filterParts = filterExpression.split(filterOperStr)
+    if len(filterParts) != 2:
+        logging.error("Filter expression must be of the form '<property> "
+            "<operator> <value> <unit>'\nE.g., 'Flux >= 10 Jy'")
+        return (None, None, None, None)
+
+    # Get the column to filter on
+    filterProp = filterParts[0].strip().lower()
+    if filterProp not in tableio.allowedColumnNames:
+        logging.error('Column name "{0}" is not a valid column.'.format(colName))
+        return (None, None, None, None)
+
+    # Get the filter value(s)
+    filterValAndUnits = filterParts[1].strip()
+    if tableio.allowedColumnDefaults[filterProp] == 'N/A':
+        # Column values are strings. Allow only '==' and '!=' operators
+        if filterOperStr not in ['=', '==', '!=']:
+            logging.error("Filter operator '{0}' not allow with string columns. "
+                "Supported operators are '!=' or '=' (or '==')".format(filterOperStr))
+            return (None, None, None, None)
+
+        # Check for a list of values
+        if '[' in filterValAndUnits and ']' in filterValAndUnits:
+            filterVal = filterValAndUnits.split(']')[0].strip('[')
+            filterValParts = filterVal.split(',')
+            filterVal = []
+            for val in filterValParts:
+                val = val.strip()
+                val = val.strip('"')
+                val = val.strip("'")
+                filterVal.append(val)
+        else:
+            filterVal = filterValAndUnits.split(' ')[0].strip()
+            filterVal = filterVal.strip('"')
+            filterVal = filterVal.strip("'")
+    else:
+        # The column to filter is type float
+        try:
+            filterVal = float(filterValAndUnits.split(' ')[0].strip())
+        except ValueError:
+            logging.error('Filter value not understood. Make sure the value is '
+                'separated from the units (if any)')
+            return (None, None, None, None)
+
+    # Try and get the units
+    try:
+        filterUnits = filterValAndUnits.split(']')
+        if len(filterUnits) == 1:
+            filterUnits = filterUnits[0].split(' ')[1].strip()
+        else:
+            filterUnits = filterUnits[1].strip()
+    except IndexError:
+        filterUnits = None
+    if filterUnits == '':
+        filterUnits = None
+
+    if type(filterVal) is str and filterUnits is not None:
+        logging.error('Units are not allowed with string columns.')
+        return (None, None, None, None)
+
+    return (filterProp, filterOper, filterVal, filterUnits)
+
+
+def convertOperStr(operStr):
+    """
+    Returns operator function corresponding to string.
+    """
+    import operator as op
+
+    filterOperStr = None
+    ops = {'!=':op.ne, '<=':op.le, '>=':op.ge, '>':op.gt, '<':op.lt,
+        '==':op.eq, '=':op.eq}
+    for op in ops:
+        if op in operStr:
+            if filterOperStr is None:
+                filterOperStr = op
+            elif len(op) > len(filterOperStr):
+                # Pick longer match
+                filterOperStr = op
+    if filterOperStr is None:
+        logging.error("Filter operator '{0}' not understood. Supported "
+            "operators are '!=', '<=', '>=', '>', '<', '=' (or '==')".
+            format(operStr))
+        return None, None
+
+    return ops[filterOperStr], filterOperStr
+
+
+def getFilterIndices(colVals, filterOper, filterVal, useRegEx=False):
+    """
+    Returns the indices that correspond to the input filter expression
+    """
+    import operator as op
+    import fnmatch
+    import re
+
+    if type(filterVal) is not list:
+        filterVal = [filterVal]
+    filterInd = []
+    for val in filterVal:
+        if type(val) is str:
+            # String -> use regular expression or Unix filename matching search
+            if filterOper is op.eq:
+                if useRegEx:
+                    filt = [i for i, item in enumerate(colVals) if re.search(val, item) is not None]
+                else:
+                    filt = [i for i, item in enumerate(colVals) if fnmatch.fnmatch(item, val)]
+            elif filterOper is op.ne:
+                if useRegEx:
+                    filt = [i for i, item in enumerate(colVals) if re.search(val, item) is None]
+                else:
+                    filt = [i for i, item in enumerate(colVals) if not fnmatch.fnmatch(item, val)]
+            else:
+                logging.error("Filter operator '{0}' not allow with string columns. "
+                    "Supported operators are '!=' or '=' (or '==')".format(filterOper))
+                return None
+        else:
+            filtBool = filterOper(colVals, val)
+            filt = [f for f in range(len(colVals)) if filtBool[f]]
+        filterInd += filt
+
+    return filterInd
+
+
+def getMaskValues(mask, RARad, DecRad):
+    """
+    Returns an array of mask values for each (RA, Dec) pair in radians
+    """
+    import math
+    import pyrap
+
+    try:
+        maskdata = pyrap.images.image(mask)
+        maskval = maskdata.getdata()[0][0]
+    except:
+        loggin.error("Error opening mask file '{0}'".format(mask))
+        return None
+
+    vals = []
+    for raRad, decRad in zip(RARad, DecRad):
+        (a, b, _, _) = maskdata.toworld([0, 0, 0, 0])
+        (_, _, pixY, pixX) = maskdata.topixel([a, b, decRad, raRad])
+        try:
+            # != is a XOR for booleans
+            if (not maskval[math.floor(pixY)][math.floor(pixX)]) != False:
+                vals.append(True)
+            else:
+                vals.append(False)
+        except:
+            vals.append(False)
+
+    return np.array(vals)
+
+
+def applyBeam(beamMS, fluxes, RADeg, DecDeg):
+    """
+    Returns flux attenuated by primary beam.
+    """
+    try:
+        import pyrap.tables as pt
+    except ImportError:
+        logger.error('Could not import pyrap.tables')
+    try:
+        import lofar.stationresponse as lsr
+    except ImportError:
+        logger.error('Could not import lofar.stationresponse')
+
+    t = pt.table(beamMS, ack=False)
+    tt = t.query('ANTENNA1==0 AND ANTENNA2==1', columns='TIME')
+    time = tt.getcol("TIME")
+    time = min(time) + ( max(time) - min(time) ) / 2.
+    t.close()
+
+    attFluxes = []
+    sr = lsr.stationresponse(beamMS, False, True)
+    for flux, RA, Dec in zip(fluxes, RADeg, DecDeg):
+        # Use station 0 to compute the beam and get mid channel
+        sr.setDirection(RA*np.pi/180., Dec*np.pi/180.)
+        beam = sr.evaluateStation(time, 0)
+        r = abs(beam[int(len(beam)/2.)])
+        beam = ( r[0][0] + r[1][1] ) / 2.
+        attFluxes.append(flux * beam)
+
+    return np.array(attFluxes)

lsmtool/operations/add.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# This operation implements adding of sources to the sky model
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+import logging
+from .. import tableio
+
+logging.debug('Loading ADD module.')
+
+
+def run(step, parset, LSM):
+
+    outFile = parset.getString('.'.join(["LSMTool.Steps", step, "OutFile"]), '' )
+    colNamesVals = {}
+    for colName in tableio.inputColumnNames:
+        colNamesVals[colName] = parset.getString('.'.join(["LSMTool.Steps",
+            step, tableio.inputColumnNames[colName]]), '' )
+
+    result = add(LSM, colNamesVals)
+
+    # Write to outFile
+    if outFile == '' or outFile is None:
+        outFile = LSM._fileName
+    LSM.writeFile(outFile, clobber=True)
+
+    return 0
+
+
+def add(LSM, colNamesVals):
+
+    LSM.setRowValues(colNamesVals)

lsmtool/operations/concatenate.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# This operation implements joining of two sky models
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+import logging
+
+logging.debug('Loading CONCATENATE module.')
+
+
+def run(step, parset, LSM):
+
+    outFile = parset.getString('.'.join(["LSMTool.Steps", step, "OutFile"]), '' )
+    skyModel2 = parset.getString('.'.join(["LSMTool.Steps", step, "Skymodel2"]), '' )
+    matchBy = parset.getString('.'.join(["LSMTool.Steps", step, "MatchBy"]), '' )
+    radius = parset.getString('.'.join(["LSMTool.Steps", step, "Radius"]), '' )
+    keep = parset.getString('.'.join(["LSMTool.Steps", step, "KeepMatches"]), '' )
+
+    concatenate(LSM, skyModel2, matchBy, radius, keep)
+
+    # Write to outFile
+    if outFile == '' or outFile is None:
+        outFile = LSM._fileName
+    LSM.writeFile(outFile, clobber=True)
+
+    return 0
+
+
+def concatenate(LSM1, LSM2, matchBy='name', radius=10.0, keep='all'):
+    """
+    Concatenate two sky models
+    """
+    from astropy.table import vstack, Column
+    from astropy.coordinates import ICRS
+    from astropy import units as u
+    import numpy as np
+    from .. import skymodel
+
+    if type(LSM2) is str:
+        LSM2 = skymodel.SkyModel(LSM2)
+
+    if (LSM1._hasPatches and not LSM2._hasPatches):
+         LSM2.group('every')
+    if (LSM2._hasPatches and not LSM1._hasPatches):
+         LSM1.group('every')
+    table1 = LSM1.table.copy()
+    table2 = LSM2.table.copy()
+
+    # Due to a bug in astropy, the spectral index column must be removed before
+    # joining
+    nameCol1 = table1['Name']
+    nameCol2 = table2['Name']
+    spCol1 = table1['SpectralIndex']
+    spCol1Indx = table1.index_column('SpectralIndex')
+    spCol2 = table2['SpectralIndex']
+    table1.remove_column('SpectralIndex')
+    table2.remove_column('SpectralIndex')
+
+    if matchBy.lower() == 'name':
+        LSM1.table = vstack([table1, table2])
+    if matchBy.lower() == 'patch':
+        LSM1.table = vstack([table1, table2])
+    elif matchBy.lower() == 'radius':
+        # Create catalogs
+        catalog1 = ICRS(LSM1.getColValues('RA'), LSM1.getColValues('Dec'),
+            unit=(u.degree, u.degree))
+        catalog2 = ICRS(LSM2.getColValues('RA'), LSM2.getColValues('Dec'),
+            unit=(u.degree, u.degree))
+        idx, d2d, d3d = catalog1.match_to_catalog_sky(catalog2)
+
+        matches = np.where(d2d.value <= radius)
+
+        matchCol1 = np.array(range(len(LSM1)))
+        matchCol2 = np.array(range(len(LSM2))) + len(LSM1)
+
+        # Set values to be the same for the matches
+        matchCol2[idx[matches]] = matchCol1[matches]
+
+        # Now add columns and stack
+        col1 = Column(name='match', data=matchCol1)
+        col2 = Column(name='match', data=matchCol2)
+        table1.add_column(col1)
+        table2.add_column(col2)
+        LSM1.table = vstack([table1, table2])
+
+    # Add spectral index column back
+    spData = np.zeros((len(LSM1), 2), dtype=np.float)
+    for i, name in enumerate(LSM1.getColValues('Name')):
+        if name in nameCol2:
+            indx = np.where(nameCol2.data == name)
+            spData[i] = spCol2[indx]
+        else:
+            indx = np.where(nameCol1.data == name)
+            spData[i] = spCol1[indx]
+    spCol = Column(name='SpectralIndex', data=spData)
+    LSM1.table.add_column(spCol, index=spCol1Indx)
+
+    if keep == 'from1' or keep == 'from2':
+        # Remove any duplicates
+        if matchBy.lower() == 'name':
+            colName = 'Name'
+        elif matchBy.lower() == 'radius':
+            colName = 'match'
+        vals = LSM1.table[colName]
+        toRemove = []
+        for val in vals:
+            indx = np.where(vals == val)[0]
+            if len(indx) > 1:
+                if keep == 'from1':
+                    toRemove.append(indx[1:])
+                else:
+                    toRemove.append(indx[0])
+        LSM1.table.remove_rows(toRemove)
+
+    # Rename any duplicates
+    names = LSM1.getColValues('Name')
+    for name in set(names):
+        indx = np.where(names == name)[0]
+        if len(indx) > 1:
+            LSM1.table['Name'][indx[0]] = name + '_1'
+            LSM1.table['Name'][indx[1]] = name + '_2'
+
+    if matchBy.lower() == 'radius':
+        LSM1.table.remove_column('match')
+
+    if LSM1._hasPatches:
+        LSM1._updateGroups()
+