diff --git a/compress_pipeline.cwl b/compress_pipeline.cwl
index 74d63485d72a2cf2f6e2f65b5db6b22ac4e4f67a..f83ba79ec6e8ae9946ae3b31a8fa3dab36817975 100644
--- a/compress_pipeline.cwl
+++ b/compress_pipeline.cwl
@@ -1,5 +1,5 @@
class: Workflow
-cwlVersion: v1.0
+cwlVersion: v1.1
id: compress_pipeline_cwl
label: compress_pipeline.cwl
inputs:
@@ -24,6 +24,14 @@ outputs:
outputSource:
- format_ingest/ingest
type: Any
+ - id: uv_plot
+ type: File
+ outputSource:
+ - plot_uvw_coverage/uv_plot
+ - id: uv_coverage
+ type: File
+ outputSource:
+ - inspect_flagging_dataloss/flags_output
steps:
- id: extract_sip_meta
in:
@@ -76,8 +84,16 @@ steps:
source: msin
out:
- id: output
+ - id: flags_output
run: steps/inspect_flagging_dataloss.cwl
label: inspect_flagging_dataloss
+ - id: plot_uvw_coverage
+ in:
+ - id: input_file
+ source: inspect_flagging_dataloss/flags_output
+ out:
+ - id: uv_plot
+ run: steps/plot_uvw_coverage.cwl
- id: compress
in:
- id: directory
diff --git a/download_and_compress_pipeline.cwl b/download_and_compress_pipeline.cwl
index a90a90fe70e8dd2bf579aded81c58b8b4118b055..fc5dad3045331cf4e9fd672d115d0f7e3ac08133 100644
--- a/download_and_compress_pipeline.cwl
+++ b/download_and_compress_pipeline.cwl
@@ -1,5 +1,5 @@
class: Workflow
-cwlVersion: v1.0
+cwlVersion: v1.1
id: compress_pipeline_cwl
label: compress_pipeline.cwl
inputs:
@@ -28,6 +28,14 @@ outputs:
outputSource:
- compress/inspect
type: Any[]
+ - id: uv_coverage
+ type: File[]
+ outputSource:
+ - compress/uv_coverage
+ - id: uv_plot
+ type: File[]
+ outputSource:
+ - compress/uv_plot
steps:
- id: fetch_data
in:
@@ -52,6 +60,8 @@ steps:
- id: inspect
- id: logfile
- id: ingest
+ - id: uv_coverage
+ - id: uv_plot
run: ./compress_pipeline.cwl
requirements:
- class: ScatterFeatureRequirement
diff --git a/steps/DPPP.cwl b/steps/DPPP.cwl
index aa1aab4fb6808f04f607b547230450eef5b49cd0..08d519b0ed0fe6c618544e10001d8bc5b27ef55c 100755
--- a/steps/DPPP.cwl
+++ b/steps/DPPP.cwl
@@ -1,5 +1,5 @@
class: CommandLineTool
-cwlVersion: v1.0
+cwlVersion: v1.1
id: dppp
baseCommand:
- DPPP
diff --git a/steps/compress.cwl b/steps/compress.cwl
index 1c9b461c623646516eb35c8ea5b3da36d2564552..8e884a19f52f7ba8fd9034030555089b2489f57e 100644
--- a/steps/compress.cwl
+++ b/steps/compress.cwl
@@ -1,7 +1,7 @@
id: compress
label: compress
class: CommandLineTool
-cwlVersion: v1.0
+cwlVersion: v1.1
inputs:
- id: directory
type: Directory
diff --git a/steps/define_parset.cwl b/steps/define_parset.cwl
index 24221577eb3a7dcd5016c14e3e34ee56158fd8f5..a938095e4485904b1c1d1a88a9d57ec09c4242e2 100644
--- a/steps/define_parset.cwl
+++ b/steps/define_parset.cwl
@@ -1,7 +1,5 @@
class: CommandLineTool
-cwlVersion: v1.0
-$namespaces:
- sbg: 'https://www.sevenbridges.com/'
+cwlVersion: v1.1
id: define_parset
baseCommand:
- cp
@@ -30,7 +28,7 @@ requirements:
listing:
- entryname: input.parset
entry: |+
- steps=[flagedge,flagelev,aoflag,flagamp]
+ steps=[flagedge,flagelev,aoflag,flagamp $(inputs.demix? ',demix': '')]
#
flagedge.chan=[0..nchan/32-1,31*nchan/32..nchan-1]
flagedge.type=preflagger
@@ -41,6 +39,7 @@ requirements:
aoflag.autocorr=$(inputs.flag_autocorrelation?'True':'False')
aoflag.strategy=/usr/local/share/aoflagger/strategies/lofar-default.lua
#
+
flagbaseline.type=preflagger
flagbaseline.baseline=[]
#
diff --git a/steps/extract_sip_meta.cwl b/steps/extract_sip_meta.cwl
index e21882259afd2d444754942e207fd47163d7130c..f669e50217e1b68f2105616b5bc3f5f34abbcb0c 100644
--- a/steps/extract_sip_meta.cwl
+++ b/steps/extract_sip_meta.cwl
@@ -1,5 +1,5 @@
class: CommandLineTool
-cwlVersion: v1.0
+cwlVersion: v1.1
id: inspect_flagging_dataloss
baseCommand:
- bash
diff --git a/steps/fetch_data.cwl b/steps/fetch_data.cwl
index 10e782563ac78537ae3507472bbba602e93d5d3d..454530ed8f9d5b1d8c91009e1f270da09787d85b 100644
--- a/steps/fetch_data.cwl
+++ b/steps/fetch_data.cwl
@@ -1,7 +1,7 @@
id: fetchdata
label: fetch_data
class: CommandLineTool
-cwlVersion: v1.0
+cwlVersion: v1.1
inputs:
- id: surl_link
type: string
diff --git a/steps/inspect_compression_quality.cwl b/steps/inspect_compression_quality.cwl
index 699b9e27c00f6ceabad0f368dba75c58925f375d..4bd65b83a0d12590cfa63fe538b8862983d50963 100644
--- a/steps/inspect_compression_quality.cwl
+++ b/steps/inspect_compression_quality.cwl
@@ -1,5 +1,5 @@
class: CommandLineTool
-cwlVersion: v1.0
+cwlVersion: v1.1
$namespaces:
sbg: 'https://www.sevenbridges.com/'
id: inspect_compression_quality
diff --git a/steps/inspect_flagging_dataloss.cwl b/steps/inspect_flagging_dataloss.cwl
index 5668af87fa46797782cf9787818d35c5e61cd6f3..4214dd55c42bdfc2371e6ae4b13443c8c92a87e5 100644
--- a/steps/inspect_flagging_dataloss.cwl
+++ b/steps/inspect_flagging_dataloss.cwl
@@ -1,7 +1,5 @@
class: CommandLineTool
-cwlVersion: v1.0
-$namespaces:
- sbg: 'https://www.sevenbridges.com/'
+cwlVersion: v1.1
id: inspect_flagging_dataloss
baseCommand:
- python3
@@ -12,11 +10,20 @@ inputs:
inputBinding:
position: 0
shellQuote: false
+ - id: uvw_sampling
+ type: int?
+ default: 100
+ inputBinding:
+ position: 1
outputs:
- id: output
type: 'File'
outputBinding:
glob: metrics.json
+ - id: flags_output
+ type: 'File'
+ outputBinding:
+ glob: flags.h5
label: inspect_flagging_dataloss
requirements:
- class: ShellCommandRequirement
@@ -26,11 +33,92 @@ requirements:
entry: |
import sys
from casacore.tables import table
+ import itertools
+ import json
+ import h5py
+ import numpy as np
+ import re
+ from scipy.constants import c as LIGHT_SPEED
input_ms_path = sys.argv[1]
+ sampling = int(sys.argv[2])
+ main_table = table(input_ms_path, 'r')
+
+ antennas = table(input_ms_path + '/ANTENNA', 'r')
+ spec_window = table(input_ms_path + '/SPECTRAL_WINDOW', 'r')
+ observation = table(input_ms_path + '/OBSERVATION', 'r')
+
+ sas_id, sap, sb = re.search('L(\d*).*SAP(\d{3}).*SB(\d{3}).*',
+ observation.getcell('LOFAR_FILENAME', 0)).groups()
+ antenna_names = antennas.getcol('NAME')
+ antenna_positions = antennas.getcol('POSITION')
+
+ ref_frequency = spec_window.getcell('REF_FREQUENCY', 0)
+
+ TO_UVW = ref_frequency / LIGHT_SPEED # 1 / (c/nu)
+
+ flags = np.array(main_table.getcol('FLAG'))
+ vis = np.array(main_table.getcol('DATA'))
+
+ uvw = np.array(main_table.getcol('UVW') * TO_UVW)
+
+ antenna1 = np.array(main_table.getcol('ANTENNA1'))
+ antenna2 = np.array(main_table.getcol('ANTENNA2'))
+ time = np.array(main_table.getcol('TIME'))
+ n_antennas = antennas.nrows()
+ n_baselines = int((n_antennas + 1) * n_antennas * .5)
+ n_times = flags.shape[0] // n_baselines
+ dataloss = np.array(vis == 0, dtype=np.int)
+ dataloss_matrix, (urange, vrange, wrange) = np.histogramdd(uvw, bins=[sampling, sampling, 1], weights=np.nanmean(dataloss, axis=(1,2)))
+ flag_matrix, (urange, vrange, wrange) = np.histogramdd(uvw, bins=[sampling, sampling, 1], weights=np.nanmean(flags, axis=(1,2)))
+ coverage, (urange, vrange, wrange) = np.histogramdd(uvw, bins=[sampling, sampling, 1])
+
+ flags = flags.reshape(n_times, n_baselines, *flags.shape[1:], order='C')
+ dataloss = dataloss.reshape(n_times, n_baselines, *flags.shape[2:], order='C')
+ uvw = uvw.reshape(n_times, n_baselines, *uvw.shape[1:], order='C')
+ timestamp = time.reshape(n_times, n_baselines, order='C')[:, 0]
+
+ baselines = [
+ "%s,%s" % (antenna_names[i], antenna_names[j])
+ for i, j in itertools.product(range(n_antennas), range(n_antennas)) if i >= j]
+
+ dataloss_as_function_of_time = np.nanmean(dataloss, axis=(1, 2, 3))
+ dataloss_as_function_of_baseline = np.nanmean(dataloss, axis=(0, 2, 3))
+
+
+ flags_as_function_of_time = np.nanmean(flags, axis=(1,2,3))
+ flags_as_function_of_baseline = np.nanmean(flags, axis=(0,2,3))
+
+ metrics = dict(baselines=baselines,
+ dataloss_vs_time=dataloss_as_function_of_time.tolist(),
+ dataloss_vs_baseline=dataloss_as_function_of_baseline.tolist(),
+ flags_vs_time=flags_as_function_of_time.tolist(),
+ flags_vs_baseline=flags_as_function_of_baseline.tolist(),
+ timestamp=timestamp.tolist(),
+ sap=sap,
+ subband=sb
+ )
with open('metrics.json', 'w') as f_stream:
- pass
+ json.dump(metrics, f_stream)
+
+ with h5py.File('flags.h5', 'w') as flags_file:
+ flags_file['/FLAGS'] = flag_matrix
+ flags_file['/COVERAGE'] = coverage
+ flags_file['/DATALOSS'] = dataloss_matrix
+
+ flags_file['/'].attrs['REF_FREQUENCY'] = ref_frequency
+ flags_file['/'].attrs['BASELINES'] = baselines
+ flags_file['/'].attrs['PROJECT'] = observation.getcell('PROJECT', 0)
+ flags_file['/'].attrs['TELESCOPE'] = observation.getcell('TELESCOPE_NAME', 0)
+ flags_file['/'].attrs['SAS_ID'] = sas_id
+ flags_file['/'].attrs['SAP'] = sap
+ flags_file['/'].attrs['SUBBAND'] = sb
+
+ flags_file['/URANGE'] = urange
+ flags_file['/VRANGE'] = vrange
+ flags_file['/WRANGE'] = wrange
+
- class: DockerRequirement
- dockerPull: 'lofareosc/lofar-pipeline:latest'
+ dockerPull: 'astronsdc/lofar-ms-software:latest'
diff --git a/steps/plot_uvw_coverage.cwl b/steps/plot_uvw_coverage.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..f188063f70104a5060d8a6ef909c390ee6b7a59d
--- /dev/null
+++ b/steps/plot_uvw_coverage.cwl
@@ -0,0 +1,42 @@
+cwlVersion: v1.1
+class: CommandLineTool
+baseCommand:
+ - python3
+ - script.py
+inputs:
+ - id: input_file
+ type: File
+ inputBinding:
+ position: 0
+ - id: output_name
+ type: string?
+ default: 'uv_coverage.png'
+ inputBinding:
+ position: 1
+outputs:
+ - id: uv_plot
+ type: File
+ outputBinding:
+ glob: $(inputs.output_name)
+requirements:
+ - class: InlineJavascriptRequirement
+ - class: InitialWorkDirRequirement
+ listing:
+ - entryname: script.py
+ entry: |
+ import h5py
+ import matplotlib.pylab as plt
+ import sys
+ import numpy as np
+ from matplotlib import cm
+
+ h5_file = h5py.File(sys.argv[1], 'r')
+ output_name = sys.argv[2]
+ u = h5_file['URANGE']
+ v = h5_file['VRANGE']
+
+ plt.imshow(np.log10(h5_file['COVERAGE'][:, :, 0]), extent= [u[0], u[-1], v[0], v[-1]], cmap=cm.Blues)
+ plt.imshow(np.log10(h5_file['FLAGS'][:, :, 0]), extent= [u[0], u[-1], v[0], v[-1]] , cmap=cm.Reds)
+ plt.imshow(np.log10(h5_file['DATALOSS'][:, :, 0]), extent= [u[0], u[-1], v[0], v[-1]] , cmap=cm.Greens)
+
+ plt.savefig(output_name)
diff --git a/steps/read_surl_list.cwl b/steps/read_surl_list.cwl
index b1527d08845a46d2bc037dc98084d4fb8b7a85b3..43c7a42c2026956393e6804d2530aaeeffb3f325 100644
--- a/steps/read_surl_list.cwl
+++ b/steps/read_surl_list.cwl
@@ -1,5 +1,5 @@
class: ExpressionTool
-cwlVersion: v1.0
+cwlVersion: v1.1
id: read_surl_file
inputs:
- id: surl_list