Experimenting with identifying issues and conditional fixes.

.gitignore

+# Virtual env
+.venv
\ No newline at end of file

README.md

+# Lofar imaging compression pipeline
+
+This is a CWL workflow used to compress Lofar imaging data.
+
+## Workflow steps
+
+**TODO**
+
+## Requirements
+
+- CWL v1.2 compatible runner
+- Docker
+
+## Running the workflow
+
+```bash
+# Build the docker images
+./build_images.sh
+# Run the workflow
+cwltool compress_pipeline.cwl [--flag_autocorrelation] --msin MEASUREMENT_SET 
+```

build_images.sh

+#!/usr/bin/bash
+docker build -t imaging-compress-pipeline-identify-issues -f docker/Dockerfile.identify_issues scripts/
\ No newline at end of file

conditonal_fix.cwl

+class: Workflow
+cwlVersion: v1.2
+
+inputs:
+  msin: Directory
+outputs: []
+
+steps:
+  identify_issues:
+    run: steps/identify_issues.cwl
+    in:
+      msin: msin
+    out:
+      - issue_list
+  fix_a:
+    run: steps/dummy_fix_a.cwl
+    when: $( inputs.list.includes("FIX_ANTENNA_TABLE") )
+    in:
+      list: identify_issues/issue_list
+    out: []
+  fix_b:
+    run: steps/dummy_fix_b.cwl
+    when: $( inputs.list.includes("FIX_WEIGHT_SPECTRUM") )
+    in:
+      list: identify_issues/issue_list
+    out: []
+  fix_c:
+    run: steps/dummy_fix_a.cwl
+    when: $( inputs.list.includes("FIX_BROKEN_TILES") )
+    in:
+      list: identify_issues/issue_list
+    out: []
+  fix_d:
+    run: steps/dummy_fix_b.cwl
+    when: $( inputs.list.includes("FIX_STATION_ADDER") )
+    in:
+      list: identify_issues/issue_list
+    out: []
+
+requirements:
+  InlineJavascriptRequirement: {}

docker/Dockerfile.identify_issues

+FROM python:3.9-slim
+
+WORKDIR /scripts
+COPY requirements.txt /scripts/
+RUN pip install -r requirements.txt
+COPY fix_common_ms_issues.py /scripts/
\ No newline at end of file

scripts/fix_common_ms_issues.py

+import json
 from argparse import ArgumentParser
-from casacore.tables import table as MSTable
-from astropy.time import Time
-from datetime import timezone as tz
 from datetime import datetime
+from datetime import timezone as tz
+
+from astropy.time import Time
+from casacore.tables import table as MSTable
 
 _DAY_IN_SECONDS = 24 * 60 * 60
 
@@ -31,10 +33,10 @@ def date_to_fix_string(obs_date):
     for fix_string, (start_date, end_date) in _TIME_RANGES.items():
         if obs_date > start_date and obs_date < end_date:
             fixes.append(fix_string)
-    return fixes
+    return json.dumps(fixes)    # dump proper json for CWL
 
 def check_observing_date(msin):
-    observation_table = MSTable(f'{msin}/OBSERVATION')
+    observation_table = MSTable(f'{msin}/OBSERVATION', ack=False)
     start_mjd, end_mjd = observation_table.getcell('TIME_RANGE', 0)
     start_datetime, end_datetime = map(mjd_to_local, (start_mjd, end_mjd))
     return start_datetime, end_datetime

scripts/requirements.txt

+astropy
+python-casacore
\ No newline at end of file

steps/dummy_fix_a.cwl

+id: dummy_fix_a
+label: Dummy Fix A
+cwlVersion: v1.0
+class: CommandLineTool
+baseCommand: echo
+arguments:
+    - "A"
+inputs: []
+outputs: []

steps/dummy_fix_b.cwl

+id: dummy_fix_b
+label: Dummy Fix B
+cwlVersion: v1.0
+class: CommandLineTool
+baseCommand: echo
+arguments:
+    - "B"
+inputs: []
+outputs: []

steps/identify_issues.cwl

+id: identify_issues
+label: Identify Known issues
+class: CommandLineTool
+cwlVersion: v1.0
+hints:
+  DockerRequirement:
+    dockerImageId: imaging-compress-pipeline-identify-issues
+
+inputs:
+  - id: msin
+    type: Directory
+    inputBinding:
+      position: 1
+
+outputs:
+  - id: issue_list
+    type: string[]
+    outputBinding:
+      glob: output.txt
+      loadContents: true
+      outputEval: $(JSON.parse(self[0].contents))
+stdout: output.txt
+
+baseCommand:
+  - python3
+  - /scripts/fix_common_ms_issues.py
+
+requirements:
+  InlineJavascriptRequirement: {}