Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
V
VLBI-cwl
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Iterations
Wiki
Requirements
Jira
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Locked files
Build
Pipelines
Jobs
Pipeline schedules
Test cases
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Code review analytics
Issue analytics
Insights
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
ResearchAndDevelopment
VLBI-cwl
Commits
b172b8f9
Commit
b172b8f9
authored
1 month ago
by
Matthijs van der Wild
Browse files
Options
Downloads
Patches
Plain Diff
Move check_station_mismatch logic from CWL file to script
parent
89f5f290
Branches
Branches containing commit
Tags
Tags containing commit
1 merge request
!92
Move check_station_mismatch logic from CWL file to script
Changes
3
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
scripts/compareStationListVLBI.py
+69
-32
69 additions, 32 deletions
scripts/compareStationListVLBI.py
steps/check_station_mismatch.cwl
+14
-34
14 additions, 34 deletions
steps/check_station_mismatch.cwl
tests/test_scripts.py
+1
-1
1 addition, 1 deletion
tests/test_scripts.py
with
84 additions
and
67 deletions
scripts/compareStationListVLBI.py
100644 → 100755
+
69
−
32
View file @
b172b8f9
#from lofarpipe.support.data_map import DataMap, DataProduct
#!/usr/bin/env python3
import
argparse
import
json
import
logging
import
casacore.tables
as
pt
import
casacore.tables
as
pt
from
losoto.h5parm
import
h5parm
from
losoto.h5parm
import
h5parm
import
logging
def
plugin_main
(
args
,
**
kwargs
):
def
plugin_main
(
args
,
**
kwargs
):
"""
"""
...
@@ -22,40 +27,36 @@ def plugin_main(args, **kwargs):
...
@@ -22,40 +27,36 @@ def plugin_main(args, **kwargs):
result : dict
result : dict
Output station names to filter
Output station names to filter
"""
"""
#mapfile_in = kwargs['mapfile_in']
h5parmdb
=
kwargs
[
"
h5parmdb
"
]
h5parmdb
=
kwargs
[
'
h5parmdb
'
]
solset_name
=
kwargs
[
"
solset_name
"
]
solset_name
=
kwargs
[
'
solset_name
'
]
filter
=
kwargs
[
"
filter
"
]
filter
=
kwargs
[
'
filter
'
]
mslist
=
args
#data = DataMap.load(mapfile_in)
#mslist = [data[i].file for i in xrange(len(data))]
mslist
=
args
#['mss']
if
len
(
mslist
)
==
0
:
if
len
(
mslist
)
==
0
:
raise
ValueError
(
"
Did not find any existing directory in input MS list!
"
)
raise
ValueError
(
"
Did not find any existing directory in input MS list!
"
)
pass
else
:
else
:
MS
=
mslist
[
0
]
MS
=
mslist
[
0
]
pass
## reading ANTENNA table of MS
## reading ANTENNA table of MS
logging
.
info
(
'
Collecting information from the ANTENNA table.
'
)
logging
.
info
(
"
Collecting information from the ANTENNA table.
"
)
antennaTable
=
pt
.
table
(
MS
+
"
::ANTENNA
"
,
ack
=
False
)
antennaTable
=
pt
.
table
(
MS
+
"
::ANTENNA
"
,
ack
=
False
)
antennaNames
=
antennaTable
.
getcol
(
'
NAME
'
)
antennaNames
=
antennaTable
.
getcol
(
"
NAME
"
)
if
solset_name
==
'
vlbi
'
:
if
solset_name
==
"
vlbi
"
:
## reading in h5parm
## reading in h5parm
data
=
h5parm
(
h5parmdb
,
readonly
=
True
)
data
=
h5parm
(
h5parmdb
,
readonly
=
True
)
## reading ANTENNA information from target / phase
## reading ANTENNA information from target / phase
target
=
data
.
getSolset
(
'
target
'
)
target
=
data
.
getSolset
(
"
target
"
)
names
=
target
.
getSoltabNames
()
names
=
target
.
getSoltabNames
()
phstab
=
[
xx
for
xx
in
names
if
'
RMextract
'
not
in
xx
][
0
]
phstab
=
[
xx
for
xx
in
names
if
"
RMextract
"
not
in
xx
][
0
]
soltab
=
target
.
getSoltab
(
phstab
)
soltab
=
target
.
getSoltab
(
phstab
)
phsants
=
soltab
.
getAxisValues
(
'
ant
'
)
phsants
=
soltab
.
getAxisValues
(
"
ant
"
)
dutch_ants
=
[
xx
for
xx
in
phsants
if
'
CS
'
in
xx
or
'
RS
'
in
xx
]
dutch_ants
=
[
xx
for
xx
in
phsants
if
"
CS
"
in
xx
or
"
RS
"
in
xx
]
## reading ANTENNA information from calibrator
## reading ANTENNA information from calibrator
solset
=
data
.
getSolset
(
'
calibrator
'
)
solset
=
data
.
getSolset
(
"
calibrator
"
)
station_names
=
solset
.
getAnt
().
keys
()
station_names
=
solset
.
getAnt
().
keys
()
int_ants
=
[
xx
for
xx
in
station_names
if
'
CS
'
not
in
xx
and
'
RS
'
not
in
xx
]
int_ants
=
[
xx
for
xx
in
station_names
if
"
CS
"
not
in
xx
and
"
RS
"
not
in
xx
]
cal_dutch
=
[
xx
for
xx
in
station_names
if
'
CS
'
in
xx
or
'
RS
'
in
xx
]
cal_dutch
=
[
xx
for
xx
in
station_names
if
"
CS
"
in
xx
or
"
RS
"
in
xx
]
## remove core/remote stations not present for calibrator
## remove core/remote stations not present for calibrator
all_dutch_ants
=
[
xx
for
xx
in
dutch_ants
if
xx
in
cal_dutch
]
all_dutch_ants
=
[
xx
for
xx
in
dutch_ants
if
xx
in
cal_dutch
]
station_names
=
all_dutch_ants
+
int_ants
station_names
=
all_dutch_ants
+
int_ants
...
@@ -67,14 +68,50 @@ def plugin_main(args, **kwargs):
...
@@ -67,14 +68,50 @@ def plugin_main(args, **kwargs):
## check whether there are more stations in the target than in the calibrator solutions
## check whether there are more stations in the target than in the calibrator solutions
missing_stations
=
list
(
set
(
antennaNames
)
-
set
(
station_names
))
missing_stations
=
list
(
set
(
antennaNames
)
-
set
(
station_names
))
for
missing_station
in
missing_stations
:
for
missing_station
in
missing_stations
:
filter
+=
'
;!
'
+
missing_station
+
'
*
'
filter
+=
"
;!
"
+
missing_station
+
"
*
"
pass
#filter = filter.lstrip(';')
data
.
close
()
data
.
close
()
## return results
return
str
(
filter
)
result
=
{
'
filter
'
:
str
(
filter
)}
return
(
result
)
def
parse_arguments
():
parser
=
argparse
.
ArgumentParser
()
parser
.
add_argument
(
"
mss
"
,
help
=
"
Input data in MeasurementSet format
"
,
nargs
=
"
*
"
,
type
=
str
)
parser
.
add_argument
(
"
--solset
"
,
help
=
"
The solution set from the LINC pipeline
"
,
type
=
str
,
const
=
""
,
nargs
=
"
?
"
,
)
parser
.
add_argument
(
"
--solset_name
"
,
help
=
"
Name of the solution set
"
,
type
=
str
)
parser
.
add_argument
(
"
--filter_baselines
"
,
help
=
"
Filter constrains for the dp3_prep_target step
"
,
type
=
str
,
)
return
parser
.
parse_args
()
def
main
():
arguments
=
parse_arguments
()
filter_out
=
plugin_main
(
arguments
.
mss
,
h5parmdb
=
arguments
.
solset
,
solset_name
=
arguments
.
solset_name
,
filter
=
arguments
.
filter_baselines
)
with
open
(
"
./out.json
"
,
"
w
"
)
as
fp
:
json
.
dump
({
"
filter_out
"
:
filter_out
},
fp
)
pass
if
__name__
==
"
__main__
"
:
main
()
This diff is collapsed.
Click to expand it.
steps/check_station_mismatch.cwl
+
14
−
34
View file @
b172b8f9
...
@@ -7,30 +7,40 @@ doc: |
...
@@ -7,30 +7,40 @@ doc: |
against the list of station in the solution file and ensures
against the list of station in the solution file and ensures
both are consistent.
both are consistent.
baseCommand:
baseCommand: compareStationListVLBI.py
- python3
- compare_station_list.py
inputs:
inputs:
- id: msin
- id: msin
type: Directory[]
type: Directory[]
doc: Input MeasurementSets.
doc: Input MeasurementSets.
inputBinding:
inputBinding:
position:
0
position:
1
- id: solset
- id: solset
type: File
type: File
doc: The solution set from the LINC pipeline.
doc: The solution set from the LINC pipeline.
inputBinding:
position: 0
prefix: --solset
separate: true
- id: solset_name
- id: solset_name
type: string?
type: string?
doc: Name of the solution set.
doc: Name of the solution set.
default: vlbi
default: vlbi
inputBinding:
position: 0
prefix: --solset_name
separate: true
- id: filter_baselines
- id: filter_baselines
type: string?
type: string?
default: "*&"
default: "*&"
doc: Filter constrains for the dp3_prep_target step.
doc: Filter constrains for the dp3_prep_target step.
inputBinding:
position: 0
prefix: --filter_baselines
separate: true
outputs:
outputs:
- id: filter_out
- id: filter_out
...
@@ -53,36 +63,6 @@ outputs:
...
@@ -53,36 +63,6 @@ outputs:
requirements:
requirements:
- class: InlineJavascriptRequirement
- class: InlineJavascriptRequirement
- class: InitialWorkDirRequirement
listing:
- entryname: compare_station_list.py
entry: |
import sys
import json
import yaml
import os
from compareStationListVLBI import plugin_main as compareStationList
mss = sys.argv[1:]
try:
inputs = json.loads(r"""$(inputs)""")
except:
inputs = yaml.loads(r"""$(inputs)""")
h5parmdb = inputs['solset']['path']
solset_name = inputs['solset_name']
filter = inputs['filter_baselines']
print(mss)
output = compareStationList(mss,
h5parmdb = h5parmdb,
solset_name = solset_name,
filter = filter)
filter_out = output['filter']
cwl_output = {"filter_out": filter_out}
with open('./out.json', 'w') as fp:
json.dump(cwl_output, fp)
hints:
hints:
- class: DockerRequirement
- class: DockerRequirement
...
...
This diff is collapsed.
Click to expand it.
tests/test_scripts.py
+
1
−
1
View file @
b172b8f9
...
@@ -44,7 +44,7 @@ def test_skynet():
...
@@ -44,7 +44,7 @@ def test_skynet():
def
test_compare_stations
():
def
test_compare_stations
():
import
glob
import
glob
reference
=
{
"
filter
"
:
"
*&
"
}
reference
=
"
*&
"
filter
=
"
*&
"
filter
=
"
*&
"
solset
=
f
"
{
data_dir
}
/results_target/cal_solutions.h5
"
solset
=
f
"
{
data_dir
}
/results_target/cal_solutions.h5
"
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment