Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
V
VLBI-cwl
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Iterations
Wiki
Requirements
Jira
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Locked files
Build
Pipelines
Jobs
Pipeline schedules
Test cases
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Code review analytics
Issue analytics
Insights
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
ResearchAndDevelopment
VLBI-cwl
Commits
45ce1906
Commit
45ce1906
authored
1 month ago
by
Matthijs van der Wild
Browse files
Options
Downloads
Plain Diff
Merge branch '
RAP-1065
-findRefAnt_join' into 'master'
move logic in findRefAnt_join.cwl to separate library Closes
RAP-1065
See merge request
!91
parents
e5045d19
1b22c1da
No related branches found
No related tags found
1 merge request
!91
move logic in findRefAnt_join.cwl to separate library
Pipeline
#120151
passed
1 month ago
Stage: version
Stage: test
Changes
2
Pipelines
1
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
scripts/findRefAnt_join.py
+87
-0
87 additions, 0 deletions
scripts/findRefAnt_join.py
steps/findRefAnt_join.cwl
+14
-61
14 additions, 61 deletions
steps/findRefAnt_join.cwl
with
101 additions
and
61 deletions
scripts/findRefAnt_join.py
0 → 100755
+
87
−
0
View file @
45ce1906
#!/usr/bin/env python3
import
argparse
import
ast
import
json
import
re
import
sys
def
main
(
flagged_fraction_dict
,
station_filter
,
state
):
with
open
(
flagged_fraction_dict
,
"
r
"
)
as
f_stream
:
flagged_fraction_dict_list
=
json
.
load
(
f_stream
)
no_station_selected
=
True
while
no_station_selected
:
print
(
"
Applying station filter
"
+
str
(
station_filter
))
flagged_fraction_data
=
{}
no_station_selected
=
False
for
flagged_fraction_dict
in
flagged_fraction_dict_list
:
entry
=
ast
.
literal_eval
(
flagged_fraction_dict
)
antennas
=
entry
.
keys
()
selected_stations
=
[
station_name
for
station_name
in
antennas
if
re
.
match
(
station_filter
,
station_name
)
]
if
len
(
selected_stations
)
==
0
:
print
(
"
No stations left after filtering.
"
+
"
Station(s) do(es) not exist in all subbands.
"
+
"
No filter is used.
"
)
station_filter
=
""
no_station_selected
=
True
break
for
antenna
in
selected_stations
:
try
:
flagged_fraction_data
[
antenna
].
append
(
float
(
entry
[
antenna
]))
except
KeyError
:
flagged_fraction_data
[
antenna
]
=
[
float
(
entry
[
antenna
])]
flagged_fraction_list
=
[]
sorted_stations
=
sorted
(
flagged_fraction_data
.
keys
())
flagged_fraction_antenna
=
{}
for
antenna
in
sorted_stations
:
flagged_fraction
=
sum
(
flagged_fraction_data
[
antenna
])
/
len
(
flagged_fraction_data
[
antenna
]
)
flagged_fraction_list
.
append
(
flagged_fraction
)
flagged_fraction_antenna
[
antenna
]
=
flagged_fraction
try
:
flagged_fraction_data
[
flagged_fraction
].
append
(
antenna
)
except
KeyError
:
flagged_fraction_data
[
flagged_fraction
]
=
[
antenna
]
min_flagged_fraction
=
min
(
flagged_fraction_list
)
refant
=
flagged_fraction_data
[
min_flagged_fraction
][
0
]
print
(
"
Selected station
"
+
str
(
refant
)
+
"
as reference antenna. Fraction of flagged data is
"
+
"
{:>3}
"
.
format
(
"
{:.1f}
"
.
format
(
100
*
min_flagged_fraction
)
+
"
%
"
)
)
flagged_fraction_antenna
[
"
state
"
]
=
state
cwl_output
=
{
"
refant
"
:
str
(
refant
)}
with
open
(
"
./out.json
"
,
"
w
"
)
as
fp
:
json
.
dump
(
cwl_output
,
fp
)
with
open
(
"
./flagged_fraction_antenna.json
"
,
"
w
"
)
as
fp
:
json
.
dump
(
flagged_fraction_antenna
,
fp
)
if
__name__
==
"
__main__
"
:
parser
=
argparse
.
ArgumentParser
()
parser
.
add_argument
(
"
--flagged_fraction_dict
"
,
type
=
str
)
parser
.
add_argument
(
"
--station_filter
"
,
type
=
str
,
const
=
""
,
nargs
=
"
?
"
)
parser
.
add_argument
(
"
--state
"
,
type
=
str
)
arguments
=
parser
.
parse_args
()
main
(
arguments
.
flagged_fraction_dict
,
arguments
.
station_filter
,
arguments
.
state
)
This diff is collapsed.
Click to expand it.
steps/findRefAnt_join.cwl
+
14
−
61
View file @
45ce1906
...
...
@@ -7,9 +7,11 @@ doc: |
and outputs the fraction of flagged data and the
station name with the least amount of flagged data.
baseCommand:
- python3
- findRefAnt_join.py
baseCommand: findRefAnt_join.py
arguments:
- valueFrom: input.json
prefix: --flagged_fraction_dict
inputs:
- id: flagged_fraction_dict
...
...
@@ -20,11 +22,17 @@ inputs:
- id: filter_station
type: string?
default: '*&'
inputBinding:
position: 1
prefix: --station_filter
doc: A regular expression pattern for station names to select.
- id: state
type: string?
default: 'NONE'
inputBinding:
position: 2
prefix: --state
doc: State information for the collection of antenna statistics.
outputs:
...
...
@@ -60,65 +68,10 @@ requirements:
listing:
- entryname: input.json
entry: $(inputs.flagged_fraction_dict)
- entryname: findRefAnt_join.py
entry: |
import sys
import json
import re
import ast
inputs = json.loads(r"""$(inputs)""")
with open('input.json', 'r') as f_stream:
flagged_fraction_dict_list = json.load(f_stream)
filter_station = inputs['filter_station']
no_station_selected = True
while no_station_selected:
print('Applying station filter ' + str(filter_station))
flagged_fraction_data = {}
no_station_selected = False
for flagged_fraction_dict in flagged_fraction_dict_list:
entry = ast.literal_eval(flagged_fraction_dict)
antennas = entry.keys()
selected_stations = [ station_name for station_name in antennas if re.match(filter_station, station_name) ]
if len(selected_stations) == 0:
print('No stations left after filtering. Station(s) do(es) not exist in all subbands. No filter is used.')
filter_station = ''
no_station_selected = True
break
for antenna in selected_stations:
try:
flagged_fraction_data[antenna].append(float(entry[antenna]))
except KeyError:
flagged_fraction_data[antenna] = [float(entry[antenna])]
flagged_fraction_list = []
sorted_stations = sorted(flagged_fraction_data.keys())
flagged_fraction_antenna = {}
for antenna in sorted_stations:
flagged_fraction = sum(flagged_fraction_data[antenna]) / len(flagged_fraction_data[antenna])
flagged_fraction_list.append(flagged_fraction)
flagged_fraction_antenna[antenna] = flagged_fraction
try:
flagged_fraction_data[flagged_fraction].append(antenna)
except KeyError:
flagged_fraction_data[flagged_fraction] = [antenna]
min_flagged_fraction = min(flagged_fraction_list)
refant = flagged_fraction_data[min_flagged_fraction][0]
print('Selected station ' + str(refant) + ' as reference antenna. Fraction of flagged data is ' + '{:>3}'.format('{:.1f}'.format(100 * min_flagged_fraction) + '%'))
flagged_fraction_antenna['state'] = inputs['state']
cwl_output = {'refant': str(refant)}
with open('./out.json', 'w') as fp:
json.dump(cwl_output, fp)
with open('./flagged_fraction_antenna.json', 'w') as fp:
json.dump(flagged_fraction_antenna, fp)
hints:
- class: DockerRequirement
dockerPull: vlbi-cwl
stdout: findRefAnt.log
stderr: findRefAnt_err.log
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment