Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
LOFAR
Manage
Activity
Members
Labels
Plan
Issues
Wiki
Jira issues
Open Jira
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Locked files
Deploy
Releases
Package registry
Container Registry
Model registry
Operate
Environments
Terraform modules
Analyze
Value stream analytics
Contributor analytics
Repository analytics
Code review analytics
Insights
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
RadioObservatory
LOFAR
Commits
b5d56a74
Commit
b5d56a74
authored
6 years ago
by
Jorrit Schaap
Browse files
Options
Downloads
Patches
Plain Diff
SW-378
: processed review comment: added clarification comments
parent
4bb243ea
No related branches found
No related tags found
No related merge requests found
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
QA/QA_Common/bin/find_hdf5
+20
-0
20 additions, 0 deletions
QA/QA_Common/bin/find_hdf5
with
20 additions
and
0 deletions
QA/QA_Common/bin/find_hdf5
+
20
−
0
View file @
b5d56a74
...
@@ -86,10 +86,12 @@ if __name__ == '__main__':
...
@@ -86,10 +86,12 @@ if __name__ == '__main__':
logging
.
basicConfig
(
format
=
'
%(asctime)s %(levelname)s %(message)s
'
,
logging
.
basicConfig
(
format
=
'
%(asctime)s %(levelname)s %(message)s
'
,
level
=
logging
.
DEBUG
if
options
.
verbose
else
logging
.
WARN
)
level
=
logging
.
DEBUG
if
options
.
verbose
else
logging
.
WARN
)
# parse the options; if no specific filter for either obs or pipeline was given, then search for both.
if
not
options
.
observation
and
not
options
.
pipeline
:
if
not
options
.
observation
and
not
options
.
pipeline
:
options
.
observation
=
True
options
.
observation
=
True
options
.
pipeline
=
True
options
.
pipeline
=
True
# parse the options; if no specific filter for either lba or hba was given, then search for both.
if
options
.
lba
or
options
.
hba
:
if
options
.
lba
or
options
.
hba
:
options
.
observation
=
True
options
.
observation
=
True
options
.
pipeline
=
False
options
.
pipeline
=
False
...
@@ -105,6 +107,7 @@ if __name__ == '__main__':
...
@@ -105,6 +107,7 @@ if __name__ == '__main__':
info_dicts
=
{}
info_dicts
=
{}
# gather all info_dicts for all files...
for
file
in
files
:
for
file
in
files
:
try
:
try
:
info_dict
=
read_info_dict
(
file
)
info_dict
=
read_info_dict
(
file
)
...
@@ -113,52 +116,64 @@ if __name__ == '__main__':
...
@@ -113,52 +116,64 @@ if __name__ == '__main__':
except
:
except
:
pass
pass
# ...and filter out the files that do not match the search filters
if
not
(
options
.
observation
and
options
.
pipeline
):
if
not
(
options
.
observation
and
options
.
pipeline
):
if
options
.
observation
:
if
options
.
observation
:
# keep observations
files
=
[
f
for
f
in
files
files
=
[
f
for
f
in
files
if
f
in
info_dicts
and
'
observation
'
in
info_dicts
[
f
].
get
(
'
type
'
,
''
).
lower
()]
if
f
in
info_dicts
and
'
observation
'
in
info_dicts
[
f
].
get
(
'
type
'
,
''
).
lower
()]
if
options
.
pipeline
:
if
options
.
pipeline
:
# keep pipelines
files
=
[
f
for
f
in
files
files
=
[
f
for
f
in
files
if
f
in
info_dicts
and
'
pipeline
'
in
info_dicts
[
f
].
get
(
'
type
'
,
''
).
lower
()]
if
f
in
info_dicts
and
'
pipeline
'
in
info_dicts
[
f
].
get
(
'
type
'
,
''
).
lower
()]
if
not
(
options
.
lba
and
options
.
hba
):
if
not
(
options
.
lba
and
options
.
hba
):
if
options
.
lba
:
if
options
.
lba
:
# keep lba
files
=
[
f
for
f
in
files
files
=
[
f
for
f
in
files
if
f
in
info_dicts
and
'
lba
'
in
info_dicts
[
f
].
get
(
'
antenna_array
'
,
''
).
lower
()]
if
f
in
info_dicts
and
'
lba
'
in
info_dicts
[
f
].
get
(
'
antenna_array
'
,
''
).
lower
()]
if
options
.
hba
:
if
options
.
hba
:
# keep hba
files
=
[
f
for
f
in
files
files
=
[
f
for
f
in
files
if
f
in
info_dicts
and
'
hba
'
in
info_dicts
[
f
].
get
(
'
antenna_array
'
,
''
).
lower
()]
if
f
in
info_dicts
and
'
hba
'
in
info_dicts
[
f
].
get
(
'
antenna_array
'
,
''
).
lower
()]
if
options
.
name
:
if
options
.
name
:
# keep matching task names
files
=
[
f
for
f
in
files
if
f
in
info_dicts
and
files
=
[
f
for
f
in
files
if
f
in
info_dicts
and
fnmatch
.
fnmatch
(
info_dicts
[
f
].
get
(
'
name
'
,
''
).
lower
(),
options
.
name
.
lower
())]
fnmatch
.
fnmatch
(
info_dicts
[
f
].
get
(
'
name
'
,
''
).
lower
(),
options
.
name
.
lower
())]
if
options
.
project
:
if
options
.
project
:
# keep matching project names
files
=
[
f
for
f
in
files
if
f
in
info_dicts
and
files
=
[
f
for
f
in
files
if
f
in
info_dicts
and
(
fnmatch
.
fnmatch
(
info_dicts
[
f
].
get
(
'
project
'
,
''
).
lower
(),
options
.
project
.
lower
())
or
(
fnmatch
.
fnmatch
(
info_dicts
[
f
].
get
(
'
project
'
,
''
).
lower
(),
options
.
project
.
lower
())
or
fnmatch
.
fnmatch
(
info_dicts
[
f
].
get
(
'
project_description
'
,
''
).
lower
(),
options
.
project
.
lower
()))]
fnmatch
.
fnmatch
(
info_dicts
[
f
].
get
(
'
project_description
'
,
''
).
lower
(),
options
.
project
.
lower
()))]
if
options
.
date
:
if
options
.
date
:
# keep matching date
options
.
date
=
datetime
.
strptime
(
options
.
date
,
'
%Y-%m-%d
'
).
date
()
options
.
date
=
datetime
.
strptime
(
options
.
date
,
'
%Y-%m-%d
'
).
date
()
files
=
[
f
for
f
in
files
if
f
in
info_dicts
and
files
=
[
f
for
f
in
files
if
f
in
info_dicts
and
'
start_time
'
in
info_dicts
[
f
]
and
info_dicts
[
f
][
'
start_time
'
].
date
()
==
options
.
date
]
'
start_time
'
in
info_dicts
[
f
]
and
info_dicts
[
f
][
'
start_time
'
].
date
()
==
options
.
date
]
if
options
.
min_duration
:
if
options
.
min_duration
:
# keep matching duration
hours
,
sep
,
minutes
=
options
.
min_duration
.
partition
(
'
:
'
)
hours
,
sep
,
minutes
=
options
.
min_duration
.
partition
(
'
:
'
)
options
.
min_duration
=
timedelta
(
hours
=
int
(
hours
),
minutes
=
int
(
minutes
))
options
.
min_duration
=
timedelta
(
hours
=
int
(
hours
),
minutes
=
int
(
minutes
))
files
=
[
f
for
f
in
files
if
f
in
info_dicts
and
files
=
[
f
for
f
in
files
if
f
in
info_dicts
and
'
stop_time
'
in
info_dicts
[
f
]
and
info_dicts
[
f
][
'
stop_time
'
].
date
()
==
options
.
date
]
'
stop_time
'
in
info_dicts
[
f
]
and
info_dicts
[
f
][
'
stop_time
'
].
date
()
==
options
.
date
]
if
options
.
max_duration
:
if
options
.
max_duration
:
# keep matching duration
hours
,
sep
,
minutes
=
options
.
max_duration
.
partition
(
'
:
'
)
hours
,
sep
,
minutes
=
options
.
max_duration
.
partition
(
'
:
'
)
options
.
max_duration
=
timedelta
(
hours
=
int
(
hours
),
minutes
=
int
(
minutes
))
options
.
max_duration
=
timedelta
(
hours
=
int
(
hours
),
minutes
=
int
(
minutes
))
files
=
[
f
for
f
in
files
files
=
[
f
for
f
in
files
if
f
in
info_dicts
and
info_dicts
[
f
].
get
(
'
duration
'
,
timedelta
())
<=
options
.
max_duration
]
if
f
in
info_dicts
and
info_dicts
[
f
].
get
(
'
duration
'
,
timedelta
())
<=
options
.
max_duration
]
if
options
.
clusters
or
options
.
no_clusters
:
if
options
.
clusters
or
options
.
no_clusters
:
# keep matching have/havenot clusters
def
has_clusters
(
h5_path
):
def
has_clusters
(
h5_path
):
with
h5py
.
File
(
h5_path
,
"
r+
"
)
as
file
:
with
h5py
.
File
(
h5_path
,
"
r+
"
)
as
file
:
return
len
(
file
.
get
(
'
clustering
'
,{}))
return
len
(
file
.
get
(
'
clustering
'
,{}))
...
@@ -169,10 +184,15 @@ if __name__ == '__main__':
...
@@ -169,10 +184,15 @@ if __name__ == '__main__':
if
options
.
no_clusters
:
if
options
.
no_clusters
:
files
=
[
f
for
f
in
files
if
not
has_clusters
(
f
)]
files
=
[
f
for
f
in
files
if
not
has_clusters
(
f
)]
# the final files list now contains only the files matching all given filters.
# lastly, print the results...
if
options
.
info
:
if
options
.
info
:
# print the full file info
for
file
in
files
:
for
file
in
files
:
print
read_info_from_hdf5
(
file
,
read_data_info
=
False
)
print
read_info_from_hdf5
(
file
,
read_data_info
=
False
)
else
:
else
:
# just print the filtered filenames
print
'
\n
'
.
join
(
files
)
print
'
\n
'
.
join
(
files
)
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment