From f085aa19c6076c51a04a2f306137ca30622cd73f Mon Sep 17 00:00:00 2001
From: mancini <mancini@astron.nl>
Date: Wed, 14 Aug 2019 11:20:45 +0200
Subject: [PATCH] SSB-47: improved logging and skipping the regex check for
directories with a nesting level higher than 2 (hence all the files that
compose the MS).
---
.../lib/datacontainers/holography_observation.py | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/CAL/CalibrationCommon/lib/datacontainers/holography_observation.py b/CAL/CalibrationCommon/lib/datacontainers/holography_observation.py
index 78daff48426..5b4986b6dd9 100644
--- a/CAL/CalibrationCommon/lib/datacontainers/holography_observation.py
+++ b/CAL/CalibrationCommon/lib/datacontainers/holography_observation.py
@@ -92,6 +92,11 @@ class HolographyObservation():
self.source_name = source_name
+def nesting_level_is_reached(starting_path, path, level):
+ level_path = os.path.relpath(path, starting_path).count(os.sep)
+ return level_path >= level
+
+
def list_observations_in_path(path):
"""
List all the observations in the given path and return a list of HolographyObservation
@@ -106,6 +111,8 @@ def list_observations_in_path(path):
ms_dirs_path_pattern = '^' + os.path.join(path, ms_dir_name_pattern, 'uv$')
observations_list = []
for root, dirnames, filenames in os.walk(path, followlinks=True):
+ if nesting_level_is_reached(path, root, 2):
+ continue
match = re.match(ms_dirs_path_pattern, root)
if match:
sas_id = match.group('sas_id')
@@ -150,10 +157,12 @@ def list_observations_in_path(path):
start_mjd_in_seconds, end_mjd_in_seconds, sub_band,
frequency,
source_name))
+ logging.debug('Processing ms in path %s ... done', root)
except Exception as e:
logger.error('Observation in %s is invalid skipping... ', root)
logger.exception(e)
-
+ else:
+ logger.debug('skipping directory %s', root)
logger.info("Holography observations were successfully loaded from \"%s\".", path)
return observations_list
--
GitLab