SSB-47: improved logging and skipping the regex check for directories with a...

SSB-47: improved logging and skipping the regex check for directories with a nesting level higher than 2 (hence all the files that compose the MS).

CAL/CalibrationCommon/lib/datacontainers/holography_observation.py

@@ -92,6 +92,11 @@ class HolographyObservation():
         self.source_name = source_name
 
 
+def nesting_level_is_reached(starting_path, path, level):
+    level_path = os.path.relpath(path, starting_path).count(os.sep)
+    return level_path >= level
+
+
 def list_observations_in_path(path):
     """
     List all the observations in the given path and return a list of HolographyObservation
@@ -106,6 +111,8 @@ def list_observations_in_path(path):
     ms_dirs_path_pattern = '^' + os.path.join(path, ms_dir_name_pattern, 'uv$')
     observations_list = []
     for root, dirnames, filenames in os.walk(path, followlinks=True):
+        if nesting_level_is_reached(path, root, 2):
+            continue
         match = re.match(ms_dirs_path_pattern, root)
         if match:
             sas_id = match.group('sas_id')
@@ -150,10 +157,12 @@ def list_observations_in_path(path):
                                           start_mjd_in_seconds, end_mjd_in_seconds, sub_band,
                                           frequency,
                                           source_name))
+                logging.debug('Processing ms in path %s ... done', root)
             except Exception as e:
                 logger.error('Observation in %s is invalid skipping... ', root)
                 logger.exception(e)
-
+        else:
+            logger.debug('skipping directory %s', root)
     logger.info("Holography observations were successfully loaded from \"%s\".", path)
     return observations_list