diff --git a/SAS/DataManagement/StorageQueryService/cache.py b/SAS/DataManagement/StorageQueryService/cache.py index 6201dec3ff87c2425ce360bf6e9982dcdf7f263c..8586205e5617bce5225a7f6a7c6b86755fff00c8 100644 --- a/SAS/DataManagement/StorageQueryService/cache.py +++ b/SAS/DataManagement/StorageQueryService/cache.py @@ -218,6 +218,9 @@ class CacheManager: def _scanProjectsTree(self): try: def addSubDirectoriesToCache(directory): + if not self._cacheThreadsRunning: + return + depth = self.getDepthToProjectsDir(directory) MAX_SCAN_DEPTH=2 #depth=0 : projects @@ -227,6 +230,17 @@ class CacheManager: if depth > MAX_SCAN_DEPTH: return + if depth < MAX_SCAN_DEPTH: + logger.info('tree scan: scanning \'%s\'', directory) + sd_result = self.disk_usage.path_resolver.getSubDirectories(directory) + + if sd_result['found']: + subdir_paths = [os.path.join(directory,sd) for sd in sd_result['sub_directories']] + + for subdir_path in subdir_paths: + # recurse + addSubDirectoriesToCache(subdir_path) + add_empty_du_result_to_cache = False with self._cacheLock: path_cache = self._cache['path_du_results'] @@ -243,20 +257,6 @@ class CacheManager: # mark cache entry for directory to be updated path_cache[directory]['needs_update'] = True - if not self._cacheThreadsRunning: - return - - if depth < MAX_SCAN_DEPTH: - logger.info('tree scan: scanning \'%s\'', directory) - sd_result = self.disk_usage.path_resolver.getSubDirectories(directory) - - if sd_result['found']: - subdir_paths = [os.path.join(directory,sd) for sd in sd_result['sub_directories']] - - for subdir_path in subdir_paths: - # recurse - addSubDirectoriesToCache(subdir_path) - addSubDirectoriesToCache(self.disk_usage.path_resolver.projects_path) logger.info('tree scan complete') @@ -290,12 +290,12 @@ class CacheManager: if not entry1.get('needs_update') and entry2.get('needs_update'): return 1 - depth1 = self.getDepthToProjectsDir(entry1['path']) - depth2 = self.getDepthToProjectsDir(entry2['path']) + #depth1 = self.getDepthToProjectsDir(entry1['path']) + #depth2 = self.getDepthToProjectsDir(entry2['path']) - if depth1 != depth2: - # lower level dirs are sorted in front of higher level dirs - return depth2 - depth1 + #if depth1 != depth2: + ## lower level dirs are sorted in front of higher level dirs + #return depth1 - depth2 if entry1['cache_timestamp'] < entry2['cache_timestamp']: return -1 @@ -536,7 +536,7 @@ class CacheManager: parallel_kwargs += [{'mom_id':mom_id, 'include_scratch_paths': include_scratch_paths, 'force_update':force_update} for mom_id in mom_ids] if otdb_ids: parallel_kwargs += [{'otdb_id':otdb_id, 'include_scratch_paths': include_scratch_paths, 'force_update':force_update} for otdb_id in otdb_ids] - results = executor.map(lambda p_kwarg: self.getDiskUsageForTask(**p_kwarg), parallel_kwargs) + results = list(executor.map(lambda p_kwarg: self.getDiskUsageForTask(**p_kwarg), parallel_kwargs)) for result in results: if result.get('radb_id') is not None: @@ -553,7 +553,7 @@ class CacheManager: def getDiskUsageForPaths(self, paths, force_update=False): with futures.ThreadPoolExecutor(max_workers=os.cpu_count()) as executor: parallel_args = [(path, force_update) for path in paths] - results = executor.map(lambda p_arg: self.getDiskUsageForPath(*p_arg), parallel_args) + results = list(executor.map(lambda p_arg: self.getDiskUsageForPath(*p_arg), parallel_args)) return { result['path']:result for result in results } def getDiskUsageForPath(self, path, force_update=False):