SW-448: merge back from...

SW-448: merge back from https://svn.astron.nl/LOFAR/branches/SW-448-robinhood-bugfix. Use normal du for now instead of robinhood-du. Easy switch back to rbh-du in october possible. Made queries and scans faster.

SAS/DataManagement/StorageQueryService/cache.py

@@ -307,20 +307,24 @@ class CacheManager:
 
                     for i, cache_entry in enumerate(updateable_entries):
                         try:
-                            path = cache_entry.get('path')
-                            if path:
-                                logger.info('_updateOldEntriesInCache: examining entry %s/%s. timestamp:%s age:%s needs_update:%s path: \'%s\'',
-                                            i,
-                                            len(updateable_entries),
-                                            cache_entry['cache_timestamp'],
-                                            format_timedelta(now - cache_entry['cache_timestamp']),
-                                            cache_entry.get('needs_update', False),
-                                            path)
-
-                                #du a full update from disk, which might be (really) slow.
-                                result = du_getDiskUsageForPath(path)
-                                logger.debug('trying to update old entry in cache: %s', result)
-                                self._updateCache(result)
+                            # it might be that the cache_entry was already updated via another way
+                            # so only update it if still to old or needs_update
+                            now = datetime.datetime.utcnow()
+                            if now - cache_entry['cache_timestamp'] > MAX_CACHE_ENTRY_AGE or cache_entry.get('needs_update', False):
+                                path = cache_entry.get('path')
+                                if path:
+                                    logger.info('_updateOldEntriesInCache: examining entry %s/%s. timestamp:%s age:%s needs_update:%s path: \'%s\'',
+                                                i,
+                                                len(updateable_entries),
+                                                cache_entry['cache_timestamp'],
+                                                format_timedelta(now - cache_entry['cache_timestamp']),
+                                                cache_entry.get('needs_update', False),
+                                                path)
+
+                                    #du a full update from disk, which might be (really) slow.
+                                    result = du_getDiskUsageForPath(path)
+                                    logger.debug('trying to update old entry in cache: %s', result)
+                                    self._updateCache(result)
                         except Exception as e:
                             logger.error(str(e))
 
@@ -328,8 +332,8 @@ class CacheManager:
                             logger.info('exiting _updateCacheThread')
                             return
 
-                        if datetime.datetime.utcnow() - cacheUpdateStart > datetime.timedelta(minutes=5):
-                            # break out of cache update loop if full update takes more than 5min
+                        if datetime.datetime.utcnow() - cacheUpdateStart > datetime.timedelta(minutes=10):
+                            # break out of cache update loop if full update takes more than 1min
                             # next loop we'll start with the oldest cache entries again
                             logger.info('skipping remaining %s old cache entries updates, they will be updated next time', len(updateable_entries)-i)
                             break
@@ -338,7 +342,7 @@ class CacheManager:
                 self._updateCEP4CapacitiesInRADB()
 
                 #sleep for a while, (or stop if requested)
-                for i in range(30):
+                for i in range(10):
                     sleep(1)
                     if not self._cacheThreadsRunning:
                         logger.info('exiting _updateCacheThread')