Task #11188: performance boost in writing cache to disk. Side effect: speeds...

Task #11188: performance boost in writing cache to disk. Side effect: speeds up scheduling and webscheduler

SAS/DataManagement/StorageQueryService/cache.py

@@ -11,7 +11,6 @@ import ast
 from optparse import OptionParser
 from threading import Thread, RLock
 import os.path
-from pprint import pformat
 
 from lofar.messaging import EventMessage, ToBus
 from lofar.common.util import humanreadablesize
@@ -73,6 +72,7 @@ class CacheManager:
         self._cacheLock = RLock()
 
         self._cache = {'path_du_results': {}, 'otdb_id2path': {} }
+        self._last_cache_write_timestamp = datetime.datetime(1970, 1, 1)
         self._readCacheFromDisk()
 
         self.disk_usage = DiskUsage(mountpoint=mountpoint,
@@ -111,11 +111,16 @@ class CacheManager:
 
     def _writeCacheToDisk(self):
         try:
-            tmp_path = '/tmp/tmp_storagequery_cache.py'
-            with open(tmp_path, 'w') as file:
+            if datetime.datetime.utcnow() - self._last_cache_write_timestamp > datetime.timedelta(minutes=5):
+                tmp_path = '/tmp/tmp_storagequery_cache.py'
+                cache_str = ''
                 with self._cacheLock:
-                    file.write(pformat(self._cache) + '\n')
-            os.rename(tmp_path, self._cache_path)
+                    cache_str = str(self._cache)
+
+                with open(tmp_path, 'w') as file:
+                    file.write(cache_str)
+                os.rename(tmp_path, self._cache_path)
+                self._last_cache_write_timestamp = datetime.datetime.utcnow()
         except Exception as e:
             logger.error("Error while writing du cache: %s", e)