From db7cfe193608ba3c1e45c1d5eb47e1f90b35384e Mon Sep 17 00:00:00 2001
From: Jorrit Schaap <schaap@astron.nl>
Date: Fri, 24 Jun 2016 10:26:44 +0000
Subject: [PATCH] Task #9350: factored out diskusage in seperate module
---
.gitattributes | 1 +
.../StorageQueryService/CMakeLists.txt | 1 +
.../StorageQueryService/diskusage.py | 79 +++++++++++++++++++
SAS/DataManagement/StorageQueryService/rpc.py | 13 ++-
.../StorageQueryService/service.py | 44 +----------
5 files changed, 93 insertions(+), 45 deletions(-)
create mode 100644 SAS/DataManagement/StorageQueryService/diskusage.py
diff --git a/.gitattributes b/.gitattributes
index 914ea2e774d..404af8e4163 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -4750,6 +4750,7 @@ SAS/DataManagement/DataManagementCommon/path.py -text
SAS/DataManagement/StorageQueryService/CMakeLists.txt -text
SAS/DataManagement/StorageQueryService/__init__.py -text
SAS/DataManagement/StorageQueryService/config.py -text
+SAS/DataManagement/StorageQueryService/diskusage.py -text
SAS/DataManagement/StorageQueryService/rpc.py -text
SAS/DataManagement/StorageQueryService/service.py -text
SAS/DataManagement/StorageQueryService/storagequery -text
diff --git a/SAS/DataManagement/StorageQueryService/CMakeLists.txt b/SAS/DataManagement/StorageQueryService/CMakeLists.txt
index 1bf03b6fb94..a7f354d838c 100644
--- a/SAS/DataManagement/StorageQueryService/CMakeLists.txt
+++ b/SAS/DataManagement/StorageQueryService/CMakeLists.txt
@@ -10,6 +10,7 @@ set(_py_files
config.py
rpc.py
service.py
+ diskusage.py
)
python_install(${_py_files} DESTINATION lofar/sas/datamanagement/storagequery)
diff --git a/SAS/DataManagement/StorageQueryService/diskusage.py b/SAS/DataManagement/StorageQueryService/diskusage.py
new file mode 100644
index 00000000000..5d4abb47bfc
--- /dev/null
+++ b/SAS/DataManagement/StorageQueryService/diskusage.py
@@ -0,0 +1,79 @@
+#!/usr/bin/python
+# $Id$
+
+import logging
+import subprocess
+import socket
+from optparse import OptionParser
+from lofar.common.util import humanreadablesize
+
+logger = logging.getLogger(__name__)
+
+def getDiskUsageForPath(path):
+ cmd = ['rbh-du', '-bd', path]
+ hostname = socket.gethostname()
+ if not 'mgmt0' in hostname:
+ cmd = ['ssh', '-AXt', 'lofarsys@mgmt01.cep4.control.lofar'] + cmd
+ logger.info(' '.join(cmd))
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ out, err = proc.communicate()
+
+ logger.info(out)
+
+ if proc.returncode != 0:
+ logger.error(out)
+ return {'found': False, 'path': path, 'message': out}
+
+ # example of out
+ # Using config file '/etc/robinhood.d/tmpfs/tmp_fs_mgr_basic.conf'.
+ # /data/projects/2016LOFAROBS/L522380
+ # dir count:3906, size:16048128, spc_used:16052224
+ # file count:17568, size:42274164368, spc_used:42327519232
+
+ #parse out
+ lines = [l.strip() for l in out.split('\n')]
+ file_line = next(l for l in lines if 'file count' in l)
+ if file_line:
+ parts = [p.strip() for p in file_line.split(',')]
+ partsDict = {p.split(':')[0].strip():p.split(':')[1].strip() for p in parts}
+
+ result = {'found': True, 'disk_usage': None, 'path': path}
+
+ if 'size' in partsDict:
+ result['disk_usage'] = int(partsDict['size'])
+
+ if 'file count' in partsDict:
+ result['nr_of_files'] = int(partsDict['file count'])
+
+ result['disk_usage_readable'] = humanreadablesize(result['disk_usage'])
+ logger.info('returning: %s' % result)
+ return result
+
+ result = {'found': False, 'path': path }
+
+def main():
+ # Check the invocation arguments
+ parser = OptionParser("%prog [options] <path>",
+ description='get disk usage for (cep4) path')
+ parser.add_option('-V', '--verbose', dest='verbose', action='store_true', help='verbose logging')
+ (options, args) = parser.parse_args()
+
+ logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
+ level=logging.INFO if options.verbose else logging.WARN)
+
+ if len(args) == 0:
+ parser.print_help()
+ exit(1)
+
+ result = getDiskUsageForPath(args[0])
+
+ if result['found']:
+ print 'path %s' % result['path']
+ print 'disk_usage %s %s' % (result['disk_usage'], result['disk_usage_readbale'])
+ print 'nr_of_files %s' % result['nr_of_files']
+ else:
+ print result['message']
+ exit(1)
+
+if __name__ == '__main__':
+ main()
diff --git a/SAS/DataManagement/StorageQueryService/rpc.py b/SAS/DataManagement/StorageQueryService/rpc.py
index 4839501c7d2..c96071c843a 100644
--- a/SAS/DataManagement/StorageQueryService/rpc.py
+++ b/SAS/DataManagement/StorageQueryService/rpc.py
@@ -41,6 +41,13 @@ def main():
with StorageQueryRPC(busname=options.busname, servicename=options.servicename, broker=options.broker) as rpc:
result = rpc.getDiskUsageForOTDBId(options.otdb_id)
- if 'disk_usage' in result:
- result['disk_usage_readable'] = humanreadablesize(result['disk_usage'])
- print result
+ if result['found']:
+ print 'path %s' % result['path']
+ print 'disk_usage %s %s' % (result['disk_usage'], result['disk_usage_readable'])
+ print 'nr_of_files %s' % result['nr_of_files']
+ else:
+ print result['message']
+ exit(1)
+
+if __name__ == '__main__':
+ main()
diff --git a/SAS/DataManagement/StorageQueryService/service.py b/SAS/DataManagement/StorageQueryService/service.py
index 4424d48c850..497a82cea40 100644
--- a/SAS/DataManagement/StorageQueryService/service.py
+++ b/SAS/DataManagement/StorageQueryService/service.py
@@ -11,6 +11,7 @@ from lofar.common.util import waitForInterrupt
from lofar.sas.datamanagement.common.messagehandler import MessageHandler
from lofar.sas.datamanagement.common.config import CEP4_DATA_MOUNTPOINT
+from lofar.sas.datamanagement.storagequery.diskusage import getDiskUsageForPath
from lofar.sas.datamanagement.storagequery.config import DEFAULT_BUSNAME, DEFAULT_SERVICENAME
from lofar.sas.resourceassignment.resourceassignmentservice.config import DEFAULT_BUSNAME as RADB_BUSNAME
from lofar.sas.resourceassignment.resourceassignmentservice.config import DEFAULT_SERVICENAME as RADB_SERVICENAME
@@ -41,51 +42,10 @@ class StorageQueryHandler(MessageHandler):
def prepare_loop(self):
pass
- def getDiskUsageForPath(self, path):
- cmd = ['rbh-du', '-bd', path]
- hostname = socket.gethostname()
- if not 'mgmt0' in hostname:
- cmd = ['ssh', '-AXt', 'lofarsys@mgmt01.cep4.control.lofar'] + cmd
- logger.info(' '.join(cmd))
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- out, err = proc.communicate()
-
- logger.info(out)
-
- if proc.returncode != 0:
- logger.error(err)
- return {'found': False, 'path': path, 'message': err}
-
- # example of out
- # Using config file '/etc/robinhood.d/tmpfs/tmp_fs_mgr_basic.conf'.
- # /data/projects/2016LOFAROBS/L522380
- # dir count:3906, size:16048128, spc_used:16052224
- # file count:17568, size:42274164368, spc_used:42327519232
-
- #parse out
- lines = [l.strip() for l in out.split('\n')]
- file_line = next(l for l in lines if 'file count' in l)
- if file_line:
- parts = [p.strip() for p in file_line.split(',')]
- partsDict = {p.split(':')[0].strip():p.split(':')[1].strip() for p in parts}
-
- results = {'found': True, 'disk_usage': None, 'path': path}
-
- if 'size' in partsDict:
- results['disk_usage'] = int(partsDict['size'])
-
- if 'file count' in partsDict:
- results['nr_of_files'] = int(partsDict['file count'])
-
- logger.info('returning: %s' % results)
- return results
-
- results = {'found': False, 'path': path }
-
def getDiskUsageForOTDBId(self, otdb_id):
result = self.path_resolver.getPathForOTDBId(otdb_id)
if result['found']:
- return self.getDiskUsageForPath(result['path'])
+ return getDiskUsageForPath(result['path'])
return {'found': False, 'path': result['path']}
--
GitLab