diff --git a/SAS/DataManagement/CleanupService/config.py b/SAS/DataManagement/CleanupService/config.py
index 0e8dafdf6f6e028bf3a25ea448af70aa5595baa2..c7dd0f5a3ab60ce7622995116eb2763f049ce90c 100644
--- a/SAS/DataManagement/CleanupService/config.py
+++ b/SAS/DataManagement/CleanupService/config.py
@@ -5,3 +5,4 @@ from lofar.messaging import adaptNameToEnvironment
DEFAULT_BUSNAME = adaptNameToEnvironment('lofar.dm.command')
DEFAULT_SERVICENAME = 'CleanupService'
+CEP4_DATA_MOUNTPOINT='/data'
diff --git a/SAS/DataManagement/CleanupService/rpc.py b/SAS/DataManagement/CleanupService/rpc.py
index 369c01cf4ec1f13984540e549aa7074b0ffba0bb..f43cc6b19c8dd5892816af3f92c3b7fe5c9021f6 100644
--- a/SAS/DataManagement/CleanupService/rpc.py
+++ b/SAS/DataManagement/CleanupService/rpc.py
@@ -12,8 +12,8 @@ class CleanupRPC(RPCWrapper):
broker=None):
super(CleanupRPC, self).__init__(busname, servicename, broker)
- def foo(self):
- return self.rpc('foo')
+ def removePath(self, path):
+ return self.rpc('RemovePath', path=path)
def main():
import sys
@@ -35,4 +35,4 @@ def main():
level=logging.INFO if options.verbose else logging.WARN)
with CleanupRPC(busname=options.busname, servicename=options.servicename, broker=options.broker) as rpc:
- print rpc.foo()
+ print rpc.removePath()
diff --git a/SAS/DataManagement/CleanupService/service.py b/SAS/DataManagement/CleanupService/service.py
index 87b21b9bc4c1f1794ab817fe74f56d6694242398..e6f118d1bf46f31f46b604918e7885ee24ee9dbb 100644
--- a/SAS/DataManagement/CleanupService/service.py
+++ b/SAS/DataManagement/CleanupService/service.py
@@ -19,35 +19,83 @@ with RPC(busname, 'GetProjectDetails') as getProjectDetails:
'''
import logging
+import os.path
+from shutil import rmtree
from optparse import OptionParser
from lofar.messaging import Service
from lofar.messaging import setQpidLogLevel
from lofar.messaging.Service import MessageHandlerInterface
from lofar.common.util import waitForInterrupt
-from lofar.common.util import convertIntKeysToString
-from lofar.sas.datamanagement.cleanup.config import DEFAULT_BUSNAME, DEFAULT_SERVICENAME
+from lofar.common import isProductionEnvironment, isTestEnvironment
+from lofar.sas.datamanagement.cleanup.config import DEFAULT_BUSNAME, DEFAULT_SERVICENAME, CEP4_DATA_MOUNTPOINT
logger = logging.getLogger(__name__)
class CleanupHandler(MessageHandlerInterface):
def __init__(self, **kwargs):
super(CleanupHandler, self).__init__(**kwargs)
+ self.mountpoint = kwargs.pop("mountpoint", CEP4_DATA_MOUNTPOINT)
+ self.projects_path = os.path.join(self.mountpoint, 'projects' if isProductionEnvironment() else 'test-projects')
- self.service2MethodMap = {'foo': self._foo}
+ self.service2MethodMap = {'RemovePath': self._removePath}
def prepare_loop(self):
- pass
+ logger.info("Cleanup service started with projects_path=%s", self.projects_path)
- def _foo(self):
- return 'foo'
+ def _removePath(self, path):
+ # do various sanity checking to prevent accidental deletes
+ if not isinstance(path, basestring):
+ message = "Provided path is not a string"
+ logger.error(message)
+ return {'deleted': False, 'message': message}
-def createService(busname=DEFAULT_BUSNAME, servicename=DEFAULT_SERVICENAME, broker=None, verbose=False):
+ if not path:
+ message = "Empty path provided"
+ logger.error(message)
+ return {'deleted': False, 'message': message}
+
+ if '*' in path or '?' in path:
+ message = "Invalid path '%s': No wildcards allowed" % (path,)
+ logger.error(message)
+ return {'deleted': False, 'message': message}
+
+ # remove any trailing slashes
+ if len(path) > 1:
+ path = path.rstrip('/')
+
+ if not path.startswith(self.projects_path):
+ message = "Invalid path '%s': Path does not start with '%s'" % (path, self.projects_path)
+ logger.error(message)
+ return {'deleted': False, 'message': message}
+
+ if path[len(self.projects_path)+1:].count('/') == 0:
+ message = "Invalid path '%s': Path should be a subdir of '%s'" % (path, self.projects_path)
+ logger.error(message)
+ return {'deleted': False, 'message': message}
+
+ logger.info("Attempting to delete %s", path)
+
+ failed_paths = set()
+ def onerror(func, failed_path, excinfo):
+ logger.info("Failed to delete %s", failed_path)
+ failed_paths.add(failed_path)
+
+ if rmtree(path, onerror=onerror):
+ message = "Deleted '%s'" % (path)
+ logger.info(message)
+ return {'deleted': True, 'message': message}
+
+ return {'deleted': False, 'message': 'Failed to delete one or more files.', 'failed_paths' : list(failed_paths)}
+
+
+def createService(busname=DEFAULT_BUSNAME, servicename=DEFAULT_SERVICENAME, broker=None, mountpoint=CEP4_DATA_MOUNTPOINT, verbose=False):
return Service(servicename,
CleanupHandler,
busname=busname,
broker=broker,
use_service_methods=True,
- numthreads=2,
+ numthreads=1,
+ handler_args={'mountpoint': mountpoint},
verbose=verbose)
def main():
@@ -55,8 +103,9 @@ def main():
parser = OptionParser("%prog [options]",
description='runs the resourceassignment database service')
parser.add_option('-q', '--broker', dest='broker', type='string', default=None, help='Address of the qpid broker, default: localhost')
- parser.add_option("-b", "--busname", dest="busname", type="string", default=DEFAULT_BUSNAME, help="Name of the bus exchange on the qpid broker, default: %s" % DEFAULT_BUSNAME)
- parser.add_option("-s", "--servicename", dest="servicename", type="string", default=DEFAULT_SERVICENAME, help="Name for this service, default: %s" % DEFAULT_SERVICENAME)
+ parser.add_option("-b", "--busname", dest="busname", type="string", default=DEFAULT_BUSNAME, help="Name of the bus exchange on the qpid broker, default: %default")
+ parser.add_option("-s", "--servicename", dest="servicename", type="string", default=DEFAULT_SERVICENAME, help="Name for this service, default: default %default")
+ parser.add_option("-m", "--mountpoint", dest="mountpoint", type="string", default=CEP4_DATA_MOUNTPOINT, help="path of local cep4 mount point, default: %default")
parser.add_option('-V', '--verbose', dest='verbose', action='store_true', help='verbose logging')
(options, args) = parser.parse_args()
diff --git a/SAS/DataManagement/CleanupService/test/test_cleanup_service_and_rpc.py b/SAS/DataManagement/CleanupService/test/test_cleanup_service_and_rpc.py
index 3d6f441b0cecad3d4dd95ac7d2c05576e84fb1ec..6fb8779b520949a60fe7b3facbe61b7603795f44 100755
--- a/SAS/DataManagement/CleanupService/test/test_cleanup_service_and_rpc.py
+++ b/SAS/DataManagement/CleanupService/test/test_cleanup_service_and_rpc.py
@@ -5,8 +5,6 @@ import uuid
import datetime
import logging
from lofar.messaging import Service
-from lofar.sas.datamanagement.cleanup.service import createService
-from lofar.sas.datamanagement.cleanup.rpc import CleanupRPC
from qpid.messaging.exceptions import *
try:
@@ -17,6 +15,14 @@ except ImportError:
print 'Please source qpid profile'
exit(3)
+try:
+ from mock import MagicMock
+ from mock import patch
+except ImportError:
+ print 'Cannot run test without python MagicMock'
+ print 'Please install MagicMock: pip install mock'
+ exit(3)
+
connection = None
broker = None
@@ -32,16 +38,64 @@ try:
busname = 'test-lofarbus-%s' % (uuid.uuid1())
broker.addExchange('topic', busname)
- class TestCleanupServiceAndRPC(unittest.TestCase):
- def test(self):
- '''basic test '''
- rpc = CleanupRPC(busname=busname)
- self.assertEqual('foo', rpc.foo())
+ # the cleanup service uses shutil.rmtree under the hood
+ # so, mock/patch shutil.rmtree and fake the delete action
+ # because we do not want to delete any real data in this test
+ with patch('shutil.rmtree', autospec=True) as mock_rmtree:
+ mock = mock_rmtree.return_value
+ mock.return_value = True
+
+ # now that we have a mocked shutil.rmtree, import cleanupservice
+ from lofar.sas.datamanagement.cleanup.service import createService
+ from lofar.sas.datamanagement.cleanup.rpc import CleanupRPC
+
+ class TestCleanupServiceAndRPC(unittest.TestCase):
+ def test(self):
+ '''basic test '''
+ with CleanupRPC(busname=busname) as rpc:
+ #try some invalid input
+ self.assertFalse(rpc.removePath(None)['deleted'])
+ self.assertFalse(rpc.removePath(True)['deleted'])
+ self.assertFalse(rpc.removePath({'foo':'bar'})['deleted'])
+ self.assertFalse(rpc.removePath(['foo', 'bar'])['deleted'])
+
+ #try some dangerous paths
+ #these should not be deleted
+ result = rpc.removePath('/')
+ self.assertFalse(result['deleted'])
+ self.assertTrue('Path does not start with' in result['message'])
+
+ result = rpc.removePath('/foo/*/bar')
+ self.assertFalse(result['deleted'])
+ self.assertTrue('No wildcards allowed' in result['message'])
+
+ result = rpc.removePath('/foo/ba?r')
+ self.assertFalse(result['deleted'])
+ self.assertTrue('No wildcards allowed' in result['message'])
+
+ result = rpc.removePath('/data')
+ self.assertFalse(result['deleted'])
+ self.assertTrue('Path does not start with' in result['message'])
+
+ result = rpc.removePath('/data/test-projects/')
+ self.assertFalse(result['deleted'])
+ self.assertTrue('Path should be a subdir of' in result['message'])
+
+ result = rpc.removePath('/data/test-projects/foo')
+ self.assertFalse(result['deleted'])
+ self.assertTrue('Path should be a subdir of' in result['message'])
+
+ result = rpc.removePath('/data/test-projects/foo/')
+ self.assertFalse(result['deleted'])
+ self.assertTrue('Path should be a subdir of' in result['message'])
+
+ #try an actual delete, should work with mocked shutil.rmtree
+ self.assertTrue(rpc.removePath('/data/test-projects/foo/bar')['deleted'])
- # create and run the service
- with createService(busname=busname):
- # and run all tests
- unittest.main()
+ # create and run the service
+ with createService(busname=busname):
+ # and run all tests
+ unittest.main()
except ConnectError as ce:
logger.error(ce)