diff --git a/SAS/DataManagement/CleanupService/rpc.py b/SAS/DataManagement/CleanupService/rpc.py
index 7235f1941373c8d13b80a5f178bd5bce9acea3c9..f4024ca671e4ca1451469b1f94a04a63793bb22f 100644
--- a/SAS/DataManagement/CleanupService/rpc.py
+++ b/SAS/DataManagement/CleanupService/rpc.py
@@ -18,8 +18,8 @@ class CleanupRPC(RPCWrapper):
def removePath(self, path):
return self.rpc('RemovePath', path=path)
- def removeTaskData(self, otdb_id):
- return self.rpc('RemoveTaskData', otdb_id=otdb_id)
+ def removeTaskData(self, otdb_id, delete_is=True, delete_cs=True, delete_uv=True, delete_im=True, delete_img=True, delete_pulp=True, delete_scratch=True):
+ return self.rpc('RemoveTaskData', otdb_id=otdb_id, delete_is=delete_is, delete_cs=delete_cs, delete_uv=delete_uv, delete_im=delete_im, delete_img=delete_img, delete_pulp=delete_pulp, delete_scratch=delete_scratch)
def main():
import sys
diff --git a/SAS/DataManagement/CleanupService/service.py b/SAS/DataManagement/CleanupService/service.py
index 5fd540e30bce0fb305ad27fb173fefaffd08101c..7d07644676c642380a37b65d1e98d09de0f94b51 100644
--- a/SAS/DataManagement/CleanupService/service.py
+++ b/SAS/DataManagement/CleanupService/service.py
@@ -42,7 +42,7 @@ class CleanupHandler(MessageHandler):
'RemovePath': self._removePath,
'RemoveTaskData': self._removeTaskData }
- def _removeTaskData(self, otdb_id):
+ def _removeTaskData(self, otdb_id, delete_is=True, delete_cs=True, delete_uv=True, delete_im=True, delete_img=True, delete_pulp=True, delete_scratch=True):
logger.info("Remove task data for otdb_id %s" % (otdb_id,))
if not isinstance(otdb_id, int):
@@ -50,11 +50,35 @@ class CleanupHandler(MessageHandler):
logger.error(message)
return {'deleted': False, 'message': message}
- result = self.path_resolver.getPathForOTDBId(otdb_id)
- if result['found']:
- return self._removePath(result['path'])
-
- return {'deleted': False, 'message': result['message']}
+ path_result = self.path_resolver.getPathForOTDBId(otdb_id)
+ if path_result['found']:
+ rm_results = []
+ if delete_is and delete_cs and delete_uv and delete_im and delete_img and delete_pulp:
+ rm_results.append(self._removePath(path_result['path']))
+ else:
+ if delete_is:
+ rm_results.append(self._removePath(os.path.join(path_result['path'], 'is')))
+ if delete_cs:
+ rm_results.append(self._removePath(os.path.join(path_result['path'], 'cs')))
+ if delete_uv:
+ rm_results.append(self._removePath(os.path.join(path_result['path'], 'uv')))
+ if delete_im:
+ rm_results.append(self._removePath(os.path.join(path_result['path'], 'im')))
+ if delete_img:
+ rm_results.append(self._removePath(os.path.join(path_result['path'], 'img')))
+ if delete_pulp:
+ rm_results.append(self._removePath(os.path.join(path_result['path'], 'pulp')))
+
+ if delete_scratch and 'scratch_paths' in path_result:
+ for scratch_path in path_result['scratch_paths']:
+ rm_results.append(self._removePath(scratch_path))
+
+ rm_result = {'deleted': all(x['deleted'] for x in rm_results),
+ 'message': '\n'.join(x['message'] for x in rm_results)}
+
+ return rm_result
+
+ return {'deleted': False, 'message': path_result['message']}
def _removePath(self, path):
logger.info("Remove path: %s" % (path,))
@@ -79,16 +103,24 @@ class CleanupHandler(MessageHandler):
if len(path) > 1:
path = path.rstrip('/')
- if not path.startswith(self.path_resolver.projects_path):
- message = "Invalid path '%s': Path does not start with '%s'" % (path, self.path_resolver.projects_path)
- logger.error(message)
- return {'deleted': False, 'message': message}
+ required_base_paths = [self.path_resolver.projects_path, self.path_resolver.scratch_path, self.path_resolver.share_path]
- if path[len(self.path_resolver.projects_path)+1:].count('/') == 0:
- message = "Invalid path '%s': Path should be a subdir of '%s'" % (path, self.path_resolver.projects_path)
+ if not any(path.startswith(base_path) for base_path in required_base_paths):
+ message = "Invalid path '%s': Path does not start with any oth the base paths: '%s'" % (path, ' '.join(required_base_paths))
logger.error(message)
return {'deleted': False, 'message': message}
+ for base_path in required_base_paths:
+ if path.startswith(base_path) and path[len(base_path):].count('/') == 0:
+ message = "Invalid path '%s': Path should be a subdir of '%s'" % (path, base_path)
+ logger.error(message)
+ return {'deleted': False, 'message': message}
+
+ if not os.path.exists(path):
+ message = "Nothing to delete, path '%s' does not exist." % (path)
+ logger.warn(message)
+ return {'deleted': True, 'message': message}
+
logger.info("Attempting to delete %s", path)
failed_paths = set()
diff --git a/SAS/DataManagement/DataManagementCommon/path.py b/SAS/DataManagement/DataManagementCommon/path.py
index 86acf0e8c6dcaad3310f73056b10efb5d4346495..363efb7058849bbc9f424a9e1fe855133c027bf3 100644
--- a/SAS/DataManagement/DataManagementCommon/path.py
+++ b/SAS/DataManagement/DataManagementCommon/path.py
@@ -30,6 +30,8 @@ class PathResolver:
self.mountpoint = mountpoint
self.projects_path = os.path.join(self.mountpoint, 'projects' if isProductionEnvironment() else 'test-projects')
+ self.scratch_path = os.path.join(self.mountpoint, 'scratch', 'pipeline')
+ self.share_path = os.path.join(self.mountpoint, 'share', 'pipeline')
self.radbrpc = RADBRPC(busname=radb_busname, servicename=radb_servicename, broker=broker)
self.momrpc = MoMQueryRPC(busname=mom_busname, servicename=mom_servicename, broker=broker)
@@ -70,7 +72,14 @@ class PathResolver:
task_data_path = os.path.join(project_path, 'L%s' % task['otdb_id'])
logger.info("Found path '%s' for otdb_id=%s mom_id=%s radb_id=%s" % (task_data_path, task['otdb_id'], task['mom_id'], task['id']))
- return {'found': True, 'message': '', 'path': task_data_path}
+ path_result = {'found': True, 'message': '', 'path': task_data_path}
+
+ if task['type'] == 'pipeline':
+ scratch_path = os.path.join(self.scratch_path, 'Observation%s' % task['otdb_id'])
+ share_path = os.path.join(self.share_path, 'Observation%s' % task['otdb_id'])
+ path_result['scratch_paths'] = [scratch_path, share_path]
+
+ return path_result
return result
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEditor/lib/static/app/controllers/cleanupcontroller.js b/SAS/ResourceAssignment/ResourceAssignmentEditor/lib/static/app/controllers/cleanupcontroller.js
index d0848e7ee6ff5705a6fd65ab3083c9eb793d80a3..7ed8d63afd9c09b6358b7c211257cef35c02ef99 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentEditor/lib/static/app/controllers/cleanupcontroller.js
+++ b/SAS/ResourceAssignment/ResourceAssignmentEditor/lib/static/app/controllers/cleanupcontroller.js
@@ -19,15 +19,18 @@ cleanupControllerMod.controller('CleanupController', ['$scope', '$uibModal', '$h
};
self.deleteTaskDataWithConfirmation = function(task) {
- self.getTaskDataPath(task).then(function(path_result) {
- dataService.getTaskDiskUsageByOTDBId(task.otdb_id).then(function(du_result) {
- openDeleteConfirmationDialog(task, path_result.datapath, du_result.found ? du_result.task_directory.disk_usage_readable : "??B");
- });
+ dataService.getTaskDiskUsageByOTDBId(task.otdb_id).then(function(du_result) {
+ if(du_result.found) {
+ openDeleteConfirmationDialog(task, du_result);
+ } else {
+ alert(du_result.message);
+ }
});
};
- function deleteTaskData(task) {
- $http.delete('/rest/tasks/' + task.id + '/cleanup').error(function(result) {
+ function deleteTaskData(task, delete_is, delete_cs, delete_uv, delete_im, delete_img, delete_pulp, delete_scratch) {
+ var params = {delete_is:delete_is, delete_cs:delete_cs, delete_uv:delete_uv, delete_im:delete_im, delete_img:delete_img, delete_pulp:delete_pulp, delete_scratch:delete_scratch};
+ $http.delete('/rest/tasks/' + task.id + '/cleanup', {data: params}).error(function(result) {
console.log("Error. Could cleanup data for task " + task.id + ", " + result);
}).success(function(result) {
if(!result.deleted) {
@@ -36,24 +39,55 @@ cleanupControllerMod.controller('CleanupController', ['$scope', '$uibModal', '$h
});
};
- function openDeleteConfirmationDialog(task, path, amount) {
+ function openDeleteConfirmationDialog(task, du_result) {
+ var path = du_result.task_directory.path;
+
var modalInstance = $uibModal.open({
animation: false,
template: '<div class="modal-header">\
<h3 class="modal-title">Are you sure?</h3>\
</div>\
<div class="modal-body">\
- <p>This will delete ' + amount + ' of data in ' + path + '<br>\
+ <p>This will delete all selected data in ' + path + '<br>\
Are you sure?</p>\
+ <label ng-if="has_is" style="margin-left:24px">IS: {{amount_is}}<input style="margin-left:8px" type="checkbox" ng-model="$parent.delete_is"></label>\
+ <label ng-if="has_cs" style="margin-left:24px">CS: {{amount_cs}}<input style="margin-left:8px" type="checkbox" ng-model="$parent.delete_cs"></label>\
+ <label ng-if="has_uv" style="margin-left:24px">UV: {{amount_uv}}<input style="margin-left:8px" type="checkbox" ng-model="$parent.delete_uv"></label>\
+ <label ng-if="has_im" style="margin-left:24px">IM: {{amount_im}}<input style="margin-left:8px" type="checkbox" ng-model="$parent.delete_im"></label>\
+ <label ng-if="has_img" style="margin-left:24px">Images: {{amount_img}}<input style="margin-left:8px" type="checkbox" ng-model="$parent.delete_img"></label>\
+ <label ng-if="has_pulp" style="margin-left:24px">Pulp: {{amount_pulp}}<input style="margin-left:8px" type="checkbox" ng-model="$parent.delete_pulp"></label>\
+ <label ng-if="has_scratch" style="margin-left:24px">scratch<input style="margin-left:8px" type="checkbox" ng-model="$parent.delete_scratch"></label>\
</div>\
<div class="modal-footer">\
<button class="btn btn-primary" type="button" ng-click="ok()">OK</button>\
<button class="btn btn-warning" type="button" autofocus ng-click="cancel()">Cancel</button>\
</div>',
controller: function ($scope, $uibModalInstance) {
+ $scope.has_is = du_result.sub_directories.hasOwnProperty(path + '/is');
+ $scope.has_cs = du_result.sub_directories.hasOwnProperty(path + '/cs');
+ $scope.has_uv = du_result.sub_directories.hasOwnProperty(path + '/uv');
+ $scope.has_im = du_result.sub_directories.hasOwnProperty(path + '/im');
+ $scope.has_img = du_result.sub_directories.hasOwnProperty(path + '/img');
+ $scope.has_pulp = du_result.sub_directories.hasOwnProperty(path + '/pulp');
+ $scope.has_scratch = du_result.task_directory.hasOwnProperty('scratch_paths');;
+ $scope.amount_is = $scope.has_is ? du_result.sub_directories[path + '/is'].disk_usage_readable : '';
+ $scope.amount_cs = $scope.has_cs ? du_result.sub_directories[path + '/cs'].disk_usage_readable : '';
+ $scope.amount_uv = $scope.has_uv ? du_result.sub_directories[path + '/uv'].disk_usage_readable : '';
+ $scope.amount_im = $scope.has_im ? du_result.sub_directories[path + '/im'].disk_usage_readable : '';
+ $scope.amount_img = $scope.has_img ? du_result.sub_directories[path + '/img'].disk_usage_readable : '';
+ $scope.amount_pulp = $scope.has_pulp ? du_result.sub_directories[path + '/pulp'].disk_usage_readable : '';
+
+ $scope.delete_is = true;
+ $scope.delete_cs = true;
+ $scope.delete_uv = true;
+ $scope.delete_im = true;
+ $scope.delete_img = true;
+ $scope.delete_pulp = true;
+ $scope.delete_scratch = true;
+
$scope.ok = function () {
$uibModalInstance.close();
- deleteTaskData(task);
+ deleteTaskData(task, $scope.delete_is, $scope.delete_cs, $scope.delete_uv, $scope.delete_im, $scope.delete_img, $scope.delete_pulp, $scope.delete_scratch);
};
$scope.cancel = function () {
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEditor/lib/webservice.py b/SAS/ResourceAssignment/ResourceAssignmentEditor/lib/webservice.py
index 0f562d0b5979cb04d13b2d812112a8723145fab5..4604f453dba5583c0906e655f582de3bae5bdc82 100755
--- a/SAS/ResourceAssignment/ResourceAssignmentEditor/lib/webservice.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentEditor/lib/webservice.py
@@ -265,12 +265,26 @@ def putTask(task_id):
@app.route('/rest/tasks/<int:task_id>/cleanup', methods=['DELETE'])
def cleanupTaskData(task_id):
try:
+ delete_params = {}
+
+ if 'Content-Type' in request.headers and (request.headers['Content-Type'].startswith('application/json') or request.headers['Content-Type'].startswith('text/plain')):
+ delete_params = json.loads(request.data)
+
+ print 'delete_params:', delete_params
+
task = rarpc.getTask(task_id)
if not task:
abort(404, 'No such task (id=%s)' % task_id)
- result = curpc.removeTaskData(task['otdb_id'])
+ result = curpc.removeTaskData(task['otdb_id'],
+ delete_is=delete_params.get('delete_is', True),
+ delete_cs=delete_params.get('delete_cs', True),
+ delete_uv=delete_params.get('delete_uv', True),
+ delete_im=delete_params.get('delete_im', True),
+ delete_img=delete_params.get('delete_img', True),
+ delete_pulp=delete_params.get('delete_pulp', True),
+ delete_scratch=delete_params.get('delete_scratch', True))
logger.info(result)
return jsonify(result)
except Exception as e: