Task #10339: filter for ingestable and (un)ingested dataproducts

SAS/DataManagement/Cleanup/CleanupService/service.py

@@ -178,11 +178,11 @@ class CleanupHandler(MessageHandlerInterface):
 
             momrpc = self.path_resolver.momrpc
             dataproducts = momrpc.getDataProducts(task['mom_id']).get(task['mom_id'])
-            ingestable_dataproducts = [dp for dp in dataproducts if dp['status'] not in [None, 'has_data', 'no_data', 'aborted'] ]
-            ingested_dataproducts = [dp for dp in dataproducts if dp['status'] == 'ingested']
+            ingestable_dataproducts = [dp for dp in dataproducts if dp['status'] not in [None, 'has_data', 'no_data', 'aborted', 'populated'] ]
+            ingested_dataproducts = [dp for dp in ingestable_dataproducts if dp['status'] == 'ingested']
 
-            if len(ingested_dataproducts) != len(dataproducts) and len(dataproducts) > 0 and len(ingestable_dataproducts) > 0:
-                logger.info('ingestable_dataproducts: %s', ingestable_dataproducts)
+            if len(ingestable_dataproducts) > 0 and len(ingested_dataproducts) < len(ingestable_dataproducts) :
+                uningested_dataproducts = [dp for dp in ingestable_dataproducts if dp['status'] != 'ingested']
                 message = "Task otdb_id=%s has un-ingested dataproducts. Not deleting data." % (task['otdb_id'],)
                 logger.error(message)
                 self._sendNotification(subject='TaskDeleted', content={'deleted': False,

SAS/ResourceAssignment/ResourceAssignmentEditor/lib/mom.py

@@ -106,7 +106,7 @@ def updateTaskMomDetails(task, momrpc):
                             elif dp['status'] == 'aborted':
                                 num_ingest_aborted += 1
 
-                        ingestable_dataproducts = [dp for dp in dps if dp['status'] not in [None, 'has_data', 'no_data'] ]
+                        ingestable_dataproducts = [dp for dp in dps if dp['status'] not in [None, 'has_data', 'no_data', 'aborted', 'populated'] ]
 
                         if num_ingested > 0 and num_ingested == len(ingestable_dataproducts):
                             t['ingest_status'] = 'ingested'