Task LSMR-20: Migrating timestamps to tasks

SAS/LSMR/src/migrate_momdb_to_lsmr.py

@@ -13,7 +13,6 @@ mom_db_name = 'lofar_mom_test_lsmr'
 mom_db_user = 'root'
 mom_db_pass = 'the_great_password'
 
-
 def _execute_query(query, data=None):
     try:
         db = pymysql.connect(mom_db_host, mom_db_user, mom_db_pass, mom_db_name)
@@ -86,15 +85,18 @@ def get_project_details_from_momdb():
 def query_task_details_for_project_from_momdb(project_mom2id):
     logger.info("Querying MoM database for tasks of project %s" % project_mom2id)
     # todo: double-check the correct use of ids. What refers to a mom2id and what refers to a database entry pk does not seem systematic and is very comfusing.
-    query = '''SELECT mom2object.mom2id, mom2object.name, mom2object.description, mom2object.mom2objecttype, status.code, lofar_pipeline.template, lofar_observation.default_template
+    query = '''SELECT mom2object.mom2id, mom2object.name, mom2object.description, mom2object.mom2objecttype, status.code,
+                      lofar_pipeline.template, lofar_observation.default_template, lofar_pipeline.starttime, lofar_pipeline.endtime,
+                      lofar_observation_specification.starttime AS obs_starttime, lofar_observation_specification.endtime AS obs_endtime
                       FROM mom2object
-                      INNER JOIN mom2object as ownerproject_mom2object on mom2object.ownerprojectid = ownerproject_mom2object.id
-                      INNER JOIN mom2objectstatus on mom2object.currentstatusid = mom2objectstatus.id
-                      INNER JOIN status on mom2objectstatus.statusid = status.id
-                      LEFT JOIN lofar_pipeline on mom2object.id = lofar_pipeline.mom2objectid
-                      LEFT JOIN lofar_observation on mom2object.id = lofar_observation.mom2objectid
+                      INNER JOIN mom2object AS ownerproject_mom2object ON mom2object.ownerprojectid = ownerproject_mom2object.id
+                      INNER JOIN mom2objectstatus ON mom2object.currentstatusid = mom2objectstatus.id
+                      INNER JOIN status ON mom2objectstatus.statusid = status.id
+                      LEFT JOIN lofar_pipeline ON mom2object.id = lofar_pipeline.mom2objectid
+                      LEFT JOIN lofar_observation ON mom2object.id = lofar_observation.mom2objectid
+                      LEFT JOIN lofar_mom_test_lsmr.lofar_observation_specification ON lofar_observation.user_specification_id = lofar_observation_specification.id
                       WHERE ownerproject_mom2object.mom2id = %s
-                      AND (mom2object.mom2objecttype = 'LOFAR_OBSERVATION' or mom2object.mom2objecttype like '%%PIPELINE%%');
+                      AND (mom2object.mom2objecttype = 'LOFAR_OBSERVATION' OR mom2object.mom2objecttype LIKE '%%PIPELINE%%');
                       '''
 
     parameters = (project_mom2id,)
@@ -104,6 +106,22 @@ def query_task_details_for_project_from_momdb(project_mom2id):
     return results
 
 
+# todo: remove once proper templates are present
+def _dummy_template():
+    try:
+        return models.TaskTemplate.objects.get(name='dummy')
+    except:
+        dummy_template_details = {"name": "dummy",
+                                  "description": 'Dummy Template',
+                                  "version": '1',
+                                  "schema": {},
+                                  "realtime": False,
+                                  "queue": False,
+                                  "tags": ["DUMMY"]}
+
+        return models.TaskTemplate.objects.create(**dummy_template_details)
+
+
 def get_task_details_from_momdb(project_mom2id):
 
     logger.info("Getting task details from MoM database")
@@ -112,10 +130,30 @@ def get_task_details_from_momdb(project_mom2id):
 
     for mom_details in mom_results:
 
-        # create new lsmr details based on MoM details
-        details = {"type": None,
-                   "start_time": datetime.datetime.utcnow().isoformat(),  # todo: determine from mom db
-                   "stop_time": datetime.datetime.utcnow().isoformat(),  # todo: determine from mom db
+        # different types have some info in different spots, so they end up in different columns.
+        # put same information in same spot to keep following code same for all tasks.
+        # (maybe we want to instead separate these into different queries instead or union them in SQL?)
+
+        if 'OBSERVATION' in mom_details['mom2objecttype']:
+            type = models.TaskTypeChoice.objects.get(value='observation')
+            template_name = mom_details['default_template']
+            start_time = mom_details['obs_starttime']
+            end_time = mom_details['obs_endtime']
+        elif 'PIPELINE' in mom_details['mom2objecttype']:
+            type = models.TaskTypeChoice.objects.get(value='pipeline')
+            template_name = mom_details['template']
+            start_time = mom_details['starttime']
+            end_time = mom_details['endtime']
+        else:
+            logger.warning('Unknown type %(mom2objecttype)s' % mom_details)
+            logger.warning('Skipping %s' % mom_details)
+            continue
+
+        # create new lsmr details (leave out stuff that might go wrong)
+
+        details = {"type": type,
+                   "start_time": None,
+                   "stop_time": None,
                    "state": None,
                    "requested_state": None,
                    "specification": "{}",
@@ -123,6 +161,20 @@ def get_task_details_from_momdb(project_mom2id):
                    "template": None,
                    "tags": []}
 
+        # timestamps
+
+        if start_time is not None:
+            details['start_time'] = start_time
+        else:
+            details['start_time'] = datetime.datetime.utcfromtimestamp(0).isoformat() # not-null constraint
+
+        if end_time is not None:
+            details['stop_time'] = end_time
+        else:
+            details['stop_time'] = datetime.datetime.utcfromtimestamp(0).isoformat() # not-null constraint
+
+        # state
+
         try:
             state = models.TaskStateChoice.objects.get(value=mom_details['code'])
             details['state'] = state
@@ -132,19 +184,7 @@ def get_task_details_from_momdb(project_mom2id):
             logger.warning('Skipping %s' % mom_details)
             continue
 
-        if 'OBSERVATION' in mom_details['mom2objecttype']:
-            details['type'] = models.TaskTypeChoice.objects.get(value='observation')
-            template_name = mom_details['default_template']
-        elif 'PIPELINE' in mom_details['mom2objecttype']:
-            details['type'] = models.TaskTypeChoice.objects.get(value='pipeline')
-            template_name = mom_details['template']
-        else:
-            logger.warning('Unknown type %(mom2objecttype)s' % mom_details)
-            logger.warning('Skipping %s' % mom_details)
-            continue
-
-        # todo: create a lot of templates to reflect what was used for the actual task?
-        # todo: -> models.TaskTemplate.objects.create(...)
+        # template
 
         if template_name is not None:
             try:
@@ -152,15 +192,10 @@ def get_task_details_from_momdb(project_mom2id):
             except Exception as e:
                 logger.error("No task template matching '%s' in lsmr database! %s" % (template_name, e))
 
+                # todo: create a lot of templates to reflect what was used for the actual task?
+                # todo: raise Exception (or continue) once we have proper templates for everything.
                 logger.error("Using dummy instead!")
-                dummy_template_details = {"name": "dummy",
-                                          "description": 'Dummy Template',
-                                          "version": '1',
-                                          "schema": {},
-                                          "realtime": False,
-                                          "queue": False,
-                                          "tags": ["DUMMY"]}
-                details["template"] = models.TaskTemplate.objects.create(**dummy_template_details)
+                details["template"] = _dummy_template()
 
         else:
             logger.warning('Missing template name in MoM details!')