Merge branch 'SDC-934-discard-mechanism' into 'master'

adding (initial) discard mechanism

See merge request !294

atdb/atdb/static/taskdatabase/style.css

@@ -49,7 +49,7 @@ TD {
  font-weight: bold;
 }
 
-.scrubbed, {
+.scrubbed {
   color: darkgray;
   font-style: italic;
 }
@@ -142,3 +142,10 @@ p.title {
  color: red;
  font-weight: bold;
 }
+
+#search_box
+{
+    color: darkblue;
+  display: inline-block;
+  vertical-align: middle;
+}

atdb/logs/atdb.log

@@ -54,3 +54,19 @@ logging[14/Apr/2023 14:17:24,827] unsupported operand type(s) for +=: 'NoneType'
 [04/May/2023 14:35:52,193] task 25378 - (regular) - 116876 has no plots, skipped.
 [04/May/2023 14:35:52,294] 'NoneType' object is not subscriptable
 [04/May/2023 14:35:52,294] task 25378 - (regular) - 116876 has no plots, skipped.
+[09/May/2023 08:47:20,610] 'NoneType' object is not subscriptable
+[09/May/2023 08:47:20,610] task 25378 - (regular) - 116876 has no plots, skipped.
+[09/May/2023 08:47:20,725] 'NoneType' object is not subscriptable
+[09/May/2023 08:47:20,725] task 25378 - (regular) - 116876 has no plots, skipped.
+[09/May/2023 08:47:20,822] 'NoneType' object is not subscriptable
+[09/May/2023 08:47:20,823] task 25378 - (regular) - 116876 has no plots, skipped.
+[09/May/2023 08:47:20,932] 'NoneType' object is not subscriptable
+[09/May/2023 08:47:20,933] task 25378 - (regular) - 116876 has no plots, skipped.
+[09/May/2023 08:47:32,444] 'NoneType' object is not subscriptable
+[09/May/2023 08:47:32,445] task 25378 - (regular) - 116876 has no plots, skipped.
+[09/May/2023 08:47:33,063] 'NoneType' object is not subscriptable
+[09/May/2023 08:47:33,064] task 25378 - (regular) - 116876 has no plots, skipped.
+[09/May/2023 08:47:33,649] 'NoneType' object is not subscriptable
+[09/May/2023 08:47:33,650] task 25378 - (regular) - 116876 has no plots, skipped.
+[09/May/2023 08:47:34,165] 'NoneType' object is not subscriptable
+[09/May/2023 08:47:34,165] task 25378 - (regular) - 116876 has no plots, skipped.

atdb/taskdatabase/forms.py

 from django import forms
 
 class QualityAnnotationForm(forms.Form):
-      annotation = forms.CharField(label='',
-                                   widget=forms.Textarea(
-                                         attrs={'rows': 12,'cols': 85}
-                                   ),
-                                   max_length=1000)
+      annotation = forms.CharField(label='', widget=forms.Textarea(attrs={'rows': 12,'cols': 85}),required=False,max_length=1000)
+      return_to_page = forms.IntegerField(widget=forms.HiddenInput)
+
+
+class DiscardAnnotationForm(forms.Form):
+    annotation = forms.CharField(label='', widget=forms.Textarea(attrs={'rows': 3, 'cols': 85}), required=False,
+                                 max_length=250)
     return_to_page = forms.IntegerField(widget=forms.HiddenInput)

atdb/taskdatabase/migrations/0023_task_cleanup_policy.py

+# Generated by Django 3.1.4 on 2023-05-04 09:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('taskdatabase', '0022_workflow_tag'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='task',
+            name='cleanup_policy',
+            field=models.CharField(blank=True, max_length=30, null=True),
+        ),
+    ]

atdb/taskdatabase/migrations/0024_auto_20230509_1612.py

+# Generated by Django 3.1.4 on 2023-05-09 14:12
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('taskdatabase', '0023_task_cleanup_policy'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='task',
+            name='workflow',
+            field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, related_name='tasks', to='taskdatabase.workflow'),
+            preserve_default=False,
+        ),
+    ]

atdb/taskdatabase/migrations/0025_auto_20230509_1631.py

+# Generated by Django 3.1.4 on 2023-05-09 14:31
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('taskdatabase', '0024_auto_20230509_1612'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='task',
+            name='workflow',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='tasks', to='taskdatabase.workflow'),
+        ),
+    ]

atdb/taskdatabase/models.py

@@ -25,7 +25,7 @@ class Workflow(models.Model):
     prefetch = models.BooleanField(null=True, default=True)
 
     def __str__(self):
-        return str(self.id)
+        return str(self.id) + ' - ' + str(self.workflow_uri)
 
 
 # convert the quality information from the JSONfield into a easy parsable list for the template
@@ -73,7 +73,6 @@ class Task(models.Model):
     # Task control properties
     task_type = models.CharField(max_length=20, default="regular")
     filter = models.CharField(max_length=30, blank=True, null=True)
-    #environment = models.JSONField(null=True, blank=True)
     environment = models.CharField(max_length=255, blank=True, null=True)
     new_status = models.CharField(max_length=50, default="defining", null=True)
     status = models.CharField(db_index=True, default="unknown", max_length=50,blank=True, null=True)
@@ -84,6 +83,7 @@ class Task(models.Model):
 
     priority = models.IntegerField(default=100, null=True)
     purge_policy = models.CharField(max_length=5, default="no", blank=True, null=True)
+    cleanup_policy = models.CharField(max_length=30, blank=True, null=True)
     stage_request_id = models.IntegerField(null=True)
 
     # LOFAR properties
@@ -101,7 +101,7 @@ class Task(models.Model):
     total_processing_time = models.IntegerField(default=0, null=True, blank=True)
 
     # relationships
-    workflow = models.ForeignKey(Workflow, related_name='tasks', on_delete=models.SET_NULL, null=True, blank=True)
+    workflow = models.ForeignKey(Workflow, related_name='tasks', on_delete=models.CASCADE, null=True, blank=True)
     predecessor = models.ForeignKey('self', related_name='successors', on_delete=models.SET_NULL, null=True, blank=True)
 
     def __str__(self):
@@ -241,12 +241,23 @@ class Task(models.Model):
 
     @property
     def sasid_is_verified(self):
-
         for task in Task.objects.filter(sas_id=self.sas_id):
            if task.status not in verified_statusses:
                return False
         return True
 
+    # NV: this shows the latest status change, but because it is a derived property it cannot be sorted.
+    # This functionality was not requested, and to avoid additional requests about 'sort' functionalty
+    # it is currently commented out. Could be of use later though, so I leave it in for now.
+    # @property
+    # def latest_change(self):
+    #     qs = Status.objects.filter(task__id=self.id).order_by('-timestamp')
+    #     if len(qs) > 0:
+    #         status = qs[0]
+    #         timestamp = status.timestamp
+    #
+    #     return timestamp
+
 
 class LogEntry(models.Model):
     cpu_cycles = models.IntegerField(null=True,blank=True)

atdb/taskdatabase/serializers.py

 from rest_framework import serializers
 from .models import Status, Task, Workflow, LogEntry, Configuration, Job, PostProcessingRule, Monitor, LatestMonitor
 
-
 class WorkflowSerializer(serializers.ModelSerializer):
 
     class Meta:
@@ -39,7 +38,7 @@ class TaskWriteSerializer(serializers.ModelSerializer):
     class Meta:
         model = Task
         fields = ('id','task_type','filter','predecessor','successors',
-                  'project','sas_id','priority','purge_policy','resume',
+                  'project','sas_id','priority','purge_policy','cleanup_policy','resume',
                   'new_workflow_id','new_workflow_uri','workflow',
                   'stage_request_id',
                   'status','new_status','quality',
@@ -59,7 +58,6 @@ class TaskWriteSerializer(serializers.ModelSerializer):
 
         workflow_uri = self.initial_data['new_workflow_uri']
         if workflow_uri:
-            #workflow_uri = validated_data['new_workflow_uri']
             workflow = Workflow.objects.get(workflow_uri=workflow_uri)
             task.workflow = workflow
             task.save()
@@ -94,7 +92,7 @@ class TaskReadSerializer(serializers.ModelSerializer):
         model = Task
         fields = ['id','task_type','creationTime','filter',
                   'predecessor','predecessor_status','successors',
-                  'project','sas_id','priority','purge_policy','resume',
+                  'project','sas_id','priority','purge_policy','cleanup_policy','resume',
                   'workflow',
                   'stage_request_id',
                   'status','new_status','quality',
@@ -125,13 +123,12 @@ class TaskReadSerializerFast(serializers.ModelSerializer):
     class Meta:
         model = Task
         fields = ['id','task_type','creationTime','filter','predecessor','predecessor_status',
-                  'project','sas_id','priority','purge_policy','resume',
+                  'project','sas_id','priority','purge_policy','cleanup_policy','resume',
                   'workflow',
                   'stage_request_id',
                   'status','new_status','quality',
                   'inputs','outputs','metrics','archive',
                   'size_to_process', 'size_processed', 'total_processing_time',
-
                   ]
         read_only_fields = fields
 

atdb/taskdatabase/services/algorithms.py

@@ -829,7 +829,7 @@ def construct_summary(task):
                         line += '<tr><td colspan="3">'
                         for filename in added:
                             line += '<tr><td colspan="3">' + filename + '<td></tr>'
-                except:
+                except Exception as error:
                     pass
 
                 try:
@@ -838,14 +838,13 @@ def construct_summary(task):
                         line += '<th>Deleted</th>'
                         for filename in deleted:
                             line += '<tr><td colspan="3">' +filename + '<td></tr>'
-
-                except:
+                except Exception as error:
                     pass
 
                 results += line
 
-        except:
-            pass
+        except Exception as error:
+            logger.error(error)
 
     totals += '<th>Totals</th><th></th><th></th>'
     totals += '<tr><td colspan="2"><b>Input size</b></td><td>' + str(total_size_input) + '</td></tr>'

atdb/taskdatabase/static/taskdatabase/style.css

@@ -49,7 +49,7 @@ TD {
  font-weight: bold;
 }
 
-.scrubbed, {
+.scrubbed {
   color: darkgray;
   font-style: italic;
 }
@@ -142,3 +142,15 @@ p.title {
  color: red;
  font-weight: bold;
 }
+
+#search_box
+{
+    color: darkblue;
+  display: inline-block;
+  vertical-align: middle;
+}
+
+.img {
+    color: white;
+    font-family: "Courier New";
+}
\ No newline at end of file

atdb/taskdatabase/tables.py

@@ -25,7 +25,7 @@ class TaskTable(tables.Table):
     class Meta:
         model = Task
         template_name = "django_tables2/bootstrap4.html"
-        fields = ("id", "workflow","filter","priority","status","quality","project","sas_id","resume","actions","buttons")
+        fields = ("id", "workflow.id","filter","priority","status","quality","project","sas_id","resume","actions","buttons")
 
     # columns that need specific rendering
     status = StatusColumn()

atdb/taskdatabase/templates/astronauth/navbar.html

@@ -28,6 +28,7 @@
                 <li><a class="nav-link" href="{% url 'quality' %}">Quality</a></li>
                 <li><a class="nav-link" href="{% url 'validation' %}">Validation</a></li>
                 <li><a class="nav-link" href="{% url 'failures' %}">Failures</a></li>
+                <li><a class="nav-link" href="{% url 'discarded' %}">Discarded</a></li>
                 <li><a class="nav-link" href="{% url 'finished' %}">Finished</a></li>
                 <li><a class="nav-link" href="{% url 'monitoring' %}">Monitoring</a></li>
                 <li><a class="nav-link" href="{% url 'diagram' %}">Diagram</a></li>

atdb/taskdatabase/templates/taskdatabase/archived/tasks.html

@@ -26,9 +26,9 @@
                 </td>
                 <td>
                     <a class="open-modal btn btn-primary btn-sm"
-                           href="{% url 'workflow-details' task.workflow %}"
-                           data-popup-url="{% url 'workflow-details' task.workflow %}"
-                           target="_blank"><i class="fas fa-project-diagram"></i> {{ task.workflow }}
+                           href="{% url 'workflow-details' task.workflow.id %}"
+                           data-popup-url="{% url 'workflow-details' task.workflow.id %}"
+                           target="_blank"><i class="fas fa-project-diagram"></i> {{ task.workflow.id }}
                         </a></td>
                 </td>
                 {% if task.resume  %}

atdb/taskdatabase/templates/taskdatabase/discarded/clear_filter_button.html

+<a href="{% url 'clear-filter' 'discarded' %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-window-close"></i> Clear Filter</a>

atdb/taskdatabase/templates/taskdatabase/discarded/headers.html

+<tr>
+    <th width="5%">
+        <a href="{% url 'sort-tasks' '-pk' 'discarded' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        ID
+        <a href="{% url 'sort-tasks' 'id' 'discarded' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>Details</th>
+     <th width="8%">
+        <a href="{% url 'sort-tasks' '-status' 'discarded' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        Status
+        <a href="{% url 'sort-tasks' 'status' 'discarded' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th width="8%">
+        <a href="{% url 'sort-tasks' '-project' 'discarded' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        Project
+        <a href="{% url 'sort-tasks' 'project' 'discarded' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th width="8%">
+        <a href="{% url 'sort-tasks' '-sas_id' 'discarded' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        SAS_ID
+        <a href="{% url 'sort-tasks' 'sas_id' 'discarded' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>
+        <a href="{% url 'sort-tasks' '-filter' 'discarded' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        Filter
+        <a href="{% url 'sort-tasks' 'filter' 'discarded' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>Cleanup Policy</th>
+    <th>Reason</th>
+</tr>
\ No newline at end of file