various database changes

- adding job table
see https://support.astron.nl/jira/browse/SDC-142

atdb/taskdatabase/migrations/0004_auto_20210330_1420.py

+# Generated by Django 3.1.4 on 2021-03-30 12:20
+
+import datetime
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('taskdatabase', '0003_auto_20210301_1206'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Job',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('type', models.CharField(blank=True, default=None, max_length=20, null=True)),
+                ('task_id', models.IntegerField(blank=True, null=True)),
+                ('job_id', models.IntegerField(blank=True, null=True)),
+                ('metadata', models.JSONField(blank=True, null=True)),
+            ],
+        ),
+        migrations.AddField(
+            model_name='logentry',
+            name='service',
+            field=models.CharField(blank=True, max_length=30, null=True),
+        ),
+        migrations.AddField(
+            model_name='logentry',
+            name='size_processed',
+            field=models.PositiveBigIntegerField(blank=True, default=0, null=True),
+        ),
+        migrations.AddField(
+            model_name='workflow',
+            name='oi_size_fraction',
+            field=models.FloatField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='task',
+            name='creationTime',
+            field=models.DateTimeField(blank=True, default=datetime.datetime.utcnow, verbose_name='CreationTime'),
+        ),
+    ]

atdb/taskdatabase/migrations/0005_auto_20210330_1437.py

+# Generated by Django 3.1.4 on 2021-03-30 12:37
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('taskdatabase', '0004_auto_20210330_1420'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='job',
+            name='type',
+            field=models.CharField(blank=True, db_index=True, default=None, max_length=20, null=True),
+        ),
+    ]

atdb/taskdatabase/models.py

@@ -11,6 +11,7 @@ class Workflow(models.Model):
     repository = models.CharField(max_length=100, blank=True, null=True)
     commit_id = models.CharField(max_length=15, blank=True, null=True)
     path = models.CharField(max_length=100, blank=True, null=True)
+    oi_size_fraction = models.FloatField(blank=True, null=True)
 
     def __str__(self):
         return str(self.id)
@@ -61,11 +62,12 @@ class LogEntry(models.Model):
     cpu_cycles = models.IntegerField(null=True,blank=True)
     wall_clock_time = models.IntegerField(null=True,blank=True)
     url_to_log_file = models.CharField(max_length=100, blank=True, null=True)
+    service = models.CharField(max_length=30, blank=True, null=True)
     step_name = models.CharField(max_length=30, blank=True, null=True)
     timestamp = models.DateTimeField(blank=True, null=True)
     status = models.CharField(max_length=50,default="defined", blank=True, null=True)
     description = models.CharField(max_length=100, blank=True, null=True)
-
+    size_processed = models.PositiveBigIntegerField(default=0, null=True, blank=True)
     # relationships
     task = models.ForeignKey(Task, related_name='log_entries', on_delete=models.CASCADE, null=False)
 
@@ -93,3 +95,14 @@ class Configuration(models.Model):
     # the representation of the value in the REST API
     def __str__(self):
         return str(self.key)
+
+
+class Job(models.Model):
+    type = models.CharField(db_index=True, max_length=20, default=None,null=True, blank=True)
+    task_id = models.IntegerField(null=True, blank=True)
+    job_id = models.IntegerField(null=True, blank=True)
+    metadata = models.JSONField(null=True, blank=True)
+
+    # the representation of the value in the REST API
+    def __str__(self):
+        return str(self.id)
\ No newline at end of file

atdb/taskdatabase/serializers.py

 from rest_framework import serializers
-from .models import Status, Task, Workflow, LogEntry, Configuration
+from .models import Status, Task, Workflow, LogEntry, Configuration, Job
 
 
 
@@ -154,3 +154,10 @@ class ConfigurationSerializer(serializers.ModelSerializer):
     class Meta:
         model = Configuration
         fields = "__all__"
+
+
+class JobSerializer(serializers.ModelSerializer):
+
+    class Meta:
+        model = Job
+        fields = "__all__"

atdb/taskdatabase/templates/taskdatabase/details/workflow_details.html

@@ -12,6 +12,7 @@
             <tr><td><b>repository</b></td><td><a href="{{ workflow.repository }}" target="_blank">{{ workflow.repository }} </a></td></tr>
             <tr><td><b>commit_id</b></td><td>{{ workflow.commit_id }}</td></tr>
             <tr><td><b>path</b></td><td>{{ workflow.path }}</td></tr>
+            <tr><td><b>oi_size_fraction</b></td><td>{{ workflow.oi_size_fraction }}</td></tr>
             <tr>
                 <td><b>workflows list</b></td><td><a href="{% url 'workflows-api' %}" class="btn btn-secondary btn-sm" role="button" target="_blank">REST API</a></td>
             </tr>

atdb/taskdatabase/templates/taskdatabase/index.html

@@ -80,7 +80,7 @@
         {% include 'taskdatabase/pagination.html' %}
        </div>
     </div>
-    <p class="footer"> Version 1.0.0 (26 mar 2021 - 19:00)
+    <p class="footer"> Version 1.0.0 (30 mar 2021 - 14:00)
 
 </div>
 

atdb/taskdatabase/urls.py

@@ -43,6 +43,9 @@ urlpatterns = [
     path('configuration/', views.ConfigurationListViewAPI.as_view()),
     path('configuration/<int:pk>/', views.ConfigurationDetailsViewAPI.as_view(), name='configuration-detail-view-api'),
 
+    path('jobs/', views.JobListViewAPI.as_view()),
+    path('jobs/<int:pk>/', views.JobDetailsViewAPI.as_view(), name='job-detail-view-api'),
+
     # --- custom requests ---
     # /atdb/get_size?status__in=defined,staged
     path('tasks/get_size', views.GetSizeView.as_view(), name='get-size-view'),

atdb/taskdatabase/views.py

@@ -23,7 +23,7 @@ from django_tables2 import SingleTableView
 
 from django.conf import settings
 
-from .models import Task, Status, Workflow, LogEntry, Configuration
+from .models import Task, Status, Workflow, LogEntry, Configuration, Job
 from .tables import TaskTable
 
 from django.db.models import Q
@@ -33,7 +33,8 @@ from .serializers import \
     TaskReadSerializerFast, \
     WorkflowSerializer,\
     LogEntrySerializer,\
-    ConfigurationSerializer
+    ConfigurationSerializer,\
+    JobSerializer
 
 from .services import algorithms
 
@@ -104,6 +105,17 @@ class ConfigurationFilter(filters.FilterSet):
             'key': ['exact', 'icontains'],
         }
 
+class JobFilter(filters.FilterSet):
+
+    class Meta:
+        model = Job
+
+        fields = {
+            'type': ['exact', 'icontains'],
+            'task_id': ['exact'],
+            'job_id': ['exact'],
+        }
+
 # ---------- Tables2 Views (experimental) -----------
 # implementation with tables2: http://localhost:8000/atdb/tables2
 class QueryView(SingleTableMixin, FilterView):
@@ -421,6 +433,21 @@ class ConfigurationDetailsViewAPI(generics.RetrieveUpdateDestroyAPIView):
     queryset = Configuration.objects.all()
     serializer_class = ConfigurationSerializer
 
+# example: /atdb/job/
+class JobListViewAPI(generics.ListCreateAPIView):
+    model = Job
+    queryset = Job.objects.all()
+    serializer_class = JobSerializer
+
+    filter_backends = (filters.DjangoFilterBackend,)
+    filter_class = JobFilter
+
+
+# example: /atdb/job/5/
+class JobDetailsViewAPI(generics.RetrieveUpdateDestroyAPIView):
+    model = Job
+    queryset = Job.objects.all()
+    serializer_class = JobSerializer
 
 # --- controller resources, triggered by a button in the GUI or directoy with a URL ---
 # set task status to 'new_status' - called from the GUI