Merge branch 'SDC-1422-workflow-table-database-changes' into 'master'

add some fields to workflow and activity for the future aggregation functionality.

See merge request !360

atdb/taskdatabase/migrations/0047_activity_storage_location_and_more.py

+# Generated by Django 5.0 on 2024-07-16 11:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('taskdatabase', '0046_rename_is_combined_activity_is_aggregated_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='activity',
+            name='storage_location',
+            field=models.CharField(blank=True, default='unknown', max_length=250, null=True),
+        ),
+        migrations.AddField(
+            model_name='workflow',
+            name='aggregation_script',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AddField(
+            model_name='workflow',
+            name='aggregation_strategy',
+            field=models.CharField(choices=[('none', 'none'), ('wait_for_summary_task', 'wait_for_summary_task'), ('collect_h5', 'collect_h5')], default='none', max_length=50),
+        ),
+        migrations.AddField(
+            model_name='workflow',
+            name='quality_thresholds',
+            field=models.CharField(blank=True, max_length=50, null=True),
+        ),
+    ]

atdb/taskdatabase/migrations/0048_alter_activity_storage_location.py

+# Generated by Django 5.0 on 2024-07-16 11:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('taskdatabase', '0047_activity_storage_location_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='activity',
+            name='storage_location',
+            field=models.CharField(blank=True, default=None, max_length=250, null=True),
+        ),
+    ]

atdb/taskdatabase/models.py

@@ -15,6 +15,11 @@ logger = logging.getLogger(__name__)
 # constants
 datetime_format_string = '%Y-%m-%dT%H:%M:%SZ'
 
+AGGREGATION_STRATEGY_CHOICES = (
+    ("none", "none"),
+    ("wait_for_summary_task", "wait_for_summary_task"),
+    ("collect_h5", "collect_h5"),
+)
 
 class Workflow(models.Model):
     description = models.CharField(max_length=500, blank=True, null=True)
@@ -28,6 +33,11 @@ class Workflow(models.Model):
     default_parameters = models.JSONField(null=True, blank=True)
     prefetch = models.BooleanField(null=True, default=True)
 
+    # this is a fixed list of options, because when an option is added it also requires changes in the aggregator service
+    aggregation_strategy = models.CharField(max_length=50, choices = AGGREGATION_STRATEGY_CHOICES, default="none")
+    aggregation_script = models.CharField(max_length=100, blank=True, null=True)
+    quality_thresholds = models.CharField(max_length=50, blank=True, null=True)
+
     def __str__(self):
         return str(self.id) + ' - ' + str(self.workflow_uri)
 
@@ -120,6 +130,8 @@ class Activity(models.Model):
 
     # flag set (and used) by the aggregator service, so that it doesn't do double work
     is_aggregated = models.BooleanField(default=False)
+    # storage_location for aggregation files
+    storage_location = models.CharField(max_length=250, blank=True, null=True, default=None)
 
     finished_fraction = models.FloatField(blank=True, null=True)
     ingested_fraction = models.FloatField(blank=True, null=True)

atdb/taskdatabase/templates/taskdatabase/details/workflow_details.html

@@ -14,6 +14,12 @@
             <tr><td><b>repository</b></td><td><a href="{{ workflow.repository }}" target="_blank">{{ workflow.repository }} </a></td></tr>
             <tr><td><b>commit_id</b></td><td>{{ workflow.commit_id }}</td></tr>
             <tr><td><b>path</b></td><td>{{ workflow.path }}</td></tr>
+
+            {% if workflow.aggregation_strategy != "none" %}
+            <tr><td><b>aggregration strategy</b></td><td>{{ workflow.aggregation_strategy }}</td></tr>
+            <tr><td><b>aggregration script</b></td><td>{{ workflow.aggregation_script }}</td></tr>
+            {% endif %}
+
             <tr><td><b>oi_size_fraction</b></td><td>{{ workflow.oi_size_fraction }}</td></tr>
             <tr>
                 <td><b>workflows list</b></td><td><a href="{% url 'workflows-api' %}" class="btn btn-secondary btn-sm" role="button" target="_blank">REST API</a></td>

atdb/taskdatabase/templates/taskdatabase/index.html

@@ -31,7 +31,7 @@
             {% include 'taskdatabase/pagination.html' %}
         </div>
     </div>
-    <p class="footer"> Version 15 Jul 2024
+    <p class="footer"> Version 16 Jul 2024
 </div>
 
 {% include 'taskdatabase/refresh.html' %}