Implementation of creating work specifications for a group

ldvspec/lofardata/forms.py

@@ -2,7 +2,7 @@ from django.core.exceptions import ValidationError
 
 from .models import WorkSpecification, DataProductFilter, Group
 
-from django.forms import ModelForm, CharField
+from django.forms import ModelForm, CharField, ChoiceField, ModelChoiceField
 
 
 class WorkSpecificationForm(ModelForm):
@@ -10,6 +10,7 @@ class WorkSpecificationForm(ModelForm):
         super().__init__(*args, **kwargs)
         self.fields['predecessor_specification'].required = False
         self.fields['filters'].required = False
+        self.fields['group'].required = False
 
     def _extract_filters(self):
         filter_names = [filter_name[0] for filter_name in DataProductFilter.objects.all().values_list('field')]

ldvspec/lofardata/templates/lofardata/index.html

@@ -14,7 +14,6 @@
                href="{% url 'specification-create' %}">
                 <span class="icon icon--plus"></span></a>
             <a class="button button--secondary button--icon-button margin-left"
-               style="display: none"
                title="Create a new group"
                href="{% url 'group-create' %}">
                 <span class="icon icon--layer-plus"></span></a>

ldvspec/lofardata/templates/lofardata/workspecification/create_update.html

@@ -24,10 +24,9 @@
                         <!-- Standard input fields -->
                         <div class="custom--div-margin">
                             <h3 class="text text--primary text--title">Selection</h3>
-                            <div class="flex-wrapper flex-wrapper--row" id="div_id_group" style="display: none">
+                            <div class="flex-wrapper flex-wrapper--row" id="div_id_group">
                                 <div class="flex-wrapper flex-wrapper--row custom__input--fixed-min-width">
-                                    <label class="input__label"
-                                           for="id_group">{{ form.group.label }}</label>
+                                    <label class="input__label">{{ form.group.label }}</label>
                                     <a class="tooltip-dias tooltip-dias-right custom--tooltip"
                                        data-tooltip="{{ form.group.help_text }}">m</a>
                                 </div>

ldvspec/lofardata/tests/test_dataproduct.py

-import django.test as dtest
 import rest_framework.test as rtest
 from django.contrib.auth.models import User
 import rest_framework.status as response_status
 from django.db import IntegrityError
 
+import django.test as dtest
+
 from lofardata.models import DataProduct
 
 test_object_value = dict(obs_id='12345', oid_source='SAS', dataproduct_source='lofar',
@@ -29,6 +30,7 @@ class TestDatabaseInteraction(dtest.TestCase):
             self.assertEqual(getattr(dp, field), test_values.get(field))
 
     def test_insert_with_helper_function(self):
+        test_object_value['surl'] = test_object_value['surl'] + 'rrr'
         test_values = dict(test_object_value)
 
         dp = DataProduct.insert_dataproduct(**test_values)

ldvspec/lofardata/tests/test_dataproductinfo.py

-import unittest
-
 from lofardata.models import DataProduct
 from lofardata.view_helpers.dataproductinfo import retrieve_combined_information
 
+import django.test as dtest
+
 test_objects = [dict(obs_id='12345', oid_source='SAS', dataproduct_source='lofar',
                      dataproduct_type='observation',
                      project='LT10_10',
                      location='lta-head.lofar.psnc.pl',
                      activity='secret stuff',
-                     surl='srm://lta-head.lofar.psnc.pl:4884/subfolder/some_nice.tar',
+                     surl='srm://lta-head.lofar.psnc.pl:4884/subfolder/a_some_nice.tar',
                      filesize=40,
                      antenna_set='HBA Dual Inner',
                      instrument_filter='110-190 MHz',
@@ -114,7 +114,7 @@ test_objects = [dict(obs_id='12345', oid_source='SAS', dataproduct_source='lofar
                      dysco_compression=False)]
 
 
-class RetrieveDataProductInformation(unittest.TestCase):
+class RetrieveDataProductInformation(dtest.TestCase):
 
     def setUp(self):
         for test_object in test_objects:

ldvspec/lofardata/tests/test_group_creation.py

-import unittest
+import types
+from unittest import mock
 
-from lofardata.models import WorkSpecification, Group
+import django.test as dtest
 
-class WorkSpecificationCreation(unittest.TestCase):
-    def test_create_group_with_one_obs_id(self):
-        pass
+from django.contrib.auth.models import User
+from lofardata.models import WorkSpecification, Group, ATDBProcessingSite
+from lofardata.view_helpers.specification import create_work_specifications_for_group, \
+    create_work_specification_for_group
 
-    def test_create_group_with_multiple_obs_ids(self):
-        pass
+
+class WorkSpecificationCreationDatabaseInteraction(dtest.TestCase):
+    def setUp(self):
+        self.user, _ = User.objects.get_or_create()
+        self.site, _ = ATDBProcessingSite.objects.get_or_create(
+            name="marvel", url="http://marvel.com", access_token="dummy"
+        )
+        self.group, _ = Group.objects.get_or_create(name="x-men", processing_site=self.site, selected_workflow="claw",
+                                                    selected_workflow_tag="attack")
+
+        self.delay_return_value = types.SimpleNamespace(id='5')
+
+    @mock.patch('lofardata.tasks.define_work_specification.delay')
+    def test_create_group_with_one_obs_id(self, mock_delay):
+        mock_delay.return_value = self.delay_return_value
+        create_work_specification_for_group(self.group, self.user, "123")
+
+        work_specification = list(WorkSpecification.objects.filter(group=self.group.id))
+
+        self.assertEqual(len(work_specification), 1)
+
+        actual_inherited_group_values = [(x.processing_site, x.selected_workflow, x.selected_workflow_tag) for x in
+                                         work_specification if x.group is self.group]
+
+        for actual_inherited_group_value in actual_inherited_group_values:
+            self.assertTupleEqual(actual_inherited_group_value,
+                                  (self.site, self.group.selected_workflow, self.group.selected_workflow_tag))
+
+    @mock.patch('lofardata.tasks.define_work_specification.delay')
+    def test_create_group_with_multiple_obs_ids(self, mock_delay):
+        mock_delay.return_value = self.delay_return_value
+        create_work_specifications_for_group(self.group, self.user, ["123", "456", " 781 "])
+
+        work_specification = list(WorkSpecification.objects.filter(group=self.group.id))
+
+        self.assertEqual(len(work_specification), 3)
+
+        actual_inherited_group_values = [(x.processing_site, x.selected_workflow, x.selected_workflow_tag) for x in
+                                         work_specification if x.group is self.group]
+
+        for actual_inherited_group_value in actual_inherited_group_values:
+            self.assertTupleEqual(actual_inherited_group_value,
+                                  (self.site, self.group.selected_workflow, self.group.selected_workflow_tag))
+
+    @mock.patch('lofardata.tasks.define_work_specification.delay')
+    def test_create_group_with_filters(self, mock_delay):
+        mock_delay.return_value = self.delay_return_value
+        obs_ids = ['123', '456']
+        create_work_specifications_for_group(self.group, self.user, obs_ids)
+
+        work_specification = list(WorkSpecification.objects.filter(group=self.group.id))
+
+        actual_obs_ids = [x.filters['obs_id'] for x in work_specification if x.filters['obs_id'] in obs_ids]
+
+        # assertCountEqual: Asserts that two iterables have the same elements, the same number of
+        # times, without regard to order.
+        self.assertCountEqual(actual_obs_ids, obs_ids)
+
+    @mock.patch("lofardata.view_helpers.specification.set_post_submit_values")
+    @mock.patch('lofardata.tasks.define_work_specification.delay')
+    def test_create_group_with_post_submit_values(self, mock_delay, post_submit_mock):
+        mock_delay.return_value = self.delay_return_value
+        create_work_specifications_for_group(self.group, self.user, ["123"])
+        self.assertTrue(post_submit_mock.called)

ldvspec/lofardata/tests/test_split_obs_ids.py

+import unittest
+
+from lofardata.view_helpers.specification import split_obs_ids_string
+
+
+class TestSplitObsIds(unittest.TestCase):
+    def test_split_obs_ids_separate_by_comma(self):
+        actual = split_obs_ids_string('1,2,3')
+        self.assertEqual(actual, ['1', '2', '3'])
+
+    def test_split_obs_ids_trim_spaces(self):
+        actual = split_obs_ids_string('4  ,  5,6      ')
+        self.assertEqual(actual, ['4', '5', '6'])
+
+    def test_split_obs_ids_trim_newlines(self):
+        actual = split_obs_ids_string('4\n,5\n\n,\n6\n')
+        self.assertEqual(actual, ['4', '5', '6'])
+
+    def test_split_obs_ids_trim_tabs(self):
+        actual = split_obs_ids_string('4\t,5\t\t,\t6\t')
+        self.assertEqual(actual, ['4', '5', '6'])
\ No newline at end of file

ldvspec/lofardata/tests/test_workspecification_creation.py

@@ -3,11 +3,11 @@ import unittest
 from django.contrib.auth.models import User
 
 from lofardata.models import WorkSpecification
-from lofardata.views import set_post_submit_values
+from lofardata.view_helpers.specification import set_post_submit_values
+
 
 class WorkSpecificationCreation(unittest.TestCase):
     def test_set_created_by_when_already_set(self):
-
         existing_user = User(pk=1, username='existing')
         new_user = User(pk=2, username='new')
 

ldvspec/lofardata/view_helpers/specification.py

+from typing import List
+
+from django.contrib.auth.models import User
+
+from lofardata.models import Group, WorkSpecification
+
+
+def set_post_submit_values(specification, user):
+    specification.async_task_result = None
+    specification.is_ready = False
+    if specification.created_by is None:
+        specification.created_by = user
+
+
+def split_obs_ids_string(obs_ids_string: str) -> List[str]:
+    return [obs_id.strip() for obs_id in obs_ids_string.split(',')]
+
+
+def create_work_specifications_for_group(group: Group, created_by: User, obs_ids: List[str]):
+    for obs_id in obs_ids:
+        create_work_specification_for_group(group, created_by, obs_id)
+
+
+def create_work_specification_for_group(group: Group, created_by: User, obs_id: str):
+    workspecification = WorkSpecification(
+        processing_site=group.processing_site,
+        selected_workflow=group.selected_workflow,
+        selected_workflow_tag=group.selected_workflow_tag,
+        filters={"obs_id": obs_id},
+        group=group
+    )
+
+    set_post_submit_values(workspecification, created_by)
+
+    workspecification.save()

ldvspec/lofardata/views.py

@@ -31,6 +31,8 @@ from .serializers import (
 )
 from .tasks import insert_task_into_atdb
 from .view_helpers import inputs_processor, dataproductinfo, filters_preprocessor
+from .view_helpers.specification import set_post_submit_values, split_obs_ids_string, \
+    create_work_specifications_for_group
 
 
 class DynamicFilterSet(filters.FilterSet):
@@ -67,14 +69,6 @@ def api(request):
     atdb_hosts = ATDBProcessingSite.objects.values("name", "url")
     return render(request, "lofardata/api.html", {"atdb_hosts": atdb_hosts})
 
-
-def set_post_submit_values(specification, user):
-    specification.async_task_result = None
-    specification.is_ready = False
-    if specification.created_by is None:
-        specification.created_by = user
-
-
 class Specifications(ListView):
     serializer_class = WorkSpecificationSerializer
     template_name = "lofardata/index.html"
@@ -116,7 +110,6 @@ class WorkSpecificationCreateUpdateView(LoginRequiredMixin, CanAccessWorkSpecifi
             specification = None
         context["filters"] = filters_preprocessor.preprocess(specification)
         context["processing_sites"] = list(ATDBProcessingSite.objects.values("name", "url"))
-
         return context
 
     def create_successor(self, specification):
@@ -311,3 +304,14 @@ class GroupCreateUpdateView(LoginRequiredMixin, UpdateView):
         context = super().get_context_data(**kwargs)
         context["processing_sites"] = list(ATDBProcessingSite.objects.values("name", "url"))
         return context
+
+    def form_valid(self, form):
+        group = form.save()
+        obs_ids = split_obs_ids_string(form.data.get("obs_ids"))
+        create_work_specifications_for_group(group, self.request.user, obs_ids)
+
+        return super().form_valid(form)
+
+    def get_success_url(self, **kwargs):
+        return reverse_lazy("index")
+