Merge branch 'package-update-clean' into 'main'

Package update clean

See merge request !48

MIGRATION.md

@@ -18,7 +18,9 @@ location VARCHAR(50),
 activity VARCHAR(50),
 surl VARCHAR(200),
 filesize BIGINT,
-additional_meta JSON);
+antenna_set VARCHAR(50),
+antenna_set VARCHAR(50),
+dysco_compression BOOLEAN);
 
 There are 3 tables in the ldvadmin database:
 - raw_dataproducts
@@ -29,17 +31,17 @@ They all need to be queried to retrieve the wanted information via the commands
 
 raw_dataproducts:
 >INSERT INTO lofardata_dataproduct 
-SELECT obsid AS obs_id, obsid_source AS oid_source, 'LOFAR LTA', dp_type AS dataproduct_type, project, location, activity, uri AS surl, size AS filesize, json_build_object('dysco_compression', dysco, 'antenna_set', antenna_set, 'instrument_filter', instrument_filter) AS additional_meta 
+SELECT obsid AS obs_id, obsid_source AS oid_source, 'LOFAR LTA', dp_type AS dataproduct_type, project, location, activity, uri AS surl, size AS filesize, antenna_set, instrument_filter, dysco AS dysco_compression
 FROM astrowise.raw_dataproducts;
 
 pl_dataproducts:
 >INSERT INTO lofardata_dataproduct 
-SELECT obsid AS obs_id, obsid_source AS oid_source, 'LOFAR LTA', dp_type AS dataproduct_type, project, location, activity, uri AS surl, size AS filesize, json_build_object('dysco_compression', dysco) AS additional_meta 
+SELECT obsid AS obs_id, obsid_source AS oid_source, 'LOFAR LTA', dp_type AS dataproduct_type, project, location, activity, uri AS surl, size AS filesize, NULL, NULL, dysco AS dysco_compression
 FROM astrowise.pl_dataproducts;
 
 unsp_dataproducts:
 >INSERT INTO lofardata_dataproduct 
-SELECT obsid AS obs_id, obsid_source AS oid_source, 'LOFAR LTA', dp_type AS dataproduct_type, project, location, activity, uri AS surl, size AS filesize, json_build_object('dysco_compression', dysco) AS additional_meta 
+SELECT obsid AS obs_id, obsid_source AS oid_source, 'LOFAR LTA', dp_type AS dataproduct_type, project, location, activity, uri AS surl, size AS filesize, dysco AS dysco_compression 
 FROM astrowise.unsp_dataproducts;
 
 ### Remove duplicates

README.md

@@ -166,12 +166,12 @@ Vary: Accept
             "activity": "Raw Observation",
             "surl": "srm://lofar-srm.fz-juelich.de:8443/pnfs/fz-juelich.de/data/lofar/ops/projects/lc0_043/102092/L102092_SAP000_SB261_uv.MS_8d9ea7c0.tar",
             "filesize": 1477939200,
-            "additional_meta": "{'dysco_compression': False}",
+            "dysco_compression": "False",
             "location": "Juelich"
         },
 ```
 
-- Enter the filter in json format in the `filter` textbox, for example `{"obs_id": 102092, "additional_meta__dysco_compression": true}`
+- Enter the filter in json format in the `filter` fields, for example `{"obs_id": 102092, "dysco_compression": true}`
 - Choose valid workflow, for example `imaging_compress_pipeline_v02`
 (see the worklows endpoint in the ATDB API for an overview of valid workflows: https://sdc.astron.nl:5554/atdb/workflows/)
 
@@ -188,7 +188,7 @@ Vary: Accept
     "created_on": "2022-08-15T07:07:39.479546Z",
     "filters": {
         "obs_id": 102092,
-        "additional_meta__dysco_compression": true
+        "dysco_compression": true
     },
     "inputs": null,
     "selected_workflow": "imaging_compress_pipeline_v02",

ldvspec/ldvspec/settings/base.py

@@ -36,16 +36,11 @@ INSTALLED_APPS = [
     'rest_framework.authtoken',
     'corsheaders',
     'django_filters',
-    'django_extensions',
     'uws',
     'crispy_forms',
-    'crispy_bootstrap5',
     'widget_tweaks',
 ]
 
-CRISPY_ALLOWED_TEMPLATE_PACKS = "bootstrap5"
-
-CRISPY_TEMPLATE_PACK = "bootstrap5"
 MIDDLEWARE = [
     'corsheaders.middleware.CorsMiddleware',
     'django.middleware.security.SecurityMiddleware',

ldvspec/lofardata/migrations/0014_alter_dataproduct_additional_meta.py

-# Generated by Django 3.2 on 2022-12-15 15:01
-
-import django.contrib.postgres.fields.jsonb
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('lofardata', '0013_auto_20221214_1338'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='dataproduct',
-            name='additional_meta',
-            field=django.contrib.postgres.fields.jsonb.JSONField(),
-        ),
-    ]

ldvspec/lofardata/migrations/0014_auto_20230111_1059.py

+# Generated by Django 3.2 on 2023-01-11 10:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('lofardata', '0013_auto_20221214_1338'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='dataproduct',
+            name='additional_meta',
+        ),
+        migrations.AddField(
+            model_name='dataproduct',
+            name='antenna_set',
+            field=models.CharField(blank=True, default='', max_length=50, null=True),
+        ),
+        migrations.AddField(
+            model_name='dataproduct',
+            name='dysco_compression',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='dataproduct',
+            name='instrument_filter',
+            field=models.CharField(blank=True, default='', max_length=50, null=True),
+        ),
+    ]

ldvspec/lofardata/models.py

 import logging
 from celery.result import AsyncResult
 from django.contrib.auth.models import User
-from django.contrib.postgres.fields import ArrayField, JSONField
+from django.contrib.postgres.fields import ArrayField
 from django.db import models, IntegrityError
 from django.utils.translation import gettext_lazy as _
 logger = logging.getLogger(__name__)
@@ -17,7 +17,9 @@ class DataProduct(models.Model):
     activity = models.CharField(max_length=50)
     surl = models.CharField(max_length=200, unique=True)
     filesize = models.PositiveBigIntegerField()
-    additional_meta = JSONField()
+    antenna_set = models.CharField(max_length=50, blank=True, null=True, default="")
+    instrument_filter = models.CharField(max_length=50, blank=True, null=True, default="")
+    dysco_compression = models.BooleanField(default=False)
 
     @staticmethod
     def insert_dataproduct(
@@ -30,7 +32,9 @@ class DataProduct(models.Model):
             activity,
             surl,
             filesize,
-            additional_meta,
+            antenna_set,
+            instrument_filter,
+            dysco_compression
     ):
 
         dp = DataProduct(
@@ -42,7 +46,9 @@ class DataProduct(models.Model):
             location=location,
             activity=activity,
             filesize=filesize,
-            additional_meta=additional_meta,
+            antenna_set=antenna_set,
+            instrument_filter=instrument_filter,
+            dysco_compression=dysco_compression,
             surl=surl,
         )
         dp.save()

ldvspec/lofardata/templates/lofardata/base.html

@@ -13,9 +13,6 @@
     <!-- loads the path to static files -->
     <link href='https://fonts.googleapis.com/css?family=Raleway' rel='stylesheet' type='text/css'>
 
-    <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.4.2/css/all.css"
-          integrity="sha384-/rXc/GQVaYpyDdyxK+ecHPVYJSN9bmVFBvjA/9eOB+pb3F2w2N6fc5qB9Ew5yIns" crossorigin="anonymous">
-
     <link rel="stylesheet" type="text/css" href="{% static 'lofardata/styling/dias.css' %}">
 
     {% block extra_js %}{% endblock %}

ldvspec/lofardata/tests/test_data_filter.py

@@ -9,7 +9,9 @@ test_object_values = [dict(obs_id='12345', oid_source='SAS', dataproduct_source=
                            activity='observation',
                            surl='srm://surfsara.nl:4884/...',
                            filesize=40,
-                           additional_meta={'dysco_compression': True}),
+                           antenna_set="HBA Dual Inner",
+                           instrument_filter="110-190 MHz",
+                           dysco_compression=True),
                       dict(obs_id='12346', oid_source='SAS', dataproduct_source='lofar',
                            dataproduct_type='observation',
                            project='LT10_10',
@@ -17,7 +19,9 @@ test_object_values = [dict(obs_id='12345', oid_source='SAS', dataproduct_source=
                            activity='pipeline',
                            surl='srm://surfsara.nl:4884/---',
                            filesize=40,
-                           additional_meta={'dysco_compression': True}),
+                           antenna_set="HBA Dual Inner",
+                           instrument_filter="110-190 MHz",
+                           dysco_compression=True),
                       dict(obs_id='12347', oid_source='SAS', dataproduct_source='lofar',
                            dataproduct_type='observation',
                            project='LT10_10',
@@ -25,7 +29,9 @@ test_object_values = [dict(obs_id='12345', oid_source='SAS', dataproduct_source=
                            activity='observation',
                            surl='srm://surfsara.nl:4884/xxx',
                            filesize=40,
-                           additional_meta={'dysco_compression': True})]
+                           antenna_set=None,
+                           instrument_filter=None,
+                           dysco_compression=True)]
 
 
 class TestDataFilter(rtest.APITestCase):
@@ -55,4 +61,4 @@ class TestDataFilter(rtest.APITestCase):
         self.assertEqual(2, response.data['count'])
         self.assertEqual('observation', response.data['results'][0]['activity'])
 
-        self.assertIn('activity_type', self.get_current_query_parameters())
\ No newline at end of file
+        self.assertIn('activity_type', self.get_current_query_parameters())

ldvspec/lofardata/tests/test_dataproduct.py

@@ -13,7 +13,9 @@ test_object_value = dict(obs_id='12345', oid_source='SAS', dataproduct_source='l
                          activity='observation',
                          surl='srm://lta-head.lofar.psnc.pl:4884/subfolder/some_nice.tar',
                          filesize=40,
-                         additional_meta={'dysco_compression': True})
+                         antenna_set="HBA Dual Inner",
+                         instrument_filter= "110-190 MHz",
+                         dysco_compression=True)
 
 
 class TestDatabaseInteraction(dtest.TestCase):
@@ -127,4 +129,4 @@ class TestRESTAPI(rtest.APITestCase):
 
         response = self.client.post('/ldvspec/api/v1/insert_dataproduct/', data=test_payload, format='json')
 
-        self.assertEqual(response_status.HTTP_400_BAD_REQUEST, response.status_code)
\ No newline at end of file
+        self.assertEqual(response_status.HTTP_400_BAD_REQUEST, response.status_code)

ldvspec/lofardata/tests/test_workrequest.py

@@ -25,7 +25,7 @@ class TestWorkSpecificationRequest(rtest.APITestCase):
         # Insert test data
         DataProduct.insert_dataproduct(12345, 'lta',
                                        'myniceinstrument', 'ms', 'blubby', 'location-uri', 'pipline',
-                                       'srm://lofarlta/mynice_file.tar.gz', 123456, {'dysco_compression': False})
+                                       'srm://lofarlta/mynice_file.tar.gz', 123456, None, None, False)
 
         response = self.client.post('/ldvspec/api/v1/workspecification/',
                                     data={'filters': {'obs_id': 12345}}, format='json')
@@ -49,14 +49,14 @@ class TestWorkSpecificationRequest(rtest.APITestCase):
         # Insert test data
         DataProduct.insert_dataproduct(12345, 'lta',
                                        'myniceinstrument', 'ms', 'blubby', 'location-uri', 'pipline',
-                                       'srm://lofarlta/mynice_file.tar.gz', 123456, {'dysco_compression': False})
+                                       'srm://lofarlta/mynice_file.tar.gz', 123456, None, None, False)
         DataProduct.insert_dataproduct(12345, 'lta',
                                        'myniceinstrument', 'ms', 'blubby', 'location-uri', 'pipline',
-                                       'srm://lofarlta/mynice_file_with_rhythm.tar.gz', 123456,
-                                       {'dysco_compression': True})
+                                       'srm://lofarlta/mynice_file_with_rhythm.tar.gz', 123456, "HBA Dual Inner",
+                                       "110-190 MHz", True)
 
         response = self.client.post('/ldvspec/api/v1/workspecification/',
-                                    data={'filters': {'obs_id': 12345, 'additional_meta__dysco_compression': True}},
+                                    data={'filters': {'obs_id': 12345, 'dysco_compression': True}},
                                     format='json')
 
         self.assertEqual(201, response.status_code)

ldvspec/requirements/base.txt

-Django==3.2
-djangorestframework==3.12.2
-django-filter==2.3.0
-psycopg2-binary==2.9.3
-django-cors-headers==3.6.0
-django-extensions==3.1.0
-django-bootstrap-pagination==1.7.0
-pytz==2022.1
-whitenoise==5.0.1
-six==1.15.0
-fontawesome-free==5.15.2
-pyyaml==6.0
-uritemplate==4.1.1
-sshtunnel==0.4.0
-django-uws==0.2.dev355575
-django-crispy-forms==1.14.0
-crispy-bootstrap5==0.6
-humanize==4.4.0
+Django
+django-filter==2.4.0 #newest version 1.22 not compatible
+djangorestframework
+django-cors-headers
+django-uws
+django-crispy-forms
 django-widget-tweaks
-numpy==1.23.0
\ No newline at end of file
+uritemplate
+psycopg2-binary
+whitenoise
+pyyaml
+numpy
\ No newline at end of file