diff --git a/MIGRATION.md b/MIGRATION.md
index 18f5330bdcc6694eb24574d3264397e509207507..d54a95ea91ab178795a60db82e57f425d094cacd 100644
--- a/MIGRATION.md
+++ b/MIGRATION.md
@@ -18,7 +18,9 @@ location VARCHAR(50),
activity VARCHAR(50),
surl VARCHAR(200),
filesize BIGINT,
-additional_meta JSON);
+antenna_set VARCHAR(50),
+antenna_set VARCHAR(50),
+dysco_compression BOOLEAN);
There are 3 tables in the ldvadmin database:
- raw_dataproducts
@@ -29,17 +31,17 @@ They all need to be queried to retrieve the wanted information via the commands
raw_dataproducts:
>INSERT INTO lofardata_dataproduct
-SELECT obsid AS obs_id, obsid_source AS oid_source, 'LOFAR LTA', dp_type AS dataproduct_type, project, location, activity, uri AS surl, size AS filesize, json_build_object('dysco_compression', dysco, 'antenna_set', antenna_set, 'instrument_filter', instrument_filter) AS additional_meta
+SELECT obsid AS obs_id, obsid_source AS oid_source, 'LOFAR LTA', dp_type AS dataproduct_type, project, location, activity, uri AS surl, size AS filesize, antenna_set, instrument_filter, dysco AS dysco_compression
FROM astrowise.raw_dataproducts;
pl_dataproducts:
>INSERT INTO lofardata_dataproduct
-SELECT obsid AS obs_id, obsid_source AS oid_source, 'LOFAR LTA', dp_type AS dataproduct_type, project, location, activity, uri AS surl, size AS filesize, json_build_object('dysco_compression', dysco) AS additional_meta
+SELECT obsid AS obs_id, obsid_source AS oid_source, 'LOFAR LTA', dp_type AS dataproduct_type, project, location, activity, uri AS surl, size AS filesize, NULL, NULL, dysco AS dysco_compression
FROM astrowise.pl_dataproducts;
unsp_dataproducts:
>INSERT INTO lofardata_dataproduct
-SELECT obsid AS obs_id, obsid_source AS oid_source, 'LOFAR LTA', dp_type AS dataproduct_type, project, location, activity, uri AS surl, size AS filesize, json_build_object('dysco_compression', dysco) AS additional_meta
+SELECT obsid AS obs_id, obsid_source AS oid_source, 'LOFAR LTA', dp_type AS dataproduct_type, project, location, activity, uri AS surl, size AS filesize, dysco AS dysco_compression
FROM astrowise.unsp_dataproducts;
### Remove duplicates
diff --git a/README.md b/README.md
index 499992a5d5f8c9a614f17f7a267f68f8c962d8bd..1a3dbd652d9652f3f7c4390efdb6c079af4d0b9e 100644
--- a/README.md
+++ b/README.md
@@ -166,12 +166,12 @@ Vary: Accept
"activity": "Raw Observation",
"surl": "srm://lofar-srm.fz-juelich.de:8443/pnfs/fz-juelich.de/data/lofar/ops/projects/lc0_043/102092/L102092_SAP000_SB261_uv.MS_8d9ea7c0.tar",
"filesize": 1477939200,
- "additional_meta": "{'dysco_compression': False}",
+ "dysco_compression": "False",
"location": "Juelich"
},
```
-- Enter the filter in json format in the `filter` textbox, for example `{"obs_id": 102092, "additional_meta__dysco_compression": true}`
+- Enter the filter in json format in the `filter` fields, for example `{"obs_id": 102092, "dysco_compression": true}`
- Choose valid workflow, for example `imaging_compress_pipeline_v02`
(see the worklows endpoint in the ATDB API for an overview of valid workflows: https://sdc.astron.nl:5554/atdb/workflows/)
@@ -188,7 +188,7 @@ Vary: Accept
"created_on": "2022-08-15T07:07:39.479546Z",
"filters": {
"obs_id": 102092,
- "additional_meta__dysco_compression": true
+ "dysco_compression": true
},
"inputs": null,
"selected_workflow": "imaging_compress_pipeline_v02",
diff --git a/ldvspec/ldvspec/settings/base.py b/ldvspec/ldvspec/settings/base.py
index 4261220228a12edc0774f1ef1953e18353ac0edd..73257f37a0c6a8e57f33af731e9977746f248c5d 100644
--- a/ldvspec/ldvspec/settings/base.py
+++ b/ldvspec/ldvspec/settings/base.py
@@ -36,16 +36,11 @@ INSTALLED_APPS = [
'rest_framework.authtoken',
'corsheaders',
'django_filters',
- 'django_extensions',
'uws',
'crispy_forms',
- 'crispy_bootstrap5',
'widget_tweaks',
]
-CRISPY_ALLOWED_TEMPLATE_PACKS = "bootstrap5"
-
-CRISPY_TEMPLATE_PACK = "bootstrap5"
MIDDLEWARE = [
'corsheaders.middleware.CorsMiddleware',
'django.middleware.security.SecurityMiddleware',
diff --git a/ldvspec/lofardata/migrations/0014_alter_dataproduct_additional_meta.py b/ldvspec/lofardata/migrations/0014_alter_dataproduct_additional_meta.py
deleted file mode 100644
index b24e3d80d44a675e7e27e5a4cb313a01d6dfcfc5..0000000000000000000000000000000000000000
--- a/ldvspec/lofardata/migrations/0014_alter_dataproduct_additional_meta.py
+++ /dev/null
@@ -1,19 +0,0 @@
-# Generated by Django 3.2 on 2022-12-15 15:01
-
-import django.contrib.postgres.fields.jsonb
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('lofardata', '0013_auto_20221214_1338'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='dataproduct',
- name='additional_meta',
- field=django.contrib.postgres.fields.jsonb.JSONField(),
- ),
- ]
diff --git a/ldvspec/lofardata/migrations/0014_auto_20230111_1059.py b/ldvspec/lofardata/migrations/0014_auto_20230111_1059.py
new file mode 100644
index 0000000000000000000000000000000000000000..be948aaa535ad754b6d8cbad66439b364510be9c
--- /dev/null
+++ b/ldvspec/lofardata/migrations/0014_auto_20230111_1059.py
@@ -0,0 +1,32 @@
+# Generated by Django 3.2 on 2023-01-11 10:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('lofardata', '0013_auto_20221214_1338'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='dataproduct',
+ name='additional_meta',
+ ),
+ migrations.AddField(
+ model_name='dataproduct',
+ name='antenna_set',
+ field=models.CharField(blank=True, default='', max_length=50, null=True),
+ ),
+ migrations.AddField(
+ model_name='dataproduct',
+ name='dysco_compression',
+ field=models.BooleanField(default=False),
+ ),
+ migrations.AddField(
+ model_name='dataproduct',
+ name='instrument_filter',
+ field=models.CharField(blank=True, default='', max_length=50, null=True),
+ ),
+ ]
diff --git a/ldvspec/lofardata/models.py b/ldvspec/lofardata/models.py
index 5ca806233ca3e53461a2c1a366e11c0a949a896f..3d1e3a48ada6b59bab6714c12e21a937fd665a61 100644
--- a/ldvspec/lofardata/models.py
+++ b/ldvspec/lofardata/models.py
@@ -1,7 +1,7 @@
import logging
from celery.result import AsyncResult
from django.contrib.auth.models import User
-from django.contrib.postgres.fields import ArrayField, JSONField
+from django.contrib.postgres.fields import ArrayField
from django.db import models, IntegrityError
from django.utils.translation import gettext_lazy as _
logger = logging.getLogger(__name__)
@@ -17,7 +17,9 @@ class DataProduct(models.Model):
activity = models.CharField(max_length=50)
surl = models.CharField(max_length=200, unique=True)
filesize = models.PositiveBigIntegerField()
- additional_meta = JSONField()
+ antenna_set = models.CharField(max_length=50, blank=True, null=True, default="")
+ instrument_filter = models.CharField(max_length=50, blank=True, null=True, default="")
+ dysco_compression = models.BooleanField(default=False)
@staticmethod
def insert_dataproduct(
@@ -30,7 +32,9 @@ class DataProduct(models.Model):
activity,
surl,
filesize,
- additional_meta,
+ antenna_set,
+ instrument_filter,
+ dysco_compression
):
dp = DataProduct(
@@ -42,7 +46,9 @@ class DataProduct(models.Model):
location=location,
activity=activity,
filesize=filesize,
- additional_meta=additional_meta,
+ antenna_set=antenna_set,
+ instrument_filter=instrument_filter,
+ dysco_compression=dysco_compression,
surl=surl,
)
dp.save()
diff --git a/ldvspec/lofardata/templates/lofardata/base.html b/ldvspec/lofardata/templates/lofardata/base.html
index 85dbf1ad40b13b1b6eb32c2dc96d6d74a7037989..5c989613a818c7ceeb26e3c2230846c441ce646e 100644
--- a/ldvspec/lofardata/templates/lofardata/base.html
+++ b/ldvspec/lofardata/templates/lofardata/base.html
@@ -13,9 +13,6 @@
<!-- loads the path to static files -->
<link href='https://fonts.googleapis.com/css?family=Raleway' rel='stylesheet' type='text/css'>
- <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.4.2/css/all.css"
- integrity="sha384-/rXc/GQVaYpyDdyxK+ecHPVYJSN9bmVFBvjA/9eOB+pb3F2w2N6fc5qB9Ew5yIns" crossorigin="anonymous">
-
<link rel="stylesheet" type="text/css" href="{% static 'lofardata/styling/dias.css' %}">
{% block extra_js %}{% endblock %}
diff --git a/ldvspec/lofardata/tests/test_data_filter.py b/ldvspec/lofardata/tests/test_data_filter.py
index 62800b3967a02d2839337cb48376e21390c69ffa..d3e3523d48634ef667792040d2905ab34f7774e2 100644
--- a/ldvspec/lofardata/tests/test_data_filter.py
+++ b/ldvspec/lofardata/tests/test_data_filter.py
@@ -9,7 +9,9 @@ test_object_values = [dict(obs_id='12345', oid_source='SAS', dataproduct_source=
activity='observation',
surl='srm://surfsara.nl:4884/...',
filesize=40,
- additional_meta={'dysco_compression': True}),
+ antenna_set="HBA Dual Inner",
+ instrument_filter="110-190 MHz",
+ dysco_compression=True),
dict(obs_id='12346', oid_source='SAS', dataproduct_source='lofar',
dataproduct_type='observation',
project='LT10_10',
@@ -17,7 +19,9 @@ test_object_values = [dict(obs_id='12345', oid_source='SAS', dataproduct_source=
activity='pipeline',
surl='srm://surfsara.nl:4884/---',
filesize=40,
- additional_meta={'dysco_compression': True}),
+ antenna_set="HBA Dual Inner",
+ instrument_filter="110-190 MHz",
+ dysco_compression=True),
dict(obs_id='12347', oid_source='SAS', dataproduct_source='lofar',
dataproduct_type='observation',
project='LT10_10',
@@ -25,7 +29,9 @@ test_object_values = [dict(obs_id='12345', oid_source='SAS', dataproduct_source=
activity='observation',
surl='srm://surfsara.nl:4884/xxx',
filesize=40,
- additional_meta={'dysco_compression': True})]
+ antenna_set=None,
+ instrument_filter=None,
+ dysco_compression=True)]
class TestDataFilter(rtest.APITestCase):
@@ -55,4 +61,4 @@ class TestDataFilter(rtest.APITestCase):
self.assertEqual(2, response.data['count'])
self.assertEqual('observation', response.data['results'][0]['activity'])
- self.assertIn('activity_type', self.get_current_query_parameters())
\ No newline at end of file
+ self.assertIn('activity_type', self.get_current_query_parameters())
diff --git a/ldvspec/lofardata/tests/test_dataproduct.py b/ldvspec/lofardata/tests/test_dataproduct.py
index 8744168e5fd6a04a8f37148bebd735803cbeea37..55016df18bd3f9f1730f10d88ec9dd3cd28393df 100644
--- a/ldvspec/lofardata/tests/test_dataproduct.py
+++ b/ldvspec/lofardata/tests/test_dataproduct.py
@@ -13,7 +13,9 @@ test_object_value = dict(obs_id='12345', oid_source='SAS', dataproduct_source='l
activity='observation',
surl='srm://lta-head.lofar.psnc.pl:4884/subfolder/some_nice.tar',
filesize=40,
- additional_meta={'dysco_compression': True})
+ antenna_set="HBA Dual Inner",
+ instrument_filter= "110-190 MHz",
+ dysco_compression=True)
class TestDatabaseInteraction(dtest.TestCase):
@@ -127,4 +129,4 @@ class TestRESTAPI(rtest.APITestCase):
response = self.client.post('/ldvspec/api/v1/insert_dataproduct/', data=test_payload, format='json')
- self.assertEqual(response_status.HTTP_400_BAD_REQUEST, response.status_code)
\ No newline at end of file
+ self.assertEqual(response_status.HTTP_400_BAD_REQUEST, response.status_code)
diff --git a/ldvspec/lofardata/tests/test_workrequest.py b/ldvspec/lofardata/tests/test_workrequest.py
index 82df87982791b1144365226e96edf689e5ef0e88..89a614415e78484e52cdad2da1cf5db8cfccb618 100644
--- a/ldvspec/lofardata/tests/test_workrequest.py
+++ b/ldvspec/lofardata/tests/test_workrequest.py
@@ -25,7 +25,7 @@ class TestWorkSpecificationRequest(rtest.APITestCase):
# Insert test data
DataProduct.insert_dataproduct(12345, 'lta',
'myniceinstrument', 'ms', 'blubby', 'location-uri', 'pipline',
- 'srm://lofarlta/mynice_file.tar.gz', 123456, {'dysco_compression': False})
+ 'srm://lofarlta/mynice_file.tar.gz', 123456, None, None, False)
response = self.client.post('/ldvspec/api/v1/workspecification/',
data={'filters': {'obs_id': 12345}}, format='json')
@@ -49,14 +49,14 @@ class TestWorkSpecificationRequest(rtest.APITestCase):
# Insert test data
DataProduct.insert_dataproduct(12345, 'lta',
'myniceinstrument', 'ms', 'blubby', 'location-uri', 'pipline',
- 'srm://lofarlta/mynice_file.tar.gz', 123456, {'dysco_compression': False})
+ 'srm://lofarlta/mynice_file.tar.gz', 123456, None, None, False)
DataProduct.insert_dataproduct(12345, 'lta',
'myniceinstrument', 'ms', 'blubby', 'location-uri', 'pipline',
- 'srm://lofarlta/mynice_file_with_rhythm.tar.gz', 123456,
- {'dysco_compression': True})
+ 'srm://lofarlta/mynice_file_with_rhythm.tar.gz', 123456, "HBA Dual Inner",
+ "110-190 MHz", True)
response = self.client.post('/ldvspec/api/v1/workspecification/',
- data={'filters': {'obs_id': 12345, 'additional_meta__dysco_compression': True}},
+ data={'filters': {'obs_id': 12345, 'dysco_compression': True}},
format='json')
self.assertEqual(201, response.status_code)
diff --git a/ldvspec/requirements/base.txt b/ldvspec/requirements/base.txt
index ec4a4f46a3a4c9565e622a48a08e4dee6010ab4e..648a8d79890488981677cdbc83fb22e5d1d1addc 100644
--- a/ldvspec/requirements/base.txt
+++ b/ldvspec/requirements/base.txt
@@ -1,20 +1,12 @@
-Django==3.2
-djangorestframework==3.12.2
-django-filter==2.3.0
-psycopg2-binary==2.9.3
-django-cors-headers==3.6.0
-django-extensions==3.1.0
-django-bootstrap-pagination==1.7.0
-pytz==2022.1
-whitenoise==5.0.1
-six==1.15.0
-fontawesome-free==5.15.2
-pyyaml==6.0
-uritemplate==4.1.1
-sshtunnel==0.4.0
-django-uws==0.2.dev355575
-django-crispy-forms==1.14.0
-crispy-bootstrap5==0.6
-humanize==4.4.0
+Django
+django-filter==2.4.0 #newest version 1.22 not compatible
+djangorestframework
+django-cors-headers
+django-uws
+django-crispy-forms
django-widget-tweaks
-numpy==1.23.0
\ No newline at end of file
+uritemplate
+psycopg2-binary
+whitenoise
+pyyaml
+numpy
\ No newline at end of file