diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8ad874e2280585a04f9213b366fe4c7063047c1d..392bbabca29e5ddf0fd43cd09714beef9499312c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,11 +1,49 @@
stages:
+ - test
- build
- deploy_to_test
- deploy_to_production
+workflow:
+ rules:
+ - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
+ variables:
+ DOCKER_BUILD_IMAGE_TAG: ":stable"
+ - if: $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH
+ variables:
+ DOCKER_BUILD_IMAGE_TAG: ":latest"
+
+
+test-code:
+ image: python:3.10
+ stage: test
+ services:
+ - postgres:11.0
+ variables:
+ POSTGRES_DB: ldv-spec-db
+ POSTGRES_USER: postgres
+ POSTGRES_PASSWORD: "atdb123"
+ script:
+ - cd ldvspec
+ - pip install -r requirements/dev.txt
+ - python manage.py migrate --settings ldvspec.settings.ci
+ - python manage.py test --settings ldvspec.settings.ci
+
+docker-test-build:
+ variables:
+ if: main
+ # Official docker image.
+ image: docker$DOCKER_BUILD_IMAGE_TAG
+ stage: build
+ services:
+ - docker:dind
+ script:
+ - docker build --pull -t "$CI_REGISTRY_IMAGE" ldvspec
+
+
docker-build-master:
# Official docker image.
- image: docker:latest
+ image: docker$DOCKER_BUILD_IMAGE_TAG
stage: build
services:
- docker:dind
@@ -20,7 +58,6 @@ docker-build-master:
# deploy test/dev version on 'sdc-dev.astron.nl'
docker-deploy-main-test:
-# image: docker:latest
stage: deploy_to_test
before_script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
@@ -60,7 +97,7 @@ docker-deploy-main-test:
- main
docker-deploy-main-production:
-# image: docker:latest
+ # image: docker:latest
stage: deploy_to_production
before_script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
diff --git a/ldvspec/Dockerfile b/ldvspec/Dockerfile
index f17b53251ad682fa133f4fd84267f5425dfa2a84..c6db5f535ee6b668fcc701da98eebc8a7d8df8c0 100644
--- a/ldvspec/Dockerfile
+++ b/ldvspec/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.6.7-alpine
+FROM python:3.10-alpine
ENV PYTHONUNBUFFERED 1
RUN apk update && apk add bash && apk add nano && apk add mc
RUN mkdir /src
@@ -7,7 +7,7 @@ COPY . /src/
# install dependencies
RUN \
- apk add --no-cache python3 postgresql-libs && \
+ apk add --no-cache postgresql-libs && \
apk add --no-cache --virtual .build-deps gcc python3-dev musl-dev postgresql-dev && \
pip install -r requirements/prod.txt && \
apk --purge del .build-deps
diff --git a/ldvspec/ldvspec/settings/base.py b/ldvspec/ldvspec/settings/base.py
index 6734ff05c266eee3d4adfbcc3d9ffb8068683abd..24b8ce6551cf34df22f3ac1bed4f7c0485cbf939 100644
--- a/ldvspec/ldvspec/settings/base.py
+++ b/ldvspec/ldvspec/settings/base.py
@@ -131,7 +131,4 @@ REST_FRAMEWORK = {
'PAGE_SIZE': 100
}
-try:
- ATDB_HOST = os.environ['ATDB_HOST']
-except:
- ATDB_HOST = "http://localhost:8000/atdb/"
\ No newline at end of file
+ATDB_HOST = os.environ.get('ATDB_HOST', 'http://localhost:8000/atdb/')
diff --git a/ldvspec/ldvspec/settings/ci.py b/ldvspec/ldvspec/settings/ci.py
new file mode 100644
index 0000000000000000000000000000000000000000..ead5103b7942225b19cefb40ee9300dbf27074da
--- /dev/null
+++ b/ldvspec/ldvspec/settings/ci.py
@@ -0,0 +1,24 @@
+from ldvspec.settings.base import *
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEV = True
+DEBUG = True
+
+ALLOWED_HOSTS = ["*"]
+CORS_ORIGIN_ALLOW_ALL = True
+
+DATABASES = {
+ 'default': {
+ 'ENGINE': 'django.db.backends.postgresql_psycopg2',
+ 'USER': 'postgres',
+ 'PASSWORD': 'atdb123',
+ 'NAME': 'ldv-spec-db',
+ 'HOST': 'postgres',
+ 'PORT': '5432',
+ },
+}
+
+# Password validation
+# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = []
diff --git a/ldvspec/lofardata/admin.py b/ldvspec/lofardata/admin.py
index cf044d64e6c5d0935980c65f941a89eff47f36d7..e42f80883d51fbabe15e412aaaa66445078aa39a 100644
--- a/ldvspec/lofardata/admin.py
+++ b/ldvspec/lofardata/admin.py
@@ -1,5 +1,7 @@
from django.contrib import admin
# Register your models here.
-from .models import LofarData
-admin.site.register(LofarData)
\ No newline at end of file
+from .models import DataProduct, DataProductFilter
+
+admin.site.register(DataProduct)
+admin.site.register(DataProductFilter)
diff --git a/ldvspec/lofardata/migrations/0002_defines_dataproduct.py b/ldvspec/lofardata/migrations/0002_defines_dataproduct.py
new file mode 100644
index 0000000000000000000000000000000000000000..a2b6832a26c3edaa91cea85a8761d906d78a9f6f
--- /dev/null
+++ b/ldvspec/lofardata/migrations/0002_defines_dataproduct.py
@@ -0,0 +1,49 @@
+# Generated by Django 3.1.4 on 2022-07-21 09:23
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('lofardata', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='DataLocation',
+ fields=[
+ ('name', models.CharField(max_length=50, primary_key=True, serialize=False)),
+ ('uri', models.CharField(max_length=200)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='DataProduct',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('obs_id', models.CharField(max_length=15, verbose_name='OBS_ID')),
+ ('oid_source', models.CharField(max_length=15, verbose_name='OBS_ID_SOURCE')),
+ ('dataproduct_source', models.CharField(max_length=20)),
+ ('dataproduct_type', models.CharField(max_length=50)),
+ ('project', models.CharField(max_length=15)),
+ ('activity', models.CharField(max_length=50)),
+ ('surl', models.CharField(max_length=200)),
+ ('filesize', models.PositiveBigIntegerField()),
+ ('additional_meta', models.JSONField()),
+ ('location', models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, to='lofardata.datalocation')),
+ ],
+ ),
+ migrations.CreateModel(
+ name='DataProductFilter',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('field', models.CharField(max_length=20)),
+ ('name', models.CharField(max_length=20)),
+ ('lookup_type', models.CharField(max_length=100)),
+ ],
+ ),
+ migrations.DeleteModel(
+ name='LofarData',
+ ),
+ ]
diff --git a/ldvspec/lofardata/models.py b/ldvspec/lofardata/models.py
index cece29fd462734113c3584f41e2366894fca4edd..882475e59bc4c1f53fe7916b2b76c7630bbb4f75 100644
--- a/ldvspec/lofardata/models.py
+++ b/ldvspec/lofardata/models.py
@@ -1,7 +1,68 @@
from django.db import models
+from django_filters import rest_framework as filters
+from urllib.parse import urlsplit
-class LofarData(models.Model):
- sas_id = models.CharField(verbose_name="SAS_ID",max_length=15, blank=True, null=True)
- def __str__(self):
- return self.sas_id
\ No newline at end of file
+class DataLocation(models.Model):
+ name = models.CharField(max_length=50, primary_key=True)
+ uri = models.CharField(max_length=200)
+
+ @staticmethod
+ def insert_location_from_string(location_string):
+ """
+ Insert a datalocation from a srm string (e.g. srm://surm:4321/path.tar)
+
+ :param str location_string: SRM url
+ :return: DataLocation object
+ rtype: DataLocations
+ """
+ _, netloc, *_ = urlsplit(location_string)
+
+ dataloc = DataLocation(name=netloc, uri=location_string.rstrip('/'))
+ dataloc.save()
+ return dataloc
+
+
+class DataProduct(models.Model):
+ obs_id = models.CharField(verbose_name="OBS_ID", max_length=15)
+ oid_source = models.CharField(verbose_name='OBS_ID_SOURCE', max_length=15)
+ dataproduct_source = models.CharField(max_length=20)
+ dataproduct_type = models.CharField(max_length=50)
+ project = models.CharField(max_length=15)
+ location = models.ForeignKey(DataLocation, on_delete=models.DO_NOTHING)
+ activity = models.CharField(max_length=50)
+ surl = models.CharField(max_length=200)
+ filesize = models.PositiveBigIntegerField()
+ additional_meta = models.JSONField()
+
+ @staticmethod
+ def insert_dataproduct(obs_id,
+ oid_source,
+ dataproduct_source,
+ dataproduct_type,
+ project,
+ activity,
+ surl,
+ filesize,
+ additional_meta):
+ scheme, netloc, *_ = urlsplit(surl)
+
+ dp = DataProduct(obs_id=obs_id,
+ oid_source=oid_source,
+ dataproduct_source=dataproduct_source,
+ dataproduct_type=dataproduct_type,
+ project=project,
+ location=DataLocation.insert_location_from_string('://'.join((scheme, netloc))),
+ activity=activity,
+ filesize=filesize,
+ additional_meta=additional_meta,
+ surl=surl
+ )
+ dp.save()
+ return dp
+
+
+class DataProductFilter(models.Model):
+ field = models.CharField(max_length=20)
+ name = models.CharField(max_length=20)
+ lookup_type = models.CharField(max_length=100)
diff --git a/ldvspec/lofardata/serializers.py b/ldvspec/lofardata/serializers.py
index aaf9229c414b1b20f901be87a5d9fa53618cd7a9..f685fb619d7b66f46ef00023be19a953accd413a 100644
--- a/ldvspec/lofardata/serializers.py
+++ b/ldvspec/lofardata/serializers.py
@@ -1,8 +1,25 @@
+from abc import ABC
+
from rest_framework import serializers
-from .models import LofarData
+from .models import DataProduct, DataLocation
+
+
+class DataLocationSerializer(serializers.ModelSerializer):
+ class Meta:
+ model = DataLocation
+ fields = '__all__'
-class LofarDataSerializer(serializers.ModelSerializer):
+class DataProductSerializer(serializers.ModelSerializer):
class Meta:
- model = LofarData
+ model = DataProduct
fields = "__all__"
+
+
+class DataProductFlatSerializer(serializers.ModelSerializer):
+ class Meta:
+ model = DataProduct
+ exclude = ('location',)
+
+ def create(self, validated_data):
+ return DataProduct.insert_dataproduct(**validated_data)
diff --git a/ldvspec/lofardata/templates/lofardata/base.html b/ldvspec/lofardata/templates/lofardata/base.html
index 020485dbd2f2b26f6a02a1d29810b4ed0e731bb7..e5ab1386442bf38a9a3f6bfd4fece547a9658fc5 100644
--- a/ldvspec/lofardata/templates/lofardata/base.html
+++ b/ldvspec/lofardata/templates/lofardata/base.html
@@ -34,7 +34,7 @@
</li>
<li><a class="nav-link" href="{% url 'index' %}">Home</a></li>
- <li><a class="nav-link" href="{% url 'lofardata' %}">LOFAR Data</a></li>
+ <li><a class="nav-link" href="{% url 'dataproduct' %}">LOFAR Data</a></li>
{% if user.is_authenticated %}
<a class="nav-link" href="{% url 'logout' %}" target="_blank">Logout {{ user.get_username }}</a>
diff --git a/ldvspec/lofardata/templates/lofardata/index.html b/ldvspec/lofardata/templates/lofardata/index.html
index 76803327bb3eef32421637c330b7029772fa9107..f41e698a5f62e759ff9087f7512d9892bb64452f 100644
--- a/ldvspec/lofardata/templates/lofardata/index.html
+++ b/ldvspec/lofardata/templates/lofardata/index.html
@@ -9,7 +9,8 @@
<tbody>
<tr><td>atdb_host</td><td>{{ atdb_host }}</td></tr>
- <tr><td>api</td><td><a href="{% url 'lofardata' %}">{% url 'lofardata' %}</a></td></tr>
+ <tr><td>api</td><td><a href="{% url 'dataproduct' %}">{% url 'dataproduct' %}</a></td></tr>
+ <tr><td>api-schema</td><td><a href="{% url 'openapi-schema' %}">{% url 'openapi-schema' %}</a></td></tr>
</tbody>
</table>
diff --git a/ldvspec/lofardata/tests.py b/ldvspec/lofardata/tests.py
deleted file mode 100644
index 7ce503c2dd97ba78597f6ff6e4393132753573f6..0000000000000000000000000000000000000000
--- a/ldvspec/lofardata/tests.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.
diff --git a/ldvspec/__init__.py b/ldvspec/lofardata/tests/__init__.py
similarity index 100%
rename from ldvspec/__init__.py
rename to ldvspec/lofardata/tests/__init__.py
diff --git a/ldvspec/lofardata/tests/test_data_filter.py b/ldvspec/lofardata/tests/test_data_filter.py
new file mode 100644
index 0000000000000000000000000000000000000000..d608546d11ac888d98fbfc0ba95a0710c79bd094
--- /dev/null
+++ b/ldvspec/lofardata/tests/test_data_filter.py
@@ -0,0 +1,55 @@
+import rest_framework.test as rtest
+from django.contrib.auth.models import User
+from lofardata.models import DataProduct, DataProductFilter
+
+test_object_values = [dict(obs_id='12345', oid_source='SAS', dataproduct_source='lofar',
+ dataproduct_type='observation',
+ project='LT10_10',
+ activity='observation',
+ surl='srm://surfsara.nl:4884/...',
+ filesize=40,
+ additional_meta={'dysco_compression': True}),
+ dict(obs_id='12346', oid_source='SAS', dataproduct_source='lofar',
+ dataproduct_type='observation',
+ project='LT10_10',
+ activity='pipeline',
+ surl='srm://surfsara.nl:4884/...',
+ filesize=40,
+ additional_meta={'dysco_compression': True}),
+ dict(obs_id='12347', oid_source='SAS', dataproduct_source='lofar',
+ dataproduct_type='observation',
+ project='LT10_10',
+ activity='observation',
+ surl='srm://surfsara.nl:4884/...',
+ filesize=40,
+ additional_meta={'dysco_compression': True})]
+
+
+class TestDataFilter(rtest.APITestCase):
+ def setUp(self):
+ self.user = User.objects.create_superuser('admin')
+ self.client.force_authenticate(self.user)
+
+ for test_object in test_object_values:
+ DataProduct.insert_dataproduct(**test_object)
+
+ def get_current_query_parameters(self):
+ response = self.client.get('/ldvspec/api/v1/openapi/')
+ query_parameters = response.data['paths']['/ldvspec/api/v1/data/']['get']['parameters']
+ return {parameter['name']: parameter for parameter in query_parameters}
+
+ def test_add_custom_filter(self):
+ self.assertNotIn('activity', self.get_current_query_parameters(), 'Test is invalid! Update')
+
+ dp_filter = DataProductFilter(field='activity', name='activity_type', lookup_type='exact')
+ dp_filter.save()
+
+ response = self.client.get('/ldvspec/api/v1/data/?activity_type=pipeline')
+ self.assertEqual(1, response.data['count'])
+ self.assertEqual('pipeline', response.data['results'][0]['activity'])
+
+ response = self.client.get('/ldvspec/api/v1/data/?activity_type=observation')
+ self.assertEqual(2, response.data['count'])
+ self.assertEqual('observation', response.data['results'][0]['activity'])
+
+ self.assertIn('activity_type', self.get_current_query_parameters())
\ No newline at end of file
diff --git a/ldvspec/lofardata/tests/test_dataproduct.py b/ldvspec/lofardata/tests/test_dataproduct.py
new file mode 100644
index 0000000000000000000000000000000000000000..138634b3d5811b60ce73dda190f54d1cc32a2a21
--- /dev/null
+++ b/ldvspec/lofardata/tests/test_dataproduct.py
@@ -0,0 +1,79 @@
+import django.test as dtest
+import rest_framework.test as rtest
+from django.contrib.auth.models import User
+import rest_framework.status as response_status
+from lofardata.models import DataProduct, DataLocation
+
+test_object_value = dict(obs_id='12345', oid_source='SAS', dataproduct_source='lofar',
+ dataproduct_type='observation',
+ project='LT10_10',
+ location='srm://surfsara.nl:4884/',
+ activity='observation',
+ surl='srm://surfsara.nl:4884/...',
+ filesize=40,
+ additional_meta={'dysco_compression': True})
+
+
+class TestDatabaseInteraction(dtest.TestCase):
+ def test_insert(self):
+ location = DataLocation(name='surfsara', uri='srm://surfsara.nl')
+ location.save()
+ test_values = dict(test_object_value)
+ test_values['location'] = location
+ dp = DataProduct(**test_values)
+ dp.save()
+ self.assertTrue(dp.pk is not None, 'Failed saving object')
+ dp = DataProduct.objects.get(obs_id='12345')
+ for field in test_values:
+ self.assertEqual(getattr(dp, field), test_values.get(field))
+
+ def test_insert_with_helper_function(self):
+ test_values = dict(test_object_value)
+ test_values.pop('location')
+
+ dp = DataProduct.insert_dataproduct(**test_values)
+
+ for field in test_values:
+ self.assertEqual(test_object_value.get(field), getattr(dp, field), msg=f'Field {field} does not coincide')
+
+ self.assertEqual('surfsara.nl:4884', dp.location.name)
+ self.assertEqual( 'srm://surfsara.nl:4884', dp.location.uri)
+
+
+class TestRESTAPI(rtest.APITestCase):
+ def setUp(self):
+ self.user = User.objects.create_superuser('admin')
+ self.client.force_authenticate(self.user)
+
+ def test_insert_not_allowed(self):
+ client = rtest.APIClient()
+ response = client.post('/ldvspec/api/v1/data/insert/', data={}, format='json')
+ self.assertEqual(response_status.HTTP_403_FORBIDDEN, response.status_code)
+
+ def test_insert_flat_error(self):
+ response = self.client.post('/ldvspec/api/v1/data/insert/', data={}, format='json')
+ self.assertEqual(response_status.HTTP_400_BAD_REQUEST, response.status_code)
+
+ def test_insert_flat_single(self):
+ test_payload = dict(test_object_value)
+ test_payload.pop('location')
+ response = self.client.post('/ldvspec/api/v1/data/insert/', data=test_payload, format='json')
+ self.assertEqual(response_status.HTTP_201_CREATED, response.status_code)
+
+ def test_insert_flat_multi(self):
+ test_payload = dict(test_object_value)
+ test_payload.pop('location')
+ response = self.client.post('/ldvspec/api/v1/data/insert/', data=[test_payload, test_payload], format='json')
+
+ self.assertEqual(response_status.HTTP_201_CREATED, response.status_code)
+ self.assertTrue(DataProduct.objects.count() == 2, 'Not all dataproduct have been inserted')
+ self.assertTrue(DataLocation.objects.count() == 1, 'Not all dataproduct have been inserted')
+
+ def test_insert_flat_multi_insert_single(self):
+ test_payload = dict(test_object_value)
+ test_payload.pop('location')
+ response = self.client.post('/ldvspec/api/v1/data/insert/', data=test_payload, format='json')
+
+ self.assertEqual(response_status.HTTP_201_CREATED, response.status_code)
+ self.assertTrue(DataProduct.objects.count() == 1, 'Not all dataproduct have been inserted')
+ self.assertTrue(DataLocation.objects.count() == 1, 'Not all dataproduct have been inserted')
\ No newline at end of file
diff --git a/ldvspec/lofardata/tests/test_location.py b/ldvspec/lofardata/tests/test_location.py
new file mode 100644
index 0000000000000000000000000000000000000000..86d6466c3b45485d547ffcf21c843bdae7bfdd21
--- /dev/null
+++ b/ldvspec/lofardata/tests/test_location.py
@@ -0,0 +1,55 @@
+import django.test as dtest
+import rest_framework.test as rtest
+from django.contrib.auth.models import User
+import rest_framework.status as response_status
+from lofardata.models import DataLocation
+
+
+class TestDataLocation(dtest.TestCase):
+
+ def test_insert(self):
+ location = DataLocation(name='test', uri='srm://path_to_file/')
+ location.save()
+
+ self.assertTrue(location.pk is not None, 'Cannot save object')
+
+ def test_double_insert_unique_entry(self):
+ location = DataLocation(name='test', uri='srm://path_to_file/')
+ location.save()
+
+ self.assertTrue(location.pk is not None, 'Cannot save object')
+ location_first_pk = location.pk
+
+ location = DataLocation(name='test', uri='srm://path_to_file/')
+ location.save()
+
+ self.assertTrue(location.pk is not None, 'Cannot save object')
+ location_second_pk = location.pk
+
+ self.assertTrue(location_first_pk == location_second_pk, 'Double inserted index do not coincide')
+
+ self.assertTrue(DataLocation.objects.count() == 1)
+
+ def test_insert_from_string(self):
+ dataloc = DataLocation.insert_location_from_string('srm://test_site:44321/')
+
+ self.assertTrue(dataloc.pk is not None, 'Cannot save object')
+ self.assertEqual('test_site:44321', dataloc.name)
+ self.assertEqual('srm://test_site:44321', dataloc.uri)
+
+
+class TestRESTAPI(rtest.APITestCase):
+ def setUp(self):
+ self.user = User.objects.create_superuser('admin')
+ self.client.force_authenticate(self.user)
+
+ def test_create_error(self):
+ response = self.client.post('/ldvspec/api/v1/data-location/', data=dict(name='testname'),
+ format='json')
+ self.assertEqual(response_status.HTTP_400_BAD_REQUEST, response.status_code)
+
+ def test_create(self):
+ response = self.client.post('/ldvspec/api/v1/data-location/', data=dict(name='testname', uri='srm://myniceuri/'),
+ format='json')
+ self.assertEqual(response_status.HTTP_201_CREATED, response.status_code)
+ self.assertEqual('srm://myniceuri/', DataLocation.objects.get(name='testname').uri)
\ No newline at end of file
diff --git a/ldvspec/lofardata/urls.py b/ldvspec/lofardata/urls.py
index 236c3de9ebcabd2e1765fd0144414e116a281fea..877dee282177eba1850856f5a56b5de9f9e29471 100644
--- a/ldvspec/lofardata/urls.py
+++ b/ldvspec/lofardata/urls.py
@@ -1,6 +1,6 @@
from django.urls import include, path
from django.contrib.auth import views as auth_views
-from rest_framework.authtoken import views as rest_auth_views
+from rest_framework.schemas import get_schema_view
from . import views
@@ -13,8 +13,15 @@ urlpatterns = [
path('login/', auth_views.LoginView.as_view(template_name='registration/login.html')),
# REST API
- path('lofardata/', views.LofarDataView.as_view(), name='lofardata'),
+ path('api/v1/data/', views.DataProductView.as_view(), name='dataproduct'),
+ path('api/v1/data/insert/', views.InsertMultiDataproductView.as_view(), name='dataproduct-insert'),
+ path('api/v1/data-location/', views.DataLocationView.as_view(), name='datalocation'),
+ path('api/v1/openapi/', get_schema_view(
+ title="LDV Specification",
+ description="API description",
+ version="0.0.1"
+ ), name='openapi-schema'),
# GUI
#path('', views.IndexView.as_view(), name='index'),
path('', views.index, name='index'),
diff --git a/ldvspec/lofardata/views.py b/ldvspec/lofardata/views.py
index d058d3cf4311c2ca01159c3ce6ed54dbdd5536c4..1e2b448394c04715c247e651d8d26f2517a9d0fa 100644
--- a/ldvspec/lofardata/views.py
+++ b/ldvspec/lofardata/views.py
@@ -1,26 +1,46 @@
+import logging
-from django.shortcuts import render, redirect, reverse
-from django.views.generic import ListView
+from django.shortcuts import render
from django.conf import settings
from rest_framework import generics, pagination
from rest_framework.views import APIView
-import django_filters
+from rest_framework import status
+from rest_framework.response import Response
from django_filters import rest_framework as filters
-from .models import LofarData
-from .serializers import LofarDataSerializer
+from .models import DataProduct, DataProductFilter, DataLocation
+from .serializers import DataProductSerializer, DataProductFlatSerializer, DataLocationSerializer
-# --- Filters ---
-class LofarDataFilter(filters.FilterSet):
+
+class DynamicFilterSet(filters.FilterSet):
class Meta:
- model = LofarData
+ filter_class = None
+
+ def __init__(self, *args, **kwargs):
+ self._load_filters()
+ super().__init__(*args, **kwargs)
+
+ def _load_filters(self):
+ if self.Meta.filter_class is None:
+ raise Exception('Define filter_class meta attribute')
+ for item in self.Meta.filter_class.objects.all():
+ field_obj = self.Meta.model._meta.get_field(item.field)
+ filter_class, *_ = self.filter_for_lookup(field_obj, item.lookup_type)
+ self.base_filters[item.name] = filter_class(item.field)
+
+# --- Filters ---
+class DataProductFilterSet(DynamicFilterSet):
+ class Meta:
+ model = DataProduct
+ filter_class = DataProductFilter
fields = {
- 'sas_id': ['exact', 'icontains'],
+ 'obs_id': ['exact', 'icontains'],
}
+
# ---------- GUI Views -----------
def index(request):
@@ -29,13 +49,32 @@ def index(request):
# ---------- REST API views ----------
+class DataProductView(generics.ListCreateAPIView):
+ model = DataProduct
+ serializer_class = DataProductSerializer
-class LofarDataView(generics.ListCreateAPIView):
- model = LofarData
- serializer_class = LofarDataSerializer
-
- queryset = LofarData.objects.all().order_by('sas_id')
+ queryset = DataProduct.objects.all().order_by('obs_id')
# using the Django Filter Backend - https://django-filter.readthedocs.io/en/latest/index.html
filter_backends = (filters.DjangoFilterBackend,)
- filter_class = LofarDataFilter
+ filter_class = DataProductFilterSet
+
+
+class DataLocationView(generics.ListCreateAPIView):
+ model = DataLocation
+ serializer_class = DataLocationSerializer
+ queryset = DataLocation.objects.all().order_by('name')
+
+
+class InsertMultiDataproductView(generics.CreateAPIView):
+ """
+ Add single DataProduct
+ """
+ queryset = DataProduct.objects.all()
+ serializer_class = DataProductFlatSerializer
+
+ def get_serializer(self, *args, **kwargs):
+ """ if an array is passed, set serializer to many """
+ if isinstance(kwargs.get('data', {}), list):
+ kwargs['many'] = True
+ return super().get_serializer(*args, **kwargs)
diff --git a/ldvspec/manage.py b/ldvspec/manage.py
index 110f50eed9bb0655751f09aeaca46b10b0507598..b99a0a4bbaf9cacd4d3a770bd1dd73a52b69b170 100644
--- a/ldvspec/manage.py
+++ b/ldvspec/manage.py
@@ -6,7 +6,7 @@ import sys
def main():
"""Run administrative tasks."""
- os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ldvspec.settings')
+ os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ldvspec.settings.dev')
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
diff --git a/ldvspec/requirements/base.txt b/ldvspec/requirements/base.txt
index 4bfeb5bfa8877c6936b47be6f5ccbda66ccd9797..45954f46e55c36134e02bec5a7006d63434827a9 100644
--- a/ldvspec/requirements/base.txt
+++ b/ldvspec/requirements/base.txt
@@ -9,3 +9,5 @@ pytz==2022.1
whitenoise==5.0.1
six==1.15.0
fontawesome-free==5.15.2
+pyyaml==6.0
+uritemplate==4.1.1
\ No newline at end of file