diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8ad874e2280585a04f9213b366fe4c7063047c1d..4d12b0f61fe0151b0dc2cc12f2a9383d37a81012 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,8 +1,25 @@
stages:
+ - test
- build
- deploy_to_test
- deploy_to_production
+docker-test:
+ image: python:3.6.7-alpine
+ stage: test
+ services:
+ - postgres:11.0
+ variables:
+ POSTGRES_DB: ldv-spec-db
+ POSTGRES_USER: postgres
+ POSTGRES_PASSWORD: "atdb123"
+ script:
+ - cd ldvspec
+ - pip install -r requirements/dev.txt
+ - python manage.py makemigrations
+ - python manage.py migrate
+ - python manage.py test
+
docker-build-master:
# Official docker image.
image: docker:latest
diff --git a/ldvspec/ldvspec/settings/base.py b/ldvspec/ldvspec/settings/base.py
index 6734ff05c266eee3d4adfbcc3d9ffb8068683abd..24b8ce6551cf34df22f3ac1bed4f7c0485cbf939 100644
--- a/ldvspec/ldvspec/settings/base.py
+++ b/ldvspec/ldvspec/settings/base.py
@@ -131,7 +131,4 @@ REST_FRAMEWORK = {
'PAGE_SIZE': 100
}
-try:
- ATDB_HOST = os.environ['ATDB_HOST']
-except:
- ATDB_HOST = "http://localhost:8000/atdb/"
\ No newline at end of file
+ATDB_HOST = os.environ.get('ATDB_HOST', 'http://localhost:8000/atdb/')
diff --git a/ldvspec/ldvspec/settings/ci.py b/ldvspec/ldvspec/settings/ci.py
new file mode 100644
index 0000000000000000000000000000000000000000..ead5103b7942225b19cefb40ee9300dbf27074da
--- /dev/null
+++ b/ldvspec/ldvspec/settings/ci.py
@@ -0,0 +1,24 @@
+from ldvspec.settings.base import *
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEV = True
+DEBUG = True
+
+ALLOWED_HOSTS = ["*"]
+CORS_ORIGIN_ALLOW_ALL = True
+
+DATABASES = {
+ 'default': {
+ 'ENGINE': 'django.db.backends.postgresql_psycopg2',
+ 'USER': 'postgres',
+ 'PASSWORD': 'atdb123',
+ 'NAME': 'ldv-spec-db',
+ 'HOST': 'postgres',
+ 'PORT': '5432',
+ },
+}
+
+# Password validation
+# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = []
diff --git a/ldvspec/lofardata/models.py b/ldvspec/lofardata/models.py
index 0d233fc9c6f7c4d54490f66bcda851750b9c6459..882475e59bc4c1f53fe7916b2b76c7630bbb4f75 100644
--- a/ldvspec/lofardata/models.py
+++ b/ldvspec/lofardata/models.py
@@ -1,13 +1,27 @@
from django.db import models
from django_filters import rest_framework as filters
+from urllib.parse import urlsplit
-
-
-class DataLocations(models.Model):
+class DataLocation(models.Model):
name = models.CharField(max_length=50, primary_key=True)
uri = models.CharField(max_length=200)
+ @staticmethod
+ def insert_location_from_string(location_string):
+ """
+ Insert a datalocation from a srm string (e.g. srm://surm:4321/path.tar)
+
+ :param str location_string: SRM url
+ :return: DataLocation object
+ rtype: DataLocations
+ """
+ _, netloc, *_ = urlsplit(location_string)
+
+ dataloc = DataLocation(name=netloc, uri=location_string.rstrip('/'))
+ dataloc.save()
+ return dataloc
+
class DataProduct(models.Model):
obs_id = models.CharField(verbose_name="OBS_ID", max_length=15)
@@ -15,16 +29,40 @@ class DataProduct(models.Model):
dataproduct_source = models.CharField(max_length=20)
dataproduct_type = models.CharField(max_length=50)
project = models.CharField(max_length=15)
- location = models.ForeignKey(DataLocations, on_delete=models.DO_NOTHING)
+ location = models.ForeignKey(DataLocation, on_delete=models.DO_NOTHING)
activity = models.CharField(max_length=50)
surl = models.CharField(max_length=200)
filesize = models.PositiveBigIntegerField()
additional_meta = models.JSONField()
+ @staticmethod
+ def insert_dataproduct(obs_id,
+ oid_source,
+ dataproduct_source,
+ dataproduct_type,
+ project,
+ activity,
+ surl,
+ filesize,
+ additional_meta):
+ scheme, netloc, *_ = urlsplit(surl)
+
+ dp = DataProduct(obs_id=obs_id,
+ oid_source=oid_source,
+ dataproduct_source=dataproduct_source,
+ dataproduct_type=dataproduct_type,
+ project=project,
+ location=DataLocation.insert_location_from_string('://'.join((scheme, netloc))),
+ activity=activity,
+ filesize=filesize,
+ additional_meta=additional_meta,
+ surl=surl
+ )
+ dp.save()
+ return dp
-class DataProductFilter(models.Model):
+class DataProductFilter(models.Model):
field = models.CharField(max_length=20)
name = models.CharField(max_length=20)
lookup_type = models.CharField(max_length=100)
-
diff --git a/ldvspec/lofardata/serializers.py b/ldvspec/lofardata/serializers.py
index 189e327c18d9210cd26b23ba53bd65f3051458b1..f685fb619d7b66f46ef00023be19a953accd413a 100644
--- a/ldvspec/lofardata/serializers.py
+++ b/ldvspec/lofardata/serializers.py
@@ -1,8 +1,25 @@
+from abc import ABC
+
from rest_framework import serializers
-from .models import DataProduct
+from .models import DataProduct, DataLocation
+
+
+class DataLocationSerializer(serializers.ModelSerializer):
+ class Meta:
+ model = DataLocation
+ fields = '__all__'
-class DataProductSerializer(serializers.ModelSerializer):
+class DataProductSerializer(serializers.ModelSerializer):
class Meta:
model = DataProduct
fields = "__all__"
+
+
+class DataProductFlatSerializer(serializers.ModelSerializer):
+ class Meta:
+ model = DataProduct
+ exclude = ('location',)
+
+ def create(self, validated_data):
+ return DataProduct.insert_dataproduct(**validated_data)
diff --git a/ldvspec/lofardata/tests.py b/ldvspec/lofardata/tests.py
deleted file mode 100644
index 7ce503c2dd97ba78597f6ff6e4393132753573f6..0000000000000000000000000000000000000000
--- a/ldvspec/lofardata/tests.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.
diff --git a/ldvspec/__init__.py b/ldvspec/lofardata/tests/__init__.py
similarity index 100%
rename from ldvspec/__init__.py
rename to ldvspec/lofardata/tests/__init__.py
diff --git a/ldvspec/lofardata/tests/test_data_filter.py b/ldvspec/lofardata/tests/test_data_filter.py
new file mode 100644
index 0000000000000000000000000000000000000000..ba051c47621f23e89dd292a7ebcbaa9881045885
--- /dev/null
+++ b/ldvspec/lofardata/tests/test_data_filter.py
@@ -0,0 +1,56 @@
+import rest_framework.test as rtest
+from django.contrib.auth.models import User
+from lofardata.models import DataProduct, DataProductFilter
+
+test_object_values = [dict(obs_id='12345', oid_source='SAS', dataproduct_source='lofar',
+ dataproduct_type='observation',
+ project='LT10_10',
+ activity='observation',
+ surl='srm://surfsara.nl:4884/...',
+ filesize=40,
+ additional_meta={'dysco_compression': True}),
+ dict(obs_id='12346', oid_source='SAS', dataproduct_source='lofar',
+ dataproduct_type='observation',
+ project='LT10_10',
+ activity='pipeline',
+ surl='srm://surfsara.nl:4884/...',
+ filesize=40,
+ additional_meta={'dysco_compression': True}),
+ dict(obs_id='12347', oid_source='SAS', dataproduct_source='lofar',
+ dataproduct_type='observation',
+ project='LT10_10',
+ activity='observation',
+ surl='srm://surfsara.nl:4884/...',
+ filesize=40,
+ additional_meta={'dysco_compression': True})]
+
+
+class TestDataFilter(rtest.APITestCase):
+ def setUp(self):
+ self.client = rtest.APIClient()
+ self.user = User.objects.create_superuser('admin')
+ self.client.force_authenticate(self.user)
+
+ for test_object in test_object_values:
+ DataProduct.insert_dataproduct(**test_object)
+
+ def get_current_query_parameters(self):
+ response = self.client.get('/ldvspec/openapi/')
+ query_parameters = response.data['paths']['/ldvspec/data/']['get']['parameters']
+ return {parameter['name']: parameter for parameter in query_parameters}
+
+ def test_add_custom_filter(self):
+ self.assertNotIn('activity', self.get_current_query_parameters(), 'Test is invalid! Update')
+
+ dp_filter = DataProductFilter(field='activity', name='activity_type', lookup_type='exact')
+ dp_filter.save()
+
+ response = self.client.get('/ldvspec/data/?activity_type=pipeline')
+ self.assertEqual(1, response.data['count'])
+ self.assertEqual('pipeline', response.data['results'][0]['activity'])
+
+ response = self.client.get('/ldvspec/data/?activity_type=observation')
+ self.assertEqual(2, response.data['count'])
+ self.assertEqual('observation', response.data['results'][0]['activity'])
+
+ self.assertIn('activity_type', self.get_current_query_parameters())
\ No newline at end of file
diff --git a/ldvspec/lofardata/tests/test_dataproduct.py b/ldvspec/lofardata/tests/test_dataproduct.py
new file mode 100644
index 0000000000000000000000000000000000000000..7d63d9f5c4415f3ca275f0e83132c2671bb98b9c
--- /dev/null
+++ b/ldvspec/lofardata/tests/test_dataproduct.py
@@ -0,0 +1,80 @@
+import django.test as dtest
+import rest_framework.test as rtest
+from django.contrib.auth.models import User
+import rest_framework.status as response_status
+from lofardata.models import DataProduct, DataLocation
+
+test_object_value = dict(obs_id='12345', oid_source='SAS', dataproduct_source='lofar',
+ dataproduct_type='observation',
+ project='LT10_10',
+ location='srm://surfsara.nl:4884/',
+ activity='observation',
+ surl='srm://surfsara.nl:4884/...',
+ filesize=40,
+ additional_meta={'dysco_compression': True})
+
+
+class TestDatabaseInteraction(dtest.TestCase):
+ def test_insert(self):
+ location = DataLocation(name='surfsara', uri='srm://surfsara.nl')
+ location.save()
+ test_values = dict(test_object_value)
+ test_values['location'] = location
+ dp = DataProduct(**test_values)
+ dp.save()
+ self.assertTrue(dp.pk is not None, 'Failed saving object')
+ dp = DataProduct.objects.get(obs_id='12345')
+ for field in test_values:
+ self.assertEqual(getattr(dp, field), test_values.get(field))
+
+ def test_insert_with_helper_function(self):
+ test_values = dict(test_object_value)
+ test_values.pop('location')
+
+ dp = DataProduct.insert_dataproduct(**test_values)
+
+ for field in test_values:
+ self.assertEqual(test_object_value.get(field), getattr(dp, field), msg=f'Field {field} does not coincide')
+
+ self.assertEqual('surfsara.nl:4884', dp.location.name)
+ self.assertEqual( 'srm://surfsara.nl:4884', dp.location.uri)
+
+
+class TestRESTAPI(rtest.APITestCase):
+ def setUp(self):
+ self.client = rtest.APIClient()
+ self.user = User.objects.create_superuser('admin')
+ self.client.force_authenticate(self.user)
+
+ def test_insert_not_allowed(self):
+ client = rtest.APIClient()
+ response = client.post('/ldvspec/data/insert/', data={}, format='json')
+ self.assertEqual(response_status.HTTP_403_FORBIDDEN, response.status_code)
+
+ def test_insert_flat_error(self):
+ response = self.client.post('/ldvspec/data/insert/', data={}, format='json')
+ self.assertEqual(response_status.HTTP_400_BAD_REQUEST, response.status_code)
+
+ def test_insert_flat_single(self):
+ test_payload = dict(test_object_value)
+ test_payload.pop('location')
+ response = self.client.post('/ldvspec/data/insert/', data=test_payload, format='json')
+ self.assertEqual(response_status.HTTP_201_CREATED, response.status_code)
+
+ def test_insert_flat_multi(self):
+ test_payload = dict(test_object_value)
+ test_payload.pop('location')
+ response = self.client.post('/ldvspec/data/insert/', data=[test_payload, test_payload], format='json')
+
+ self.assertEqual(response_status.HTTP_201_CREATED, response.status_code)
+ self.assertTrue(DataProduct.objects.count() == 2, 'Not all dataproduct have been inserted')
+ self.assertTrue(DataLocation.objects.count() == 1, 'Not all dataproduct have been inserted')
+
+ def test_insert_flat_multi_insert_single(self):
+ test_payload = dict(test_object_value)
+ test_payload.pop('location')
+ response = self.client.post('/ldvspec/data/insert/', data=test_payload, format='json')
+
+ self.assertEqual(response_status.HTTP_201_CREATED, response.status_code)
+ self.assertTrue(DataProduct.objects.count() == 1, 'Not all dataproduct have been inserted')
+ self.assertTrue(DataLocation.objects.count() == 1, 'Not all dataproduct have been inserted')
\ No newline at end of file
diff --git a/ldvspec/lofardata/tests/test_location.py b/ldvspec/lofardata/tests/test_location.py
new file mode 100644
index 0000000000000000000000000000000000000000..93a10e82aede0e4461ab7adf480223153a134158
--- /dev/null
+++ b/ldvspec/lofardata/tests/test_location.py
@@ -0,0 +1,56 @@
+import django.test as dtest
+import rest_framework.test as rtest
+from django.contrib.auth.models import User
+import rest_framework.status as response_status
+from lofardata.models import DataLocation
+
+
+class TestDataLocation(dtest.TestCase):
+
+ def test_insert(self):
+ location = DataLocation(name='test', uri='srm://path_to_file/')
+ location.save()
+
+ self.assertTrue(location.pk is not None, 'Cannot save object')
+
+ def test_double_insert_unique_entry(self):
+ location = DataLocation(name='test', uri='srm://path_to_file/')
+ location.save()
+
+ self.assertTrue(location.pk is not None, 'Cannot save object')
+ location_first_pk = location.pk
+
+ location = DataLocation(name='test', uri='srm://path_to_file/')
+ location.save()
+
+ self.assertTrue(location.pk is not None, 'Cannot save object')
+ location_second_pk = location.pk
+
+ self.assertTrue(location_first_pk == location_second_pk, 'Double inserted index do not coincide')
+
+ self.assertTrue(DataLocation.objects.count() == 1)
+
+ def test_insert_from_string(self):
+ dataloc = DataLocation.insert_location_from_string('srm://test_site:44321/')
+
+ self.assertTrue(dataloc.pk is not None, 'Cannot save object')
+ self.assertEqual('test_site:44321', dataloc.name)
+ self.assertEqual('srm://test_site:44321', dataloc.uri)
+
+
+class TestRESTAPI(rtest.APITestCase):
+ def setUp(self):
+ self.client = rtest.APIClient()
+ self.user = User.objects.create_superuser('admin')
+ self.client.force_authenticate(self.user)
+
+ def test_create_error(self):
+ response = self.client.post('/ldvspec/data-location/', data=dict(name='testname'),
+ format='json')
+ self.assertEqual(response_status.HTTP_400_BAD_REQUEST, response.status_code)
+
+ def test_create(self):
+ response = self.client.post('/ldvspec/data-location/', data=dict(name='testname', uri='srm://myniceuri/'),
+ format='json')
+ self.assertEqual(response_status.HTTP_201_CREATED, response.status_code)
+ self.assertEqual('srm://myniceuri/', DataLocation.objects.get(name='testname').uri)
\ No newline at end of file
diff --git a/ldvspec/lofardata/urls.py b/ldvspec/lofardata/urls.py
index 99257b63f946d7419d123ebfe0e89c14fc4cec47..e9c770eaceb6d20ce701f2840350487203486786 100644
--- a/ldvspec/lofardata/urls.py
+++ b/ldvspec/lofardata/urls.py
@@ -15,6 +15,9 @@ urlpatterns = [
# REST API
path('data/', views.DataProductView.as_view(), name='dataproduct'),
+ path('data/insert/', views.InsertMultiDataproductView.as_view(), name='dataproduct-insert'),
+ path('data-location/', views.DataLocationView.as_view(), name='datalocation'),
+
path('openapi/', get_schema_view(
title="LDV Specification",
description="API description",
diff --git a/ldvspec/lofardata/views.py b/ldvspec/lofardata/views.py
index 92120167993ee234e0de3838aa144633d3e0e6c1..ef34e427c9357157e7495a4863da45eb0e27d2ef 100644
--- a/ldvspec/lofardata/views.py
+++ b/ldvspec/lofardata/views.py
@@ -1,12 +1,17 @@
+import logging
+
from django.shortcuts import render
from django.conf import settings
from rest_framework import generics, pagination
+from rest_framework.views import APIView
+from rest_framework import status
+from rest_framework.response import Response
from django_filters import rest_framework as filters
-from .models import DataProduct, DataProductFilter
-from .serializers import DataProductSerializer
+from .models import DataProduct, DataProductFilter, DataLocation
+from .serializers import DataProductSerializer, DataProductFlatSerializer, DataLocationSerializer
class DynamicFilterSet(filters.FilterSet):
@@ -18,16 +23,12 @@ class DynamicFilterSet(filters.FilterSet):
super().__init__(*args, **kwargs)
def _load_filters(self):
- print('Current filters are', self.base_filters)
if self.Meta.filter_class is None:
raise Exception('Define filter_class meta attribute')
-
- print('Filter to be added', self.Meta.filter_class.objects.all())
for item in self.Meta.filter_class.objects.all():
field_obj = self.Meta.model._meta.get_field(item.field)
filter_class, *_ = self.filter_for_lookup(field_obj, item.lookup_type)
self.base_filters[item.name] = filter_class(item.field)
- print('Final filters are', self.base_filters)
# --- Filters ---
@@ -48,7 +49,6 @@ def index(request):
# ---------- REST API views ----------
-
class DataProductView(generics.ListCreateAPIView):
model = DataProduct
serializer_class = DataProductSerializer
@@ -59,5 +59,22 @@ class DataProductView(generics.ListCreateAPIView):
filter_backends = (filters.DjangoFilterBackend,)
filter_class = DataProductFilterSet().__class__
- def get_queryset(self):
- return self.queryset.order_by('obs_id')
+
+class DataLocationView(generics.ListCreateAPIView):
+ model = DataLocation
+ serializer_class = DataLocationSerializer
+ queryset = DataLocation.objects.all().order_by('name')
+
+
+class InsertMultiDataproductView(generics.CreateAPIView):
+ """
+ Add single DataProduct
+ """
+ queryset = DataProduct.objects.all()
+ serializer_class = DataProductFlatSerializer
+
+ def get_serializer(self, *args, **kwargs):
+ """ if an array is passed, set serializer to many """
+ if isinstance(kwargs.get('data', {}), list):
+ kwargs['many'] = True
+ return super().get_serializer(*args, **kwargs)
diff --git a/ldvspec/manage.py b/ldvspec/manage.py
index 110f50eed9bb0655751f09aeaca46b10b0507598..b99a0a4bbaf9cacd4d3a770bd1dd73a52b69b170 100644
--- a/ldvspec/manage.py
+++ b/ldvspec/manage.py
@@ -6,7 +6,7 @@ import sys
def main():
"""Run administrative tasks."""
- os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ldvspec.settings')
+ os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ldvspec.settings.dev')
try:
from django.core.management import execute_from_command_line
except ImportError as exc: