Add test and refactor views

.gitlab-ci.yml

 stages:
+  - test
   - build
   - deploy_to_test
   - deploy_to_production
 
+docker-test:
+  image: python:3.6.7-alpine
+  stage: test
+  services:
+    - postgres:11.0
+  variables:
+    POSTGRES_DB: ldv-spec-db
+    POSTGRES_USER: postgres
+    POSTGRES_PASSWORD: "atdb123"
+  script:
+    - cd ldvspec
+    - pip install -r requirements/dev.txt
+    - python manage.py makemigrations
+    - python manage.py migrate
+    - python manage.py test
+
 docker-build-master:
   # Official docker image.
   image: docker:latest

ldvspec/ldvspec/settings/base.py

@@ -131,7 +131,4 @@ REST_FRAMEWORK = {
     'PAGE_SIZE': 100
 }
 
-try:
-    ATDB_HOST = os.environ['ATDB_HOST']
-except:
-    ATDB_HOST = "http://localhost:8000/atdb/"
\ No newline at end of file
+ATDB_HOST = os.environ.get('ATDB_HOST', 'http://localhost:8000/atdb/')

ldvspec/ldvspec/settings/ci.py

+from ldvspec.settings.base import *
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEV = True
+DEBUG = True
+
+ALLOWED_HOSTS = ["*"]
+CORS_ORIGIN_ALLOW_ALL = True
+
+DATABASES = {
+    'default': {
+         'ENGINE': 'django.db.backends.postgresql_psycopg2',
+         'USER': 'postgres',
+         'PASSWORD': 'atdb123',
+         'NAME': 'ldv-spec-db',
+         'HOST': 'postgres',
+         'PORT': '5432',
+    },
+}
+
+# Password validation
+# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = []

ldvspec/lofardata/models.py

 from django.db import models
 from django_filters import rest_framework as filters
+from urllib.parse import urlsplit
 
 
-
-
-class DataLocations(models.Model):
+class DataLocation(models.Model):
     name = models.CharField(max_length=50, primary_key=True)
     uri = models.CharField(max_length=200)
 
+    @staticmethod
+    def insert_location_from_string(location_string):
+        """
+        Insert a datalocation from a srm string (e.g. srm://surm:4321/path.tar)
+
+        :param str location_string: SRM url
+        :return: DataLocation object
+        rtype: DataLocations
+        """
+        _, netloc, *_ = urlsplit(location_string)
+
+        dataloc = DataLocation(name=netloc, uri=location_string.rstrip('/'))
+        dataloc.save()
+        return dataloc
+
 
 class DataProduct(models.Model):
     obs_id = models.CharField(verbose_name="OBS_ID", max_length=15)
@@ -15,16 +29,40 @@ class DataProduct(models.Model):
     dataproduct_source = models.CharField(max_length=20)
     dataproduct_type = models.CharField(max_length=50)
     project = models.CharField(max_length=15)
-    location = models.ForeignKey(DataLocations, on_delete=models.DO_NOTHING)
+    location = models.ForeignKey(DataLocation, on_delete=models.DO_NOTHING)
     activity = models.CharField(max_length=50)
     surl = models.CharField(max_length=200)
     filesize = models.PositiveBigIntegerField()
     additional_meta = models.JSONField()
 
+    @staticmethod
+    def insert_dataproduct(obs_id,
+                           oid_source,
+                           dataproduct_source,
+                           dataproduct_type,
+                           project,
+                           activity,
+                           surl,
+                           filesize,
+                           additional_meta):
+        scheme, netloc, *_ = urlsplit(surl)
+
+        dp = DataProduct(obs_id=obs_id,
+                         oid_source=oid_source,
+                         dataproduct_source=dataproduct_source,
+                         dataproduct_type=dataproduct_type,
+                         project=project,
+                         location=DataLocation.insert_location_from_string('://'.join((scheme, netloc))),
+                         activity=activity,
+                         filesize=filesize,
+                         additional_meta=additional_meta,
+                         surl=surl
+                         )
+        dp.save()
+        return dp
 
-class DataProductFilter(models.Model):
 
+class DataProductFilter(models.Model):
     field = models.CharField(max_length=20)
     name = models.CharField(max_length=20)
     lookup_type = models.CharField(max_length=100)
-

ldvspec/lofardata/serializers.py

+from abc import ABC
+
 from rest_framework import serializers
-from .models import DataProduct
+from .models import DataProduct, DataLocation
+
+
+class DataLocationSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = DataLocation
+        fields = '__all__'
 
-class DataProductSerializer(serializers.ModelSerializer):
 
+class DataProductSerializer(serializers.ModelSerializer):
     class Meta:
         model = DataProduct
         fields = "__all__"
+
+
+class DataProductFlatSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = DataProduct
+        exclude = ('location',)
+
+    def create(self, validated_data):
+        return DataProduct.insert_dataproduct(**validated_data)

ldvspec/lofardata/tests.py

-from django.test import TestCase
-
-# Create your tests here.

ldvspec/lofardata/tests/test_data_filter.py

+import rest_framework.test as rtest
+from django.contrib.auth.models import User
+from lofardata.models import DataProduct, DataProductFilter
+
+test_object_values = [dict(obs_id='12345', oid_source='SAS', dataproduct_source='lofar',
+                           dataproduct_type='observation',
+                           project='LT10_10',
+                           activity='observation',
+                           surl='srm://surfsara.nl:4884/...',
+                           filesize=40,
+                           additional_meta={'dysco_compression': True}),
+                      dict(obs_id='12346', oid_source='SAS', dataproduct_source='lofar',
+                           dataproduct_type='observation',
+                           project='LT10_10',
+                           activity='pipeline',
+                           surl='srm://surfsara.nl:4884/...',
+                           filesize=40,
+                           additional_meta={'dysco_compression': True}),
+                      dict(obs_id='12347', oid_source='SAS', dataproduct_source='lofar',
+                           dataproduct_type='observation',
+                           project='LT10_10',
+                           activity='observation',
+                           surl='srm://surfsara.nl:4884/...',
+                           filesize=40,
+                           additional_meta={'dysco_compression': True})]
+
+
+class TestDataFilter(rtest.APITestCase):
+    def setUp(self):
+        self.client = rtest.APIClient()
+        self.user = User.objects.create_superuser('admin')
+        self.client.force_authenticate(self.user)
+
+        for test_object in test_object_values:
+            DataProduct.insert_dataproduct(**test_object)
+
+    def get_current_query_parameters(self):
+        response = self.client.get('/ldvspec/openapi/')
+        query_parameters = response.data['paths']['/ldvspec/data/']['get']['parameters']
+        return {parameter['name']: parameter for parameter in query_parameters}
+
+    def test_add_custom_filter(self):
+        self.assertNotIn('activity', self.get_current_query_parameters(), 'Test is invalid! Update')
+
+        dp_filter = DataProductFilter(field='activity', name='activity_type', lookup_type='exact')
+        dp_filter.save()
+
+        response = self.client.get('/ldvspec/data/?activity_type=pipeline')
+        self.assertEqual(1, response.data['count'])
+        self.assertEqual('pipeline', response.data['results'][0]['activity'])
+
+        response = self.client.get('/ldvspec/data/?activity_type=observation')
+        self.assertEqual(2, response.data['count'])
+        self.assertEqual('observation', response.data['results'][0]['activity'])
+
+        self.assertIn('activity_type', self.get_current_query_parameters())
\ No newline at end of file

ldvspec/lofardata/tests/test_dataproduct.py

+import django.test as dtest
+import rest_framework.test as rtest
+from django.contrib.auth.models import User
+import rest_framework.status as response_status
+from lofardata.models import DataProduct, DataLocation
+
+test_object_value = dict(obs_id='12345', oid_source='SAS', dataproduct_source='lofar',
+                         dataproduct_type='observation',
+                         project='LT10_10',
+                         location='srm://surfsara.nl:4884/',
+                         activity='observation',
+                         surl='srm://surfsara.nl:4884/...',
+                         filesize=40,
+                         additional_meta={'dysco_compression': True})
+
+
+class TestDatabaseInteraction(dtest.TestCase):
+    def test_insert(self):
+        location = DataLocation(name='surfsara', uri='srm://surfsara.nl')
+        location.save()
+        test_values = dict(test_object_value)
+        test_values['location'] = location
+        dp = DataProduct(**test_values)
+        dp.save()
+        self.assertTrue(dp.pk is not None, 'Failed saving object')
+        dp = DataProduct.objects.get(obs_id='12345')
+        for field in test_values:
+            self.assertEqual(getattr(dp, field), test_values.get(field))
+
+    def test_insert_with_helper_function(self):
+        test_values = dict(test_object_value)
+        test_values.pop('location')
+
+        dp = DataProduct.insert_dataproduct(**test_values)
+
+        for field in test_values:
+            self.assertEqual(test_object_value.get(field), getattr(dp, field), msg=f'Field {field} does not coincide')
+
+        self.assertEqual('surfsara.nl:4884', dp.location.name)
+        self.assertEqual( 'srm://surfsara.nl:4884', dp.location.uri)
+
+
+class TestRESTAPI(rtest.APITestCase):
+    def setUp(self):
+        self.client = rtest.APIClient()
+        self.user = User.objects.create_superuser('admin')
+        self.client.force_authenticate(self.user)
+
+    def test_insert_not_allowed(self):
+        client = rtest.APIClient()
+        response = client.post('/ldvspec/data/insert/', data={}, format='json')
+        self.assertEqual(response_status.HTTP_403_FORBIDDEN, response.status_code)
+
+    def test_insert_flat_error(self):
+        response = self.client.post('/ldvspec/data/insert/', data={}, format='json')
+        self.assertEqual(response_status.HTTP_400_BAD_REQUEST, response.status_code)
+
+    def test_insert_flat_single(self):
+        test_payload = dict(test_object_value)
+        test_payload.pop('location')
+        response = self.client.post('/ldvspec/data/insert/', data=test_payload, format='json')
+        self.assertEqual(response_status.HTTP_201_CREATED, response.status_code)
+
+    def test_insert_flat_multi(self):
+        test_payload = dict(test_object_value)
+        test_payload.pop('location')
+        response = self.client.post('/ldvspec/data/insert/', data=[test_payload, test_payload], format='json')
+
+        self.assertEqual(response_status.HTTP_201_CREATED, response.status_code)
+        self.assertTrue(DataProduct.objects.count() == 2, 'Not all dataproduct have been inserted')
+        self.assertTrue(DataLocation.objects.count() == 1, 'Not all dataproduct have been inserted')
+
+    def test_insert_flat_multi_insert_single(self):
+        test_payload = dict(test_object_value)
+        test_payload.pop('location')
+        response = self.client.post('/ldvspec/data/insert/', data=test_payload, format='json')
+
+        self.assertEqual(response_status.HTTP_201_CREATED, response.status_code)
+        self.assertTrue(DataProduct.objects.count() == 1, 'Not all dataproduct have been inserted')
+        self.assertTrue(DataLocation.objects.count() == 1, 'Not all dataproduct have been inserted')
\ No newline at end of file

ldvspec/lofardata/tests/test_location.py

+import django.test as dtest
+import rest_framework.test as rtest
+from django.contrib.auth.models import User
+import rest_framework.status as response_status
+from lofardata.models import DataLocation
+
+
+class TestDataLocation(dtest.TestCase):
+
+    def test_insert(self):
+        location = DataLocation(name='test', uri='srm://path_to_file/')
+        location.save()
+
+        self.assertTrue(location.pk is not None, 'Cannot save object')
+
+    def test_double_insert_unique_entry(self):
+        location = DataLocation(name='test', uri='srm://path_to_file/')
+        location.save()
+
+        self.assertTrue(location.pk is not None, 'Cannot save object')
+        location_first_pk = location.pk
+
+        location = DataLocation(name='test', uri='srm://path_to_file/')
+        location.save()
+
+        self.assertTrue(location.pk is not None, 'Cannot save object')
+        location_second_pk = location.pk
+
+        self.assertTrue(location_first_pk == location_second_pk, 'Double inserted index do not coincide')
+
+        self.assertTrue(DataLocation.objects.count() == 1)
+
+    def test_insert_from_string(self):
+        dataloc = DataLocation.insert_location_from_string('srm://test_site:44321/')
+
+        self.assertTrue(dataloc.pk is not None, 'Cannot save object')
+        self.assertEqual('test_site:44321', dataloc.name)
+        self.assertEqual('srm://test_site:44321', dataloc.uri)
+
+
+class TestRESTAPI(rtest.APITestCase):
+    def setUp(self):
+        self.client = rtest.APIClient()
+        self.user = User.objects.create_superuser('admin')
+        self.client.force_authenticate(self.user)
+
+    def test_create_error(self):
+        response = self.client.post('/ldvspec/data-location/', data=dict(name='testname'),
+                                    format='json')
+        self.assertEqual(response_status.HTTP_400_BAD_REQUEST, response.status_code)
+
+    def test_create(self):
+        response = self.client.post('/ldvspec/data-location/', data=dict(name='testname', uri='srm://myniceuri/'),
+                                    format='json')
+        self.assertEqual(response_status.HTTP_201_CREATED, response.status_code)
+        self.assertEqual('srm://myniceuri/', DataLocation.objects.get(name='testname').uri)
\ No newline at end of file

ldvspec/lofardata/urls.py

@@ -15,6 +15,9 @@ urlpatterns = [
 
     # REST API
     path('data/', views.DataProductView.as_view(), name='dataproduct'),
+    path('data/insert/', views.InsertMultiDataproductView.as_view(), name='dataproduct-insert'),
+    path('data-location/', views.DataLocationView.as_view(), name='datalocation'),
+
     path('openapi/', get_schema_view(
         title="LDV Specification",
         description="API description",

ldvspec/lofardata/views.py

+import logging
+
 from django.shortcuts import render
 from django.conf import settings
 
 from rest_framework import generics, pagination
+from rest_framework.views import APIView
 
+from rest_framework import status
+from rest_framework.response import Response
 from django_filters import rest_framework as filters
 
-from .models import DataProduct, DataProductFilter
-from .serializers import DataProductSerializer
+from .models import DataProduct, DataProductFilter, DataLocation
+from .serializers import DataProductSerializer, DataProductFlatSerializer, DataLocationSerializer
 
 
 class DynamicFilterSet(filters.FilterSet):
@@ -18,16 +23,12 @@ class DynamicFilterSet(filters.FilterSet):
         super().__init__(*args, **kwargs)
 
     def _load_filters(self):
-        print('Current filters are', self.base_filters)
         if self.Meta.filter_class is None:
             raise Exception('Define filter_class meta attribute')
-
-        print('Filter to be added', self.Meta.filter_class.objects.all())
         for item in self.Meta.filter_class.objects.all():
             field_obj = self.Meta.model._meta.get_field(item.field)
             filter_class, *_ = self.filter_for_lookup(field_obj, item.lookup_type)
             self.base_filters[item.name] = filter_class(item.field)
-        print('Final filters are', self.base_filters)
 
 
 # --- Filters ---
@@ -48,7 +49,6 @@ def index(request):
 
 
 # ---------- REST API views ----------
-
 class DataProductView(generics.ListCreateAPIView):
     model = DataProduct
     serializer_class = DataProductSerializer
@@ -59,5 +59,22 @@ class DataProductView(generics.ListCreateAPIView):
     filter_backends = (filters.DjangoFilterBackend,)
     filter_class = DataProductFilterSet().__class__
 
-    def get_queryset(self):
-        return self.queryset.order_by('obs_id')
+
+class DataLocationView(generics.ListCreateAPIView):
+    model = DataLocation
+    serializer_class = DataLocationSerializer
+    queryset = DataLocation.objects.all().order_by('name')
+
+
+class InsertMultiDataproductView(generics.CreateAPIView):
+    """
+    Add single DataProduct
+    """
+    queryset = DataProduct.objects.all()
+    serializer_class = DataProductFlatSerializer
+
+    def get_serializer(self, *args, **kwargs):
+        """ if an array is passed, set serializer to many """
+        if isinstance(kwargs.get('data', {}), list):
+            kwargs['many'] = True
+        return super().get_serializer(*args, **kwargs)

ldvspec/manage.py

@@ -6,7 +6,7 @@ import sys
 
 def main():
     """Run administrative tasks."""
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ldvspec.settings')
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ldvspec.settings.dev')
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc: