From 40dce357a15e4b52936830c2893d16ff4b71cc0f Mon Sep 17 00:00:00 2001
From: Fanna Lautenbach <lautenbach@astron.nl>
Date: Wed, 30 Nov 2022 11:20:24 +0100
Subject: [PATCH] make surl unique + migration + test
---
.../migrations/0010_alter_dataproduct_surl.py | 18 ++++++++++++++++++
ldvspec/lofardata/models.py | 14 +++++++++++---
ldvspec/lofardata/tests/test_dataproduct.py | 18 ++++++++++++++----
3 files changed, 43 insertions(+), 7 deletions(-)
create mode 100644 ldvspec/lofardata/migrations/0010_alter_dataproduct_surl.py
diff --git a/ldvspec/lofardata/migrations/0010_alter_dataproduct_surl.py b/ldvspec/lofardata/migrations/0010_alter_dataproduct_surl.py
new file mode 100644
index 00000000..47808b6e
--- /dev/null
+++ b/ldvspec/lofardata/migrations/0010_alter_dataproduct_surl.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.2 on 2022-11-30 09:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('lofardata', '0009_auto_20221110_1307'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='dataproduct',
+ name='surl',
+ field=models.CharField(max_length=200, unique=True),
+ ),
+ ]
diff --git a/ldvspec/lofardata/models.py b/ldvspec/lofardata/models.py
index 7a59d59e..b2515eee 100644
--- a/ldvspec/lofardata/models.py
+++ b/ldvspec/lofardata/models.py
@@ -1,12 +1,13 @@
+import logging
from urllib.parse import urlsplit
from celery.result import AsyncResult
from django.contrib.auth.models import User
from django.contrib.postgres.fields import ArrayField
-from django.db import models
+from django.db import models, IntegrityError
from django.utils.translation import gettext_lazy as _
-from django_filters.rest_framework import FilterSet
+logger = logging.getLogger(__name__)
class DataLocation(models.Model):
name = models.CharField(max_length=50, primary_key=True)
@@ -44,7 +45,7 @@ class DataProduct(models.Model):
project = models.CharField(max_length=50)
location = models.ForeignKey(DataLocation, on_delete=models.DO_NOTHING)
activity = models.CharField(max_length=50)
- surl = models.CharField(max_length=200)
+ surl = models.CharField(max_length=200, unique=True)
filesize = models.PositiveBigIntegerField()
additional_meta = models.JSONField()
@@ -79,6 +80,13 @@ class DataProduct(models.Model):
dp.save()
return dp
+ def save(self, *args, **kwargs):
+ try:
+ super(DataProduct, self).save(*args, **kwargs)
+ except IntegrityError:
+ logger.warning("Surl '{}' is already present".format(self.surl))
+ pass
+
class DataFilterType(models.TextChoices):
DROPDOWN = 'Dropdown', _('Dropdown')
diff --git a/ldvspec/lofardata/tests/test_dataproduct.py b/ldvspec/lofardata/tests/test_dataproduct.py
index 47c0c656..2460f673 100644
--- a/ldvspec/lofardata/tests/test_dataproduct.py
+++ b/ldvspec/lofardata/tests/test_dataproduct.py
@@ -2,12 +2,13 @@ import django.test as dtest
import rest_framework.test as rtest
from django.contrib.auth.models import User
import rest_framework.status as response_status
+from django.db import IntegrityError
+
from lofardata.models import DataProduct, DataLocation
test_object_value = dict(obs_id='12345', oid_source='SAS', dataproduct_source='lofar',
dataproduct_type='observation',
project='LT10_10',
- location='srm://surfsara.nl:4884/',
activity='observation',
surl='srm://surfsara.nl:4884/subfolder/some_nice.tar',
filesize=40,
@@ -29,7 +30,6 @@ class TestDatabaseInteraction(dtest.TestCase):
def test_insert_with_helper_function(self):
test_values = dict(test_object_value)
- test_values.pop('location')
dp = DataProduct.insert_dataproduct(**test_values)
@@ -39,6 +39,15 @@ class TestDatabaseInteraction(dtest.TestCase):
self.assertEqual('Sara', dp.location.name)
self.assertEqual('surfsara.nl:4884', dp.location.uri)
+ def test_insert_non_unique_surl(self):
+ test_values = dict(test_object_value)
+ DataProduct.insert_dataproduct(**test_values)
+
+ try:
+ DataProduct.insert_dataproduct(**test_values)
+ except IntegrityError:
+ self.fail("Inserting a dataproduct with a surl that's already present raised an Integrity error unexpectedly. It should pass quietly!")
+
class TestRESTAPI(rtest.APITestCase):
def setUp(self):
@@ -63,7 +72,8 @@ class TestRESTAPI(rtest.APITestCase):
def test_insert_flat_multi(self):
test_payload = dict(test_object_value)
test_payload.pop('location')
- response = self.client.post('/ldvspec/api/v1/insert_dataproduct/', data=[test_payload, test_payload], format='json')
+ response = self.client.post('/ldvspec/api/v1/insert_dataproduct/', data=[test_payload, test_payload],
+ format='json')
self.assertEqual(response_status.HTTP_201_CREATED, response.status_code)
self.assertTrue(DataProduct.objects.count() == 2, 'Not all dataproduct have been inserted')
@@ -76,4 +86,4 @@ class TestRESTAPI(rtest.APITestCase):
self.assertEqual(response_status.HTTP_201_CREATED, response.status_code)
self.assertTrue(DataProduct.objects.count() == 1, 'Not all dataproduct have been inserted')
- self.assertTrue(DataLocation.objects.count() == 1, 'Not all dataproduct have been inserted')
\ No newline at end of file
+ self.assertTrue(DataLocation.objects.count() == 1, 'Not all dataproduct have been inserted')
--
GitLab