diff --git a/README.md b/README.md
index cc4e8af902bbb607ca28da3d77347a011e30e2bf..21f8995ce38dfce975d548f06ad16249cf9967ce 100644
--- a/README.md
+++ b/README.md
@@ -36,11 +36,14 @@ Currently they are still mostly the original ATDB diagrams. They will be adapted
-## Manual deploy in Docker
+## CI/CD (semi) automatic deploy in Docker
For the `master' branch there is a CI/CD pipeline in place which builds and deploys the backend at https://sdc.astron.nl:5554/atdb/
+
The deploy step requires pushing the 'play' button in the gitlab pipelines section.
This is done to not have a mandatory deploy for every minor commmit.
+## Manual deploy in Docker (alternative to CI/CD)
+
### initial
> cd ~/my_docker
diff --git a/atdb/taskdatabase/admin.py b/atdb/taskdatabase/admin.py
index 134b52d8d4ce0a91433c371a5605868778fe2f45..d38dc14d6aa10359b4b6febc150643f7ce434f1f 100644
--- a/atdb/taskdatabase/admin.py
+++ b/atdb/taskdatabase/admin.py
@@ -1,6 +1,5 @@
from django.contrib import admin
-from .models import Status, DataProduct, Observation
+from .models import Status, Observation
admin.site.register(Status)
-admin.site.register(DataProduct)
admin.site.register(Observation)
\ No newline at end of file
diff --git a/atdb/taskdatabase/migrations/0003_auto_20210114_1133.py b/atdb/taskdatabase/migrations/0003_auto_20210114_1133.py
new file mode 100644
index 0000000000000000000000000000000000000000..51495fc7aaff428e1b6f6bed287fa0df137e3d5f
--- /dev/null
+++ b/atdb/taskdatabase/migrations/0003_auto_20210114_1133.py
@@ -0,0 +1,26 @@
+# Generated by Django 3.1.4 on 2021-01-14 10:33
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('taskdatabase', '0002_taskobject_metadata'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='observation',
+ name='progress',
+ field=models.CharField(blank=True, default='0%', max_length=40, null=True),
+ ),
+ migrations.AlterField(
+ model_name='taskobject',
+ name='task_type',
+ field=models.CharField(default='observation', max_length=20),
+ ),
+ migrations.DeleteModel(
+ name='DataProduct',
+ ),
+ ]
diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py
index 8fa8af5b69107c9d079c156dea2dda799124a7c2..3436af0afca80cc5d843b5531fec01031e0c4d3a 100644
--- a/atdb/taskdatabase/models.py
+++ b/atdb/taskdatabase/models.py
@@ -7,30 +7,12 @@ from django.db.models import Sum
datetime_format_string = '%Y-%m-%dT%H:%M:%SZ'
TASK_TYPE_OBSERVATION = 'observation'
-TASK_TYPE_DATAPRODUCT = 'dataproduct'
TYPE_VISIBILITY = 'visibility'
-# common model functions
-def get_sum_from_dataproduct_field(taskID, field):
- """
- sum the values of a field for certain taskID. (used to sum total of dataproduct sizes)
- :param taskID:
- :param field:
- :return:
- """
- query = field + '__sum'
- sum_value = DataProduct.objects.filter(taskID=taskID).aggregate(Sum(field))[query]
- if sum_value == None:
- sum_value = 0.0
- return sum_value
-
-"""
-a base class of both Observation and Dataproducts
-"""
class TaskObject(models.Model):
name = models.CharField(max_length=100, default="unknown")
- task_type = models.CharField(max_length=20, default=TASK_TYPE_DATAPRODUCT)
+ task_type = models.CharField(max_length=20, default=TASK_TYPE_OBSERVATION)
taskID = models.CharField('runId', db_index=True, max_length=30, blank=True, null=True)
creationTime = models.DateTimeField(default=datetime.utcnow, blank=True)
@@ -99,31 +81,6 @@ class Observation(TaskObject):
duration = 0
return duration
- @property
- def size(self):
- # sum the sizes of all dataproducts with this taskID. In Mb
- size = get_sum_from_dataproduct_field(self.taskID,'size')
- return size
-
def __str__(self):
return str(self.taskID)
-
-class DataProduct(TaskObject):
- # properties
- filename = models.CharField(max_length=200, default="unknown")
- description = models.CharField(max_length=255, default="unknown")
- dataproduct_type = models.CharField('type', default=TYPE_VISIBILITY, max_length=50)
- size = models.BigIntegerField(default=0)
-
- # relationships
- parent = models.ForeignKey(Observation, related_name='generated_dataproducts', on_delete=models.CASCADE, null=False)
-
- # this translates a view-name (from urls.py) back to a url, to avoid hardcoded url's in the html templates
- # bad : <td><a href="/atdb/observations/{{ observation.id }}/" target="_blank">{{ observation.taskID }} </a> </td>
- # good: <td><a href="{{ observation.get_absolute_url }}" target="_blank">{{ observation.taskID }} </a> </td>
- def get_absolute_url(self):
- return reverse('dataproduct-detail-view-api', kwargs={'pk': self.pk})
-
- def __str__(self):
- return self.filename
diff --git a/atdb/taskdatabase/serializers.py b/atdb/taskdatabase/serializers.py
index 39040fcd1196f42f143b2f816e32679a346679b0..02d7a04c6dd8cfbdeaf2e5dc065deefb2d22b253 100644
--- a/atdb/taskdatabase/serializers.py
+++ b/atdb/taskdatabase/serializers.py
@@ -1,5 +1,5 @@
from rest_framework import serializers
-from .models import DataProduct, Observation, Status, TaskObject
+from .models import Observation, Status, TaskObject
import logging
logger = logging.getLogger(__name__)
@@ -11,37 +11,7 @@ class StatusSerializer(serializers.ModelSerializer):
fields = ('id','name','timestamp','property_taskID','property_task_type')
-class DataProductSerializer(serializers.ModelSerializer):
- # this adds a 'parent_observation' list with hyperlinks to the DataProduct API.
- # note that 'generatedByObservation' is not defined in the DataProduct model, but in the Observation model.
-
- parent = serializers.PrimaryKeyRelatedField(
- many=False,
- queryset=Observation.objects.all(),
- required=False,
- )
-
- status_history = serializers.StringRelatedField(
- many=True,
- required=False,
- )
-
- class Meta:
- model = DataProduct
- fields = ('id','task_type','name','filename','description',
- 'taskID','creationTime','size',
- 'my_status','new_status','status_history','parent',
- 'data_location','node','quality','metadata')
-
-
class ObservationSerializer(serializers.ModelSerializer):
- # this adds a 'generated_dataproducts' list with hyperlinks to the Observation API.
- # note that 'generated_dataproducts' is not defined in the DataProduct model, but comes from the related_field in Observation.parent.
-
- generated_dataproducts = serializers.StringRelatedField(
- many=True,
- required=False,
- )
status_history = serializers.StringRelatedField(
many=True,
@@ -53,8 +23,7 @@ class ObservationSerializer(serializers.ModelSerializer):
fields = ('id','task_type', 'name','taskID',
'field_name','field_ra','field_dec',
'creationTime','starttime','endtime', 'duration', 'size',
- 'my_status','new_status','status_history',
- 'generated_dataproducts',
+ 'my_status','new_status','status_history','generated_dataproducts',
'data_location', 'node',
'skip_auto_ingest','observing_mode',
'quality','metadata','progress')
diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py
index 5e61721adb643d22d8042d3d2f2f5232106d657c..6d9d0004d6e311dc6d95100251c225b2e3d9132c 100644
--- a/atdb/taskdatabase/services/algorithms.py
+++ b/atdb/taskdatabase/services/algorithms.py
@@ -8,7 +8,7 @@ import time
import datetime
import logging
from .common import timeit
-from ..models import Observation, DataProduct
+from ..models import Observation
DATE_FORMAT = "%Y-%m-%d"
TIME_FORMAT = "%Y-%m-%d %H:%M:%SZ"
@@ -115,45 +115,3 @@ def get_next_observation(my_status, observing_mode, datawriter):
return taskID,minutes_left
-# /atdb/post_dataproducts
-def add_dataproducts(dataproducts):
- """
- :param taskID: taskid of the observation
- :param dataproducts: json list of dataproducts to be added to the provided taskid
- :return:
- """
-
- taskID = None
- dps = dataproducts.get('dps')
- number_of_dataproducts = len(dps)
-
- logger.info("add_dataproducts("+str(number_of_dataproducts)+")")
-
- for dp in dps:
- # get the common fields from the observation based on the given taskid
- taskID = dp.get('taskID')
- parent = Observation.objects.get(taskID=taskID)
- parent_data_location = parent.data_location
-
- node=dp.get('node',None)
- new_status = dp.get('new_status','defined')
- data_location = dp.get('data_dir',parent_data_location)
- size = dp.get('size', 0)
- myDataProduct = DataProduct(taskID=taskID,
- node=node,
- data_location=data_location,
- filename=dp['filename'],
- name=dp['filename'],
- description=dp['filename'],
- quality=dp['quality'],
- metadata=dp['metadata'],
- task_type='dataproduct',
- new_status=new_status,
- parent=parent,
- size=size
- )
-
- logger.info('addding dataproduct: '+str(myDataProduct))
- myDataProduct.save()
-
- return taskID
diff --git a/atdb/taskdatabase/services/signals.py b/atdb/taskdatabase/services/signals.py
index cb7050b7b87a804b7db8a2dfba2b4b295f1d767b..1fce99e58a8999beb4fb61e28c4bca7b02c23dfd 100644
--- a/atdb/taskdatabase/services/signals.py
+++ b/atdb/taskdatabase/services/signals.py
@@ -5,7 +5,7 @@ from django.core.signals import request_started, request_finished
from django.contrib.auth.models import User
from django.dispatch import receiver
from django.contrib.contenttypes.models import ContentType
-from taskdatabase.models import TaskObject, Observation, DataProduct, Status
+from taskdatabase.models import TaskObject, Observation, Status
from . import jobs
"""
@@ -26,22 +26,17 @@ def request_started_handler(sender, **kwargs):
def request_finished_handler(sender, **kwargs):
logger.debug("signal : request_finished")
-#--- Observation and DataProduct signals-------------
+#--- Task signals-------------
@receiver(pre_save, sender=Observation)
def pre_save_observation_handler(sender, **kwargs):
logger.info("SIGNAL : pre_save Observation(" + str(kwargs.get('instance')) + ")")
handle_pre_save(sender, **kwargs)
-@receiver(pre_save, sender=DataProduct)
-def pre_save_dataproduct_handler(sender, **kwargs):
- logger.info("SIGNAL : pre_save DataProduct(" + str(kwargs.get('instance')) + ")")
- handle_pre_save(sender, **kwargs)
-
def handle_pre_save(sender, **kwargs):
"""
- pre_save handler for both Observation and Dataproduct. Mainly to check status changes and dispatch jobs in needed.
+ pre_save handler. Mainly to check status changes and dispatch jobs in needed.
:param (in) sender: The model class that sends the trigger
:param (in) kwargs: The instance of the object that sends the trigger.
"""
@@ -64,41 +59,6 @@ def handle_pre_save(sender, **kwargs):
myStatus = Status(name=new_status, taskObject=myTaskObject)
myStatus.save()
- #myTaskObject.new_status = None
-
- # when an observation goes to valid, calculate its total size by counting its dataproducts
- # if (myTaskObject.task_type == 'observation') and 'valid' in myTaskObject.new_status:
- # # calculate total size
- # dps = DataProduct.objects.filter(taskID=myTaskObject.taskID)
- # size = 0
- # for dp in dps:
- # size = size + dp.size
- #
- # logger.info("total size of observation "+myTaskObject.taskID+ " = "+ str(size))
- # # nv:11jun2019: this requires a database change first
- # # myTaskObject.size = size
-
- if (myTaskObject.task_type == 'observation'):
- # convert the utc timestamp to a format that Django REST API understands
- # in its GUI, otherwise null values will be put in when hitting PUT.
- s,_ = str(myStatus.timestamp).split('.')
- myTimestamp = datetime.datetime.strptime(s, "%Y-%m-%d %H:%M:%S")
-
- if 'starting' in myTaskObject.new_status:
- myTaskObject.timestamp_starting = myTimestamp
- elif 'running' in myTaskObject.new_status:
- myTaskObject.timestamp_running = myTimestamp
- elif 'completing' in myTaskObject.new_status:
- myTaskObject.timestamp_completing = myTimestamp
- elif 'ingesting' in myTaskObject.new_status:
- myTaskObject.timestamp_ingesting = myTimestamp
- elif 'archived' in myTaskObject.new_status:
- myTaskObject.timestamp_archived = myTimestamp
- elif 'aborted' in myTaskObject.new_status:
- myTaskObject.timestamp_aborted = myTimestamp
- elif 'ingest_error' in myTaskObject.new_status:
- myTaskObject.timestamp_ingest_error = myTimestamp
-
# temporarily disconnect the post_save handler to save the dataproduct (again) and avoiding recursion.
# I don't use pre_save, because then the 'created' key is not available, which is the most handy way to
# determine if this dataproduct already exists. (I could also check the database, but this is easier).
@@ -117,12 +77,6 @@ def post_save_observation_handler(sender, **kwargs):
handle_post_save(sender, **kwargs)
-@receiver(post_save, sender=DataProduct)
-def post_save_dataproduct_handler(sender, **kwargs):
- #logger.info("SIGNAL : post_save DataProduct(" + str(kwargs.get('instance')) + ")")
- handle_post_save(sender, **kwargs)
-
-
def handle_post_save(sender, **kwargs):
"""
pre_save handler for both Observation and Dataproduct. To create and write its initial status
@@ -132,7 +86,7 @@ def handle_post_save(sender, **kwargs):
logger.info("handle_post_save("+str(kwargs.get('instance'))+")")
myTaskObject = kwargs.get('instance')
- # CREATE NEW OBSERVATION / DATAPRODUCT
+ # Create new task
if kwargs['created']:
logger.info("save new "+str(myTaskObject.task_type))
@@ -143,12 +97,6 @@ def handle_post_save(sender, **kwargs):
myStatus = Status(name=myTaskObject.new_status, taskObject=myTaskObject)
myStatus.save()
- # if there already is an observation with this taskID, then add this dataproduct to it
- if (myTaskObject.task_type == 'dataproduct'):
- logger.info("update dataproduct parent = " + str(myTaskObject.taskID))
- parent = Observation.objects.get(taskID=myTaskObject.taskID)
- myTaskObject.parent=parent
-
# temporarily disconnect the post_save handler to save the dataproduct (again) and avoiding recursion.
# I don't use pre_save, because then the 'created' key is not available, which is the most handy way to
# determine if this dataproduct already exists. (I could also check the database, but this is easier).
@@ -159,14 +107,9 @@ def handle_post_save(sender, **kwargs):
def connect_signals():
#logger.info("connect_signals")
pre_save.connect(pre_save_observation_handler, sender=Observation)
- pre_save.connect(pre_save_dataproduct_handler, sender=DataProduct)
post_save.connect(post_save_observation_handler, sender=Observation)
- post_save.connect(post_save_dataproduct_handler, sender=DataProduct)
-
def disconnect_signals():
#logger.info("disconnect_signals")
pre_save.disconnect(pre_save_observation_handler, sender=Observation)
- pre_save.disconnect(pre_save_dataproduct_handler, sender=DataProduct)
post_save.disconnect(post_save_observation_handler, sender=Observation)
- post_save.disconnect(post_save_dataproduct_handler, sender=DataProduct)
\ No newline at end of file
diff --git a/atdb/taskdatabase/templates/taskdatabase/index.html b/atdb/taskdatabase/templates/taskdatabase/index.html
index a209cb6970be9be68ab6ba12951a34a45eb57009..b55c96dbefe5b35c91b8a7cf3eeecce90acbe0e3 100644
--- a/atdb/taskdatabase/templates/taskdatabase/index.html
+++ b/atdb/taskdatabase/templates/taskdatabase/index.html
@@ -45,7 +45,7 @@
</div>
{% include 'taskdatabase/pagination.html' %}
</div>
- <p class="footer"> Version 1.0.0 (7 jan 2021 - 08:30)
+ <p class="footer"> Version 1.0.0 (14 jan 2021 - 11:30)
<script type="text/javascript">
(function(seconds) {
var refresh,
diff --git a/atdb/taskdatabase/templates/taskdatabase/observations.html b/atdb/taskdatabase/templates/taskdatabase/observations.html
index 86d3f5e763e9237b99e7151752b4b2bf3a4a1092..a3bdc922371b3bf850faa19b228d7b1df25a958d 100644
--- a/atdb/taskdatabase/templates/taskdatabase/observations.html
+++ b/atdb/taskdatabase/templates/taskdatabase/observations.html
@@ -40,14 +40,6 @@
{% endif %}
</td>
<td>
- {% if observation.my_status == "complete" %}
- <a href="{% url 'dataproducts-list-view' observation.taskID %}" class="btn btn-success btn-sm" role="button">DPS</a>
- {% elif observation.my_status == "incomplete" or "error" in observation.my_status %}
- <a href="{% url 'dataproducts-list-view' observation.taskID %}" class="btn btn-danger btn-sm" role="button">DPS</a>
- {% else %}
- <a href="{% url 'dataproducts-list-view' observation.taskID %}" class="btn btn-primary btn-sm" role="button">DPS</a>
- {% endif %}
-
{% if observation.my_status == "ingesting" %}
<a href="https://alta.astron.nl/science/monitor" class="btn btn-primary btn-sm" target="_blank" role="button">Monitor</a>
{% endif %}
@@ -72,11 +64,6 @@
{% endif %}
{% endif %}
- {% if observation.my_status == "incomplete" or observation.my_status == "defined" %}
- <a href="{% url 'observation-dps-setstatus-view' observation.pk 'valid' 'valid' my_observations.number %}" class="btn btn-success btn-sm" role="button">Validate DPS</a>
- <a href="{% url 'observation-dps-setstatus-view' observation.pk 'invalid' 'removed_invisible' my_observations.number %}" class="btn btn-danger btn-sm" role="button">Failed</a>
- {% endif %}
-
{% if observation.my_status == "completed" or observation.my_status == "incomplete" %}
<a href="{% url 'observation-setstatus-view' observation.pk 'valid' my_observations.number %}" class="btn btn-primary btn-sm" role="button">Ready to Ingest</a>
{% endif %}
@@ -85,34 +72,16 @@
<a href="{% url 'observation-setstatus-view' observation.pk 'completing' my_observations.number %}" class="btn btn-success btn-sm" role="button">Completing</a>
{% endif %}
- {% if observation.my_status == "ingest error" or observation.my_status == "ingest aborted" %}
- <a href="{% url 'observation-dps-setstatus-view' observation.pk 'defined' 'defined' my_observations.number %}" class="btn btn-primary btn-sm" role="button">Retry Ingest</a>
- {% endif %}
-
-
- {% if observation.my_status == "archived" or "error" in observation.my_status or "aborted" in observation.my_status or observation.my_status == "completed" or observation.my_status == "incomplete" %}
- <a href="{% url 'observation-dps-setstatus-view' observation.pk 'removing' 'removing' my_observations.number %}" class="btn btn-warning btn-sm" role="button">Remove Data</a>
- {% endif %}
-
{% if observation.my_status == "valid" %}
<a href="{% url 'observation-setstatus-view' observation.pk 'valid_priority' my_observations.number %}" class="btn btn-warning btn-sm" role="button">Priority Ingest</a>
<a href="{% url 'observation-setstatus-view' observation.pk 'completed' my_observations.number %}" class="btn btn-primary btn-sm" role="button">Cancel Ingest</a>
{% endif %}
- {% if user.is_authenticated %}
- {% if observation.my_status == "ingesting" %}
- <a href="{% url 'observation-dps-setstatus-view' observation.pk 'ingest aborted' 'ingest aborted' my_observations.number %}" class="btn btn-danger btn-sm" role="button">Abort Ingest</a>
- {% endif %}
- {% endif %}
{% if observation.my_status == "valid_priority" %}
<a href="{% url 'observation-setstatus-view' observation.pk 'valid' my_observations.number %}" class="btn btn-primary btn-sm" role="button">Cancel Priority</a>
<a href="{% url 'observation-setstatus-view' observation.pk 'completed' my_observations.number %}" class="btn btn-primary btn-sm" role="button">Cancel Ingest</a>
{% endif %}
- {% if observation.my_status == "incomplete" %}
- <a href="{% url 'observation-dps-setstatus-view' observation.pk 'defined' 'defined' my_observations.number %}" class="btn btn-danger btn-sm" role="button">Reset</a>
- {% endif %}
-
{% if observation.my_status == "archived" %}
<a href="{% url 'observation-setquality-view' observation.pk 'data_is_good' my_observations.number %}" class="btn btn-success btn-sm" role="button">Data is Good</a>
<a href="{% url 'observation-setquality-view' observation.pk 'data_is_bad' my_observations.number %}" class="btn btn-danger btn-sm" role="button">Data is Bad</a>
@@ -123,9 +92,6 @@
<a href="{% url 'observation-setquality-view' observation.pk 'data_is_bad' my_observations.number %}" class="btn btn-danger btn-sm" role="button">Data is Bad</a>
{% endif %}
- {% if observation.my_status == "removing" %}
- <a href="{% url 'observation-dps-setstatus-view' observation.pk 'incomplete' 'incomplete' my_observations.number %}" class="btn btn-danger btn-sm" role="button">Cancel and Pray</a>
- {% endif %}
</td>
</tr>
</div>
diff --git a/atdb/taskdatabase/urls.py b/atdb/taskdatabase/urls.py
index b57453258c62ef1b703f2ec42dcb1d22e531d407..1d62b4d684d2ae861cf6cea96cb4bc1cb6a54bdc 100644
--- a/atdb/taskdatabase/urls.py
+++ b/atdb/taskdatabase/urls.py
@@ -8,16 +8,7 @@ urlpatterns = [
#path('', views.index, name='index'),
path('', views.IndexView.as_view(), name='index'),
- # ex: /atdb/task/180223003/
- path('task/<taskID>/', views.DataProductsListView.as_view(), name='dataproducts-list-view'),
-
# --- REST API ---
- # ex: /atdb/dataproducts/
- path('dataproducts/', views.DataProductListViewAPI.as_view()),
-
- # ex: /atdb/dataproducts/5/
- path('dataproducts/<int:pk>/', views.DataProductDetailsViewAPI.as_view(),name='dataproduct-detail-view-api'),
-
# ex: /atdb/observations/
path('observations/', views.ObservationListViewAPI.as_view()),
@@ -27,9 +18,6 @@ urlpatterns = [
# ex: /atdb/observations/5/
path('observations/<int:pk>/', views.ObservationDetailsViewAPI.as_view(),name='observation-detail-view-api'),
- # ex: /atdb/status/
- path('status/', views.StatusListViewAPI.as_view(),name='status-list-view-api'),
-
# --- custom requests ---
# ex: /atdb/get_next_taskid?timestamp=2019-04-05
path('get_next_taskid',
@@ -41,21 +29,11 @@ urlpatterns = [
views.GetNextObservationView.as_view(),
name='get-next-observation-view'),
- # ex: /atdb/post_dataproducts&taskid=190405034
- path('post_dataproducts',
- views.PostDataproductsView.as_view(),
- name='post-dataproducts-view'),
-
# --- controller resources ---
path('observations/<int:pk>/setstatus/<new_status>/<page>',
views.ObservationSetStatus,
name='observation-setstatus-view'),
- path('observations/<int:pk>/setstatus_dps/<new_dps_status>/<new_obs_status>/<page>',
- views.ObservationSetStatusDataProducts,
- name='observation-dps-setstatus-view'),
- path('dataproducts/<int:pk>/setstatus/<new_status>',
- views.DataProductSetStatusView,
- name='dataproduct-setstatus-view'),
+
# set the quality field to 'good' or 'bad' (and transmit it to ALTA)
path('observations/<int:pk>/setquality/<quality>/<page>',
views.ObservationSetQuality,
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index 0ef8ec318e366aa933c93b2128f7c5cb04b88127..641651897d2fb101aa40e0cf1395574517034c6d 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -13,9 +13,9 @@ from django.template import loader
from django.shortcuts import render, redirect
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
-from .models import DataProduct, Observation, Status
+from .models import Observation, Status
from django.db.models import Q
-from .serializers import DataProductSerializer, ObservationSerializer, StatusSerializer
+from .serializers import ObservationSerializer, StatusSerializer
from .forms import FilterForm
from .services import algorithms
@@ -48,25 +48,6 @@ class ObservationFilter(filters.FilterSet):
'quality': ['exact', 'icontains'],
}
-# example: /atdb/dataproducts?status__in=created,archived
-class DataProductFilter(filters.FilterSet):
-
- class Meta:
- model = DataProduct
-
- fields = {
- 'dataproduct_type': ['exact', 'in'], # ../dataproducts?dataProductType=IMAGE,VISIBILITY
- 'description': ['exact', 'icontains'],
- 'name': ['exact', 'icontains'],
- 'filename': ['exact', 'icontains'],
- 'taskID': ['exact', 'icontains'],
- 'creationTime': ['gt', 'lt', 'gte', 'lte', 'contains', 'exact'],
- 'size': ['gt', 'lt', 'gte', 'lte', 'contains', 'exact'],
- 'parent__taskID': ['exact', 'in', 'icontains'],
- 'my_status': ['exact', 'icontains', 'in'],
- 'data_location': ['exact', 'icontains'],
- 'node': ['exact', 'in'],
- }
# example: has 1811130001 been on 'running?'
# http://localhost:8000/atdb/status/?&taskID=181130001&name=running
@@ -175,61 +156,7 @@ def get_searched_observations(search):
return observations
-# example: /atdb/task/180323003/
-# https://docs.djangoproject.com/en/2.1/topics/class-based-views/generic-display/
-# calling this view renders the dataproducts.html template in the GUI
-# a custom pagination class to return more than the default 100 dataproducts
-class DataProductsPagination(pagination.PageNumberPagination):
- page_size = 10000
-
-class DataProductsListView(ListView):
- model = DataProduct
- context_object_name = 'my_dataproducts_list'
- template_name = 'taskdatabase/dataproducts.html'
- pagination_class = DataProductsPagination
-
- # override get_queryset to make a custom query on taskid
- def get_queryset(self):
- logger.info("DataProductsListView.get_queryset()")
- taskid = self.kwargs['taskID']
- my_queryset = DataProduct.objects.filter(taskID=taskid)
- logger.info("my_queryset retrieved")
- return my_queryset
-
-
-
# ---------- REST API views -----------
-# example: /atdb/status
-
-class StatusListViewAPI(generics.ListCreateAPIView):
- model = Status
- queryset = Status.objects.all()
- serializer_class = StatusSerializer
- pagination_class = DataProductsPagination
-
- filter_backends = (filters.DjangoFilterBackend,)
- filter_class = StatusFilter
-
-
-# example: /atdb/dataproducts/
-# calling this view serializes the dataproduct list in a REST API
-class DataProductListViewAPI(generics.ListCreateAPIView):
- model = DataProduct
- queryset = DataProduct.objects.all()
- serializer_class = DataProductSerializer
- pagination_class = DataProductsPagination
-
- # using the Django Filter Backend - https://django-filter.readthedocs.io/en/latest/index.html
- filter_backends = (filters.DjangoFilterBackend,)
- filter_class = DataProductFilter
-
-
-# example: /atdb/dataproducts/5/
-# calling this view serializes a dataproduct in the REST API
-class DataProductDetailsViewAPI(generics.RetrieveUpdateDestroyAPIView):
- model = DataProduct
- queryset = DataProduct.objects.all()
- serializer_class = DataProductSerializer
# example: /atdb/observations/
@@ -324,23 +251,6 @@ def ObservationSetStatus(request,pk,new_status,page):
return redirect('/atdb/?page='+page)
-# set the status of an observation and all its dataproducts to 'new_dps_status'
-# example: 'Validate DPS' button
-# /atdb/observations/<int:pk>/setstatus_dps/<new_dps_status>/<new_obs_status>/<page>
-def ObservationSetStatusDataProducts(request,pk,new_dps_status,new_obs_status,page):
- model = Observation
- observation = Observation.objects.get(pk=pk)
- observation.new_status = new_obs_status
- observation.save()
- taskid = observation.taskID
-
- dataproducts = DataProduct.objects.filter(taskID=taskid)
- for dataproduct in dataproducts:
- dataproduct.new_status = new_dps_status
- dataproduct.save()
-
- return redirect('/atdb/?page='+page)
-
# set the datawriter to which the observation will write to.
# /atdb/observations/<int:pk>/setdatawriter/<nr>/<page>
def SetDatawriter(request,pk,datawriter,page):
@@ -360,27 +270,9 @@ def SetDatawriter(request,pk,datawriter,page):
observation.save()
taskid = observation.taskID
- dataproducts = DataProduct.objects.filter(taskID=taskid)
- for dataproduct in dataproducts:
- dataproduct.data_location = new_data_location
- dataproduct.save()
-
return redirect('/atdb/?page='+page)
-# set the status of a dataproduct to 'new_status'
-# example: 'Validate', 'Skip' and 'Remove' buttons
-def DataProductSetStatusView(request,pk,new_status):
- model = DataProduct
- dataproduct = DataProduct.objects.get(pk=pk)
- dataproduct.new_status = new_status
- dataproduct.save()
-
- taskid = dataproduct.taskID
-
- return redirect('/atdb/task/'+taskid)
-
-
# get the next taskid based on starttime and what is currently in the database
#/atdb/get_next_taskid?timestamp=2019-04-05
class GetNextTaskIDView(generics.ListAPIView):
@@ -444,26 +336,3 @@ class GetNextObservationView(generics.ListAPIView):
})
-# add dataproducts as a batch
-# /atdb/post_dataproducts&taskid=190405034
-class PostDataproductsView(generics.CreateAPIView):
- queryset = DataProduct.objects.all()
- serializer_class = DataProductSerializer
- pagination_class = DataProductsPagination
-
- def post(self, request, *args, **kwargs):
-
- try:
- body_unicode = request.body.decode('utf-8')
- dataproducts = json.loads(body_unicode)
- except Exception as e:
- print(e)
- dataproducts = None
-
- taskID = algorithms.add_dataproducts(dataproducts)
-
- # return a response
- return Response({
- 'taskID': taskID,
- })
-