Skip to content
Snippets Groups Projects
Commit c29b322f authored by Nico Vermaas's avatar Nico Vermaas
Browse files

removing dataproducts from model

parent 565880ab
No related branches found
No related tags found
No related merge requests found
Pipeline #7857 passed
Stage: build
Stage: deploy
Hide whitespace changes
Inline Side-by-side
Showing
with 43 additions and 375 deletions
README.md
+ 4
− 1
View file @ c29b322f
...@@ -36,11 +36,14 @@ Currently they are still mostly the original ATDB diagrams. They will be adapted ...@@ -36,11 +36,14 @@ Currently they are still mostly the original ATDB diagrams. They will be adapted
![](atdb/docs/ATDB-LDV%20-%20Deployment%20Diagram.png) ![](atdb/docs/ATDB-LDV%20-%20Deployment%20Diagram.png)
## Manual deploy in Docker ## CI/CD (semi) automatic deploy in Docker
For the `master' branch there is a CI/CD pipeline in place which builds and deploys the backend at https://sdc.astron.nl:5554/atdb/ For the `master' branch there is a CI/CD pipeline in place which builds and deploys the backend at https://sdc.astron.nl:5554/atdb/
The deploy step requires pushing the 'play' button in the gitlab pipelines section. The deploy step requires pushing the 'play' button in the gitlab pipelines section.
This is done to not have a mandatory deploy for every minor commmit. This is done to not have a mandatory deploy for every minor commmit.
## Manual deploy in Docker (alternative to CI/CD)
### initial ### initial
> cd ~/my_docker > cd ~/my_docker
......
atdb/taskdatabase/admin.py
+ 1
− 2
View file @ c29b322f
from django.contrib import admin from django.contrib import admin
from .models import Status, DataProduct, Observation from .models import Status, Observation
admin.site.register(Status) admin.site.register(Status)
admin.site.register(DataProduct)
admin.site.register(Observation) admin.site.register(Observation)
\ No newline at end of file
atdb/taskdatabase/migrations/0003_auto_20210114_1133.py 0 → 100644
+ 26
− 0
View file @ c29b322f
# Generated by Django 3.1.4 on 2021-01-14 10:33
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('taskdatabase', '0002_taskobject_metadata'),
]
operations = [
migrations.AlterField(
model_name='observation',
name='progress',
field=models.CharField(blank=True, default='0%', max_length=40, null=True),
),
migrations.AlterField(
model_name='taskobject',
name='task_type',
field=models.CharField(default='observation', max_length=20),
),
migrations.DeleteModel(
name='DataProduct',
),
]
atdb/taskdatabase/models.py
+ 1
− 44
View file @ c29b322f
...@@ -7,30 +7,12 @@ from django.db.models import Sum ...@@ -7,30 +7,12 @@ from django.db.models import Sum
datetime_format_string = '%Y-%m-%dT%H:%M:%SZ' datetime_format_string = '%Y-%m-%dT%H:%M:%SZ'
TASK_TYPE_OBSERVATION = 'observation' TASK_TYPE_OBSERVATION = 'observation'
TASK_TYPE_DATAPRODUCT = 'dataproduct'
TYPE_VISIBILITY = 'visibility' TYPE_VISIBILITY = 'visibility'
# common model functions
def get_sum_from_dataproduct_field(taskID, field):
"""
sum the values of a field for certain taskID. (used to sum total of dataproduct sizes)
:param taskID:
:param field:
:return:
"""
query = field + '__sum'
sum_value = DataProduct.objects.filter(taskID=taskID).aggregate(Sum(field))[query]
if sum_value == None:
sum_value = 0.0
return sum_value
"""
a base class of both Observation and Dataproducts
"""
class TaskObject(models.Model): class TaskObject(models.Model):
name = models.CharField(max_length=100, default="unknown") name = models.CharField(max_length=100, default="unknown")
task_type = models.CharField(max_length=20, default=TASK_TYPE_DATAPRODUCT) task_type = models.CharField(max_length=20, default=TASK_TYPE_OBSERVATION)
taskID = models.CharField('runId', db_index=True, max_length=30, blank=True, null=True) taskID = models.CharField('runId', db_index=True, max_length=30, blank=True, null=True)
creationTime = models.DateTimeField(default=datetime.utcnow, blank=True) creationTime = models.DateTimeField(default=datetime.utcnow, blank=True)
...@@ -99,31 +81,6 @@ class Observation(TaskObject): ...@@ -99,31 +81,6 @@ class Observation(TaskObject):
duration = 0 duration = 0
return duration return duration
@property
def size(self):
# sum the sizes of all dataproducts with this taskID. In Mb
size = get_sum_from_dataproduct_field(self.taskID,'size')
return size
def __str__(self): def __str__(self):
return str(self.taskID) return str(self.taskID)
class DataProduct(TaskObject):
# properties
filename = models.CharField(max_length=200, default="unknown")
description = models.CharField(max_length=255, default="unknown")
dataproduct_type = models.CharField('type', default=TYPE_VISIBILITY, max_length=50)
size = models.BigIntegerField(default=0)
# relationships
parent = models.ForeignKey(Observation, related_name='generated_dataproducts', on_delete=models.CASCADE, null=False)
# this translates a view-name (from urls.py) back to a url, to avoid hardcoded url's in the html templates
# bad : <td><a href="/atdb/observations/{{ observation.id }}/" target="_blank">{{ observation.taskID }} </a> </td>
# good: <td><a href="{{ observation.get_absolute_url }}" target="_blank">{{ observation.taskID }} </a> </td>
def get_absolute_url(self):
return reverse('dataproduct-detail-view-api', kwargs={'pk': self.pk})
def __str__(self):
return self.filename
atdb/taskdatabase/serializers.py
+ 2
− 33
View file @ c29b322f
from rest_framework import serializers from rest_framework import serializers
from .models import DataProduct, Observation, Status, TaskObject from .models import Observation, Status, TaskObject
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -11,37 +11,7 @@ class StatusSerializer(serializers.ModelSerializer): ...@@ -11,37 +11,7 @@ class StatusSerializer(serializers.ModelSerializer):
fields = ('id','name','timestamp','property_taskID','property_task_type') fields = ('id','name','timestamp','property_taskID','property_task_type')
class DataProductSerializer(serializers.ModelSerializer):
# this adds a 'parent_observation' list with hyperlinks to the DataProduct API.
# note that 'generatedByObservation' is not defined in the DataProduct model, but in the Observation model.
parent = serializers.PrimaryKeyRelatedField(
many=False,
queryset=Observation.objects.all(),
required=False,
)
status_history = serializers.StringRelatedField(
many=True,
required=False,
)
class Meta:
model = DataProduct
fields = ('id','task_type','name','filename','description',
'taskID','creationTime','size',
'my_status','new_status','status_history','parent',
'data_location','node','quality','metadata')
class ObservationSerializer(serializers.ModelSerializer): class ObservationSerializer(serializers.ModelSerializer):
# this adds a 'generated_dataproducts' list with hyperlinks to the Observation API.
# note that 'generated_dataproducts' is not defined in the DataProduct model, but comes from the related_field in Observation.parent.
generated_dataproducts = serializers.StringRelatedField(
many=True,
required=False,
)
status_history = serializers.StringRelatedField( status_history = serializers.StringRelatedField(
many=True, many=True,
...@@ -53,8 +23,7 @@ class ObservationSerializer(serializers.ModelSerializer): ...@@ -53,8 +23,7 @@ class ObservationSerializer(serializers.ModelSerializer):
fields = ('id','task_type', 'name','taskID', fields = ('id','task_type', 'name','taskID',
'field_name','field_ra','field_dec', 'field_name','field_ra','field_dec',
'creationTime','starttime','endtime', 'duration', 'size', 'creationTime','starttime','endtime', 'duration', 'size',
'my_status','new_status','status_history', 'my_status','new_status','status_history','generated_dataproducts',
'generated_dataproducts',
'data_location', 'node', 'data_location', 'node',
'skip_auto_ingest','observing_mode', 'skip_auto_ingest','observing_mode',
'quality','metadata','progress') 'quality','metadata','progress')
......
atdb/taskdatabase/services/algorithms.py
+ 1
− 43
View file @ c29b322f
...@@ -8,7 +8,7 @@ import time ...@@ -8,7 +8,7 @@ import time
import datetime import datetime
import logging import logging
from .common import timeit from .common import timeit
from ..models import Observation, DataProduct from ..models import Observation
DATE_FORMAT = "%Y-%m-%d" DATE_FORMAT = "%Y-%m-%d"
TIME_FORMAT = "%Y-%m-%d %H:%M:%SZ" TIME_FORMAT = "%Y-%m-%d %H:%M:%SZ"
...@@ -115,45 +115,3 @@ def get_next_observation(my_status, observing_mode, datawriter): ...@@ -115,45 +115,3 @@ def get_next_observation(my_status, observing_mode, datawriter):
return taskID,minutes_left return taskID,minutes_left
# /atdb/post_dataproducts
def add_dataproducts(dataproducts):
"""
:param taskID: taskid of the observation
:param dataproducts: json list of dataproducts to be added to the provided taskid
:return:
"""
taskID = None
dps = dataproducts.get('dps')
number_of_dataproducts = len(dps)
logger.info("add_dataproducts("+str(number_of_dataproducts)+")")
for dp in dps:
# get the common fields from the observation based on the given taskid
taskID = dp.get('taskID')
parent = Observation.objects.get(taskID=taskID)
parent_data_location = parent.data_location
node=dp.get('node',None)
new_status = dp.get('new_status','defined')
data_location = dp.get('data_dir',parent_data_location)
size = dp.get('size', 0)
myDataProduct = DataProduct(taskID=taskID,
node=node,
data_location=data_location,
filename=dp['filename'],
name=dp['filename'],
description=dp['filename'],
quality=dp['quality'],
metadata=dp['metadata'],
task_type='dataproduct',
new_status=new_status,
parent=parent,
size=size
)
logger.info('addding dataproduct: '+str(myDataProduct))
myDataProduct.save()
return taskID
atdb/taskdatabase/services/signals.py
+ 4
− 61
View file @ c29b322f
...@@ -5,7 +5,7 @@ from django.core.signals import request_started, request_finished ...@@ -5,7 +5,7 @@ from django.core.signals import request_started, request_finished
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.dispatch import receiver from django.dispatch import receiver
from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.models import ContentType
from taskdatabase.models import TaskObject, Observation, DataProduct, Status from taskdatabase.models import TaskObject, Observation, Status
from . import jobs from . import jobs
""" """
...@@ -26,22 +26,17 @@ def request_started_handler(sender, **kwargs): ...@@ -26,22 +26,17 @@ def request_started_handler(sender, **kwargs):
def request_finished_handler(sender, **kwargs): def request_finished_handler(sender, **kwargs):
logger.debug("signal : request_finished") logger.debug("signal : request_finished")
#--- Observation and DataProduct signals------------- #--- Task signals-------------
@receiver(pre_save, sender=Observation) @receiver(pre_save, sender=Observation)
def pre_save_observation_handler(sender, **kwargs): def pre_save_observation_handler(sender, **kwargs):
logger.info("SIGNAL : pre_save Observation(" + str(kwargs.get('instance')) + ")") logger.info("SIGNAL : pre_save Observation(" + str(kwargs.get('instance')) + ")")
handle_pre_save(sender, **kwargs) handle_pre_save(sender, **kwargs)
@receiver(pre_save, sender=DataProduct)
def pre_save_dataproduct_handler(sender, **kwargs):
logger.info("SIGNAL : pre_save DataProduct(" + str(kwargs.get('instance')) + ")")
handle_pre_save(sender, **kwargs)
def handle_pre_save(sender, **kwargs): def handle_pre_save(sender, **kwargs):
""" """
pre_save handler for both Observation and Dataproduct. Mainly to check status changes and dispatch jobs in needed. pre_save handler. Mainly to check status changes and dispatch jobs in needed.
:param (in) sender: The model class that sends the trigger :param (in) sender: The model class that sends the trigger
:param (in) kwargs: The instance of the object that sends the trigger. :param (in) kwargs: The instance of the object that sends the trigger.
""" """
...@@ -64,41 +59,6 @@ def handle_pre_save(sender, **kwargs): ...@@ -64,41 +59,6 @@ def handle_pre_save(sender, **kwargs):
myStatus = Status(name=new_status, taskObject=myTaskObject) myStatus = Status(name=new_status, taskObject=myTaskObject)
myStatus.save() myStatus.save()
#myTaskObject.new_status = None
# when an observation goes to valid, calculate its total size by counting its dataproducts
# if (myTaskObject.task_type == 'observation') and 'valid' in myTaskObject.new_status:
# # calculate total size
# dps = DataProduct.objects.filter(taskID=myTaskObject.taskID)
# size = 0
# for dp in dps:
# size = size + dp.size
#
# logger.info("total size of observation "+myTaskObject.taskID+ " = "+ str(size))
# # nv:11jun2019: this requires a database change first
# # myTaskObject.size = size
if (myTaskObject.task_type == 'observation'):
# convert the utc timestamp to a format that Django REST API understands
# in its GUI, otherwise null values will be put in when hitting PUT.
s,_ = str(myStatus.timestamp).split('.')
myTimestamp = datetime.datetime.strptime(s, "%Y-%m-%d %H:%M:%S")
if 'starting' in myTaskObject.new_status:
myTaskObject.timestamp_starting = myTimestamp
elif 'running' in myTaskObject.new_status:
myTaskObject.timestamp_running = myTimestamp
elif 'completing' in myTaskObject.new_status:
myTaskObject.timestamp_completing = myTimestamp
elif 'ingesting' in myTaskObject.new_status:
myTaskObject.timestamp_ingesting = myTimestamp
elif 'archived' in myTaskObject.new_status:
myTaskObject.timestamp_archived = myTimestamp
elif 'aborted' in myTaskObject.new_status:
myTaskObject.timestamp_aborted = myTimestamp
elif 'ingest_error' in myTaskObject.new_status:
myTaskObject.timestamp_ingest_error = myTimestamp
# temporarily disconnect the post_save handler to save the dataproduct (again) and avoiding recursion. # temporarily disconnect the post_save handler to save the dataproduct (again) and avoiding recursion.
# I don't use pre_save, because then the 'created' key is not available, which is the most handy way to # I don't use pre_save, because then the 'created' key is not available, which is the most handy way to
# determine if this dataproduct already exists. (I could also check the database, but this is easier). # determine if this dataproduct already exists. (I could also check the database, but this is easier).
...@@ -117,12 +77,6 @@ def post_save_observation_handler(sender, **kwargs): ...@@ -117,12 +77,6 @@ def post_save_observation_handler(sender, **kwargs):
handle_post_save(sender, **kwargs) handle_post_save(sender, **kwargs)
@receiver(post_save, sender=DataProduct)
def post_save_dataproduct_handler(sender, **kwargs):
#logger.info("SIGNAL : post_save DataProduct(" + str(kwargs.get('instance')) + ")")
handle_post_save(sender, **kwargs)
def handle_post_save(sender, **kwargs): def handle_post_save(sender, **kwargs):
""" """
pre_save handler for both Observation and Dataproduct. To create and write its initial status pre_save handler for both Observation and Dataproduct. To create and write its initial status
...@@ -132,7 +86,7 @@ def handle_post_save(sender, **kwargs): ...@@ -132,7 +86,7 @@ def handle_post_save(sender, **kwargs):
logger.info("handle_post_save("+str(kwargs.get('instance'))+")") logger.info("handle_post_save("+str(kwargs.get('instance'))+")")
myTaskObject = kwargs.get('instance') myTaskObject = kwargs.get('instance')
# CREATE NEW OBSERVATION / DATAPRODUCT # Create new task
if kwargs['created']: if kwargs['created']:
logger.info("save new "+str(myTaskObject.task_type)) logger.info("save new "+str(myTaskObject.task_type))
...@@ -143,12 +97,6 @@ def handle_post_save(sender, **kwargs): ...@@ -143,12 +97,6 @@ def handle_post_save(sender, **kwargs):
myStatus = Status(name=myTaskObject.new_status, taskObject=myTaskObject) myStatus = Status(name=myTaskObject.new_status, taskObject=myTaskObject)
myStatus.save() myStatus.save()
# if there already is an observation with this taskID, then add this dataproduct to it
if (myTaskObject.task_type == 'dataproduct'):
logger.info("update dataproduct parent = " + str(myTaskObject.taskID))
parent = Observation.objects.get(taskID=myTaskObject.taskID)
myTaskObject.parent=parent
# temporarily disconnect the post_save handler to save the dataproduct (again) and avoiding recursion. # temporarily disconnect the post_save handler to save the dataproduct (again) and avoiding recursion.
# I don't use pre_save, because then the 'created' key is not available, which is the most handy way to # I don't use pre_save, because then the 'created' key is not available, which is the most handy way to
# determine if this dataproduct already exists. (I could also check the database, but this is easier). # determine if this dataproduct already exists. (I could also check the database, but this is easier).
...@@ -159,14 +107,9 @@ def handle_post_save(sender, **kwargs): ...@@ -159,14 +107,9 @@ def handle_post_save(sender, **kwargs):
def connect_signals(): def connect_signals():
#logger.info("connect_signals") #logger.info("connect_signals")
pre_save.connect(pre_save_observation_handler, sender=Observation) pre_save.connect(pre_save_observation_handler, sender=Observation)
pre_save.connect(pre_save_dataproduct_handler, sender=DataProduct)
post_save.connect(post_save_observation_handler, sender=Observation) post_save.connect(post_save_observation_handler, sender=Observation)
post_save.connect(post_save_dataproduct_handler, sender=DataProduct)
def disconnect_signals(): def disconnect_signals():
#logger.info("disconnect_signals") #logger.info("disconnect_signals")
pre_save.disconnect(pre_save_observation_handler, sender=Observation) pre_save.disconnect(pre_save_observation_handler, sender=Observation)
pre_save.disconnect(pre_save_dataproduct_handler, sender=DataProduct)
post_save.disconnect(post_save_observation_handler, sender=Observation) post_save.disconnect(post_save_observation_handler, sender=Observation)
post_save.disconnect(post_save_dataproduct_handler, sender=DataProduct)
\ No newline at end of file
atdb/taskdatabase/templates/taskdatabase/index.html
+ 1
− 1
View file @ c29b322f
...@@ -45,7 +45,7 @@ ...@@ -45,7 +45,7 @@
</div> </div>
{% include 'taskdatabase/pagination.html' %} {% include 'taskdatabase/pagination.html' %}
</div> </div>
<p class="footer"> Version 1.0.0 (7 jan 2021 - 08:30) <p class="footer"> Version 1.0.0 (14 jan 2021 - 11:30)
<script type="text/javascript"> <script type="text/javascript">
(function(seconds) { (function(seconds) {
var refresh, var refresh,
......
atdb/taskdatabase/templates/taskdatabase/observations.html
+ 0
− 34
View file @ c29b322f
...@@ -40,14 +40,6 @@ ...@@ -40,14 +40,6 @@
{% endif %} {% endif %}
</td> </td>
<td> <td>
{% if observation.my_status == "complete" %}
<a href="{% url 'dataproducts-list-view' observation.taskID %}" class="btn btn-success btn-sm" role="button">DPS</a>
{% elif observation.my_status == "incomplete" or "error" in observation.my_status %}
<a href="{% url 'dataproducts-list-view' observation.taskID %}" class="btn btn-danger btn-sm" role="button">DPS</a>
{% else %}
<a href="{% url 'dataproducts-list-view' observation.taskID %}" class="btn btn-primary btn-sm" role="button">DPS</a>
{% endif %}
{% if observation.my_status == "ingesting" %} {% if observation.my_status == "ingesting" %}
<a href="https://alta.astron.nl/science/monitor" class="btn btn-primary btn-sm" target="_blank" role="button">Monitor</a> <a href="https://alta.astron.nl/science/monitor" class="btn btn-primary btn-sm" target="_blank" role="button">Monitor</a>
{% endif %} {% endif %}
...@@ -72,11 +64,6 @@ ...@@ -72,11 +64,6 @@
{% endif %} {% endif %}
{% endif %} {% endif %}
{% if observation.my_status == "incomplete" or observation.my_status == "defined" %}
<a href="{% url 'observation-dps-setstatus-view' observation.pk 'valid' 'valid' my_observations.number %}" class="btn btn-success btn-sm" role="button">Validate DPS</a>
<a href="{% url 'observation-dps-setstatus-view' observation.pk 'invalid' 'removed_invisible' my_observations.number %}" class="btn btn-danger btn-sm" role="button">Failed</a>
{% endif %}
{% if observation.my_status == "completed" or observation.my_status == "incomplete" %} {% if observation.my_status == "completed" or observation.my_status == "incomplete" %}
<a href="{% url 'observation-setstatus-view' observation.pk 'valid' my_observations.number %}" class="btn btn-primary btn-sm" role="button">Ready to Ingest</a> <a href="{% url 'observation-setstatus-view' observation.pk 'valid' my_observations.number %}" class="btn btn-primary btn-sm" role="button">Ready to Ingest</a>
{% endif %} {% endif %}
...@@ -85,34 +72,16 @@ ...@@ -85,34 +72,16 @@
<a href="{% url 'observation-setstatus-view' observation.pk 'completing' my_observations.number %}" class="btn btn-success btn-sm" role="button">Completing</a> <a href="{% url 'observation-setstatus-view' observation.pk 'completing' my_observations.number %}" class="btn btn-success btn-sm" role="button">Completing</a>
{% endif %} {% endif %}
{% if observation.my_status == "ingest error" or observation.my_status == "ingest aborted" %}
<a href="{% url 'observation-dps-setstatus-view' observation.pk 'defined' 'defined' my_observations.number %}" class="btn btn-primary btn-sm" role="button">Retry Ingest</a>
{% endif %}
{% if observation.my_status == "archived" or "error" in observation.my_status or "aborted" in observation.my_status or observation.my_status == "completed" or observation.my_status == "incomplete" %}
<a href="{% url 'observation-dps-setstatus-view' observation.pk 'removing' 'removing' my_observations.number %}" class="btn btn-warning btn-sm" role="button">Remove Data</a>
{% endif %}
{% if observation.my_status == "valid" %} {% if observation.my_status == "valid" %}
<a href="{% url 'observation-setstatus-view' observation.pk 'valid_priority' my_observations.number %}" class="btn btn-warning btn-sm" role="button">Priority Ingest</a> <a href="{% url 'observation-setstatus-view' observation.pk 'valid_priority' my_observations.number %}" class="btn btn-warning btn-sm" role="button">Priority Ingest</a>
<a href="{% url 'observation-setstatus-view' observation.pk 'completed' my_observations.number %}" class="btn btn-primary btn-sm" role="button">Cancel Ingest</a> <a href="{% url 'observation-setstatus-view' observation.pk 'completed' my_observations.number %}" class="btn btn-primary btn-sm" role="button">Cancel Ingest</a>
{% endif %} {% endif %}
{% if user.is_authenticated %}
{% if observation.my_status == "ingesting" %}
<a href="{% url 'observation-dps-setstatus-view' observation.pk 'ingest aborted' 'ingest aborted' my_observations.number %}" class="btn btn-danger btn-sm" role="button">Abort Ingest</a>
{% endif %}
{% endif %}
{% if observation.my_status == "valid_priority" %} {% if observation.my_status == "valid_priority" %}
<a href="{% url 'observation-setstatus-view' observation.pk 'valid' my_observations.number %}" class="btn btn-primary btn-sm" role="button">Cancel Priority</a> <a href="{% url 'observation-setstatus-view' observation.pk 'valid' my_observations.number %}" class="btn btn-primary btn-sm" role="button">Cancel Priority</a>
<a href="{% url 'observation-setstatus-view' observation.pk 'completed' my_observations.number %}" class="btn btn-primary btn-sm" role="button">Cancel Ingest</a> <a href="{% url 'observation-setstatus-view' observation.pk 'completed' my_observations.number %}" class="btn btn-primary btn-sm" role="button">Cancel Ingest</a>
{% endif %} {% endif %}
{% if observation.my_status == "incomplete" %}
<a href="{% url 'observation-dps-setstatus-view' observation.pk 'defined' 'defined' my_observations.number %}" class="btn btn-danger btn-sm" role="button">Reset</a>
{% endif %}
{% if observation.my_status == "archived" %} {% if observation.my_status == "archived" %}
<a href="{% url 'observation-setquality-view' observation.pk 'data_is_good' my_observations.number %}" class="btn btn-success btn-sm" role="button">Data is Good</a> <a href="{% url 'observation-setquality-view' observation.pk 'data_is_good' my_observations.number %}" class="btn btn-success btn-sm" role="button">Data is Good</a>
<a href="{% url 'observation-setquality-view' observation.pk 'data_is_bad' my_observations.number %}" class="btn btn-danger btn-sm" role="button">Data is Bad</a> <a href="{% url 'observation-setquality-view' observation.pk 'data_is_bad' my_observations.number %}" class="btn btn-danger btn-sm" role="button">Data is Bad</a>
...@@ -123,9 +92,6 @@ ...@@ -123,9 +92,6 @@
<a href="{% url 'observation-setquality-view' observation.pk 'data_is_bad' my_observations.number %}" class="btn btn-danger btn-sm" role="button">Data is Bad</a> <a href="{% url 'observation-setquality-view' observation.pk 'data_is_bad' my_observations.number %}" class="btn btn-danger btn-sm" role="button">Data is Bad</a>
{% endif %} {% endif %}
{% if observation.my_status == "removing" %}
<a href="{% url 'observation-dps-setstatus-view' observation.pk 'incomplete' 'incomplete' my_observations.number %}" class="btn btn-danger btn-sm" role="button">Cancel and Pray</a>
{% endif %}
</td> </td>
</tr> </tr>
</div> </div>
......
atdb/taskdatabase/urls.py
+ 1
− 23
View file @ c29b322f
...@@ -8,16 +8,7 @@ urlpatterns = [ ...@@ -8,16 +8,7 @@ urlpatterns = [
#path('', views.index, name='index'), #path('', views.index, name='index'),
path('', views.IndexView.as_view(), name='index'), path('', views.IndexView.as_view(), name='index'),
# ex: /atdb/task/180223003/
path('task/<taskID>/', views.DataProductsListView.as_view(), name='dataproducts-list-view'),
# --- REST API --- # --- REST API ---
# ex: /atdb/dataproducts/
path('dataproducts/', views.DataProductListViewAPI.as_view()),
# ex: /atdb/dataproducts/5/
path('dataproducts/<int:pk>/', views.DataProductDetailsViewAPI.as_view(),name='dataproduct-detail-view-api'),
# ex: /atdb/observations/ # ex: /atdb/observations/
path('observations/', views.ObservationListViewAPI.as_view()), path('observations/', views.ObservationListViewAPI.as_view()),
...@@ -27,9 +18,6 @@ urlpatterns = [ ...@@ -27,9 +18,6 @@ urlpatterns = [
# ex: /atdb/observations/5/ # ex: /atdb/observations/5/
path('observations/<int:pk>/', views.ObservationDetailsViewAPI.as_view(),name='observation-detail-view-api'), path('observations/<int:pk>/', views.ObservationDetailsViewAPI.as_view(),name='observation-detail-view-api'),
# ex: /atdb/status/
path('status/', views.StatusListViewAPI.as_view(),name='status-list-view-api'),
# --- custom requests --- # --- custom requests ---
# ex: /atdb/get_next_taskid?timestamp=2019-04-05 # ex: /atdb/get_next_taskid?timestamp=2019-04-05
path('get_next_taskid', path('get_next_taskid',
...@@ -41,21 +29,11 @@ urlpatterns = [ ...@@ -41,21 +29,11 @@ urlpatterns = [
views.GetNextObservationView.as_view(), views.GetNextObservationView.as_view(),
name='get-next-observation-view'), name='get-next-observation-view'),
# ex: /atdb/post_dataproducts&taskid=190405034
path('post_dataproducts',
views.PostDataproductsView.as_view(),
name='post-dataproducts-view'),
# --- controller resources --- # --- controller resources ---
path('observations/<int:pk>/setstatus/<new_status>/<page>', path('observations/<int:pk>/setstatus/<new_status>/<page>',
views.ObservationSetStatus, views.ObservationSetStatus,
name='observation-setstatus-view'), name='observation-setstatus-view'),
path('observations/<int:pk>/setstatus_dps/<new_dps_status>/<new_obs_status>/<page>',
views.ObservationSetStatusDataProducts,
name='observation-dps-setstatus-view'),
path('dataproducts/<int:pk>/setstatus/<new_status>',
views.DataProductSetStatusView,
name='dataproduct-setstatus-view'),
# set the quality field to 'good' or 'bad' (and transmit it to ALTA) # set the quality field to 'good' or 'bad' (and transmit it to ALTA)
path('observations/<int:pk>/setquality/<quality>/<page>', path('observations/<int:pk>/setquality/<quality>/<page>',
views.ObservationSetQuality, views.ObservationSetQuality,
......
atdb/taskdatabase/views.py
+ 2
− 133
View file @ c29b322f
...@@ -13,9 +13,9 @@ from django.template import loader ...@@ -13,9 +13,9 @@ from django.template import loader
from django.shortcuts import render, redirect from django.shortcuts import render, redirect
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from .models import DataProduct, Observation, Status from .models import Observation, Status
from django.db.models import Q from django.db.models import Q
from .serializers import DataProductSerializer, ObservationSerializer, StatusSerializer from .serializers import ObservationSerializer, StatusSerializer
from .forms import FilterForm from .forms import FilterForm
from .services import algorithms from .services import algorithms
...@@ -48,25 +48,6 @@ class ObservationFilter(filters.FilterSet): ...@@ -48,25 +48,6 @@ class ObservationFilter(filters.FilterSet):
'quality': ['exact', 'icontains'], 'quality': ['exact', 'icontains'],
} }
# example: /atdb/dataproducts?status__in=created,archived
class DataProductFilter(filters.FilterSet):
class Meta:
model = DataProduct
fields = {
'dataproduct_type': ['exact', 'in'], # ../dataproducts?dataProductType=IMAGE,VISIBILITY
'description': ['exact', 'icontains'],
'name': ['exact', 'icontains'],
'filename': ['exact', 'icontains'],
'taskID': ['exact', 'icontains'],
'creationTime': ['gt', 'lt', 'gte', 'lte', 'contains', 'exact'],
'size': ['gt', 'lt', 'gte', 'lte', 'contains', 'exact'],
'parent__taskID': ['exact', 'in', 'icontains'],
'my_status': ['exact', 'icontains', 'in'],
'data_location': ['exact', 'icontains'],
'node': ['exact', 'in'],
}
# example: has 1811130001 been on 'running?' # example: has 1811130001 been on 'running?'
# http://localhost:8000/atdb/status/?&taskID=181130001&name=running # http://localhost:8000/atdb/status/?&taskID=181130001&name=running
...@@ -175,61 +156,7 @@ def get_searched_observations(search): ...@@ -175,61 +156,7 @@ def get_searched_observations(search):
return observations return observations
# example: /atdb/task/180323003/
# https://docs.djangoproject.com/en/2.1/topics/class-based-views/generic-display/
# calling this view renders the dataproducts.html template in the GUI
# a custom pagination class to return more than the default 100 dataproducts
class DataProductsPagination(pagination.PageNumberPagination):
page_size = 10000
class DataProductsListView(ListView):
model = DataProduct
context_object_name = 'my_dataproducts_list'
template_name = 'taskdatabase/dataproducts.html'
pagination_class = DataProductsPagination
# override get_queryset to make a custom query on taskid
def get_queryset(self):
logger.info("DataProductsListView.get_queryset()")
taskid = self.kwargs['taskID']
my_queryset = DataProduct.objects.filter(taskID=taskid)
logger.info("my_queryset retrieved")
return my_queryset
# ---------- REST API views ----------- # ---------- REST API views -----------
# example: /atdb/status
class StatusListViewAPI(generics.ListCreateAPIView):
model = Status
queryset = Status.objects.all()
serializer_class = StatusSerializer
pagination_class = DataProductsPagination
filter_backends = (filters.DjangoFilterBackend,)
filter_class = StatusFilter
# example: /atdb/dataproducts/
# calling this view serializes the dataproduct list in a REST API
class DataProductListViewAPI(generics.ListCreateAPIView):
model = DataProduct
queryset = DataProduct.objects.all()
serializer_class = DataProductSerializer
pagination_class = DataProductsPagination
# using the Django Filter Backend - https://django-filter.readthedocs.io/en/latest/index.html
filter_backends = (filters.DjangoFilterBackend,)
filter_class = DataProductFilter
# example: /atdb/dataproducts/5/
# calling this view serializes a dataproduct in the REST API
class DataProductDetailsViewAPI(generics.RetrieveUpdateDestroyAPIView):
model = DataProduct
queryset = DataProduct.objects.all()
serializer_class = DataProductSerializer
# example: /atdb/observations/ # example: /atdb/observations/
...@@ -324,23 +251,6 @@ def ObservationSetStatus(request,pk,new_status,page): ...@@ -324,23 +251,6 @@ def ObservationSetStatus(request,pk,new_status,page):
return redirect('/atdb/?page='+page) return redirect('/atdb/?page='+page)
# set the status of an observation and all its dataproducts to 'new_dps_status'
# example: 'Validate DPS' button
# /atdb/observations/<int:pk>/setstatus_dps/<new_dps_status>/<new_obs_status>/<page>
def ObservationSetStatusDataProducts(request,pk,new_dps_status,new_obs_status,page):
model = Observation
observation = Observation.objects.get(pk=pk)
observation.new_status = new_obs_status
observation.save()
taskid = observation.taskID
dataproducts = DataProduct.objects.filter(taskID=taskid)
for dataproduct in dataproducts:
dataproduct.new_status = new_dps_status
dataproduct.save()
return redirect('/atdb/?page='+page)
# set the datawriter to which the observation will write to. # set the datawriter to which the observation will write to.
# /atdb/observations/<int:pk>/setdatawriter/<nr>/<page> # /atdb/observations/<int:pk>/setdatawriter/<nr>/<page>
def SetDatawriter(request,pk,datawriter,page): def SetDatawriter(request,pk,datawriter,page):
...@@ -360,27 +270,9 @@ def SetDatawriter(request,pk,datawriter,page): ...@@ -360,27 +270,9 @@ def SetDatawriter(request,pk,datawriter,page):
observation.save() observation.save()
taskid = observation.taskID taskid = observation.taskID
dataproducts = DataProduct.objects.filter(taskID=taskid)
for dataproduct in dataproducts:
dataproduct.data_location = new_data_location
dataproduct.save()
return redirect('/atdb/?page='+page) return redirect('/atdb/?page='+page)
# set the status of a dataproduct to 'new_status'
# example: 'Validate', 'Skip' and 'Remove' buttons
def DataProductSetStatusView(request,pk,new_status):
model = DataProduct
dataproduct = DataProduct.objects.get(pk=pk)
dataproduct.new_status = new_status
dataproduct.save()
taskid = dataproduct.taskID
return redirect('/atdb/task/'+taskid)
# get the next taskid based on starttime and what is currently in the database # get the next taskid based on starttime and what is currently in the database
#/atdb/get_next_taskid?timestamp=2019-04-05 #/atdb/get_next_taskid?timestamp=2019-04-05
class GetNextTaskIDView(generics.ListAPIView): class GetNextTaskIDView(generics.ListAPIView):
...@@ -444,26 +336,3 @@ class GetNextObservationView(generics.ListAPIView): ...@@ -444,26 +336,3 @@ class GetNextObservationView(generics.ListAPIView):
}) })
# add dataproducts as a batch
# /atdb/post_dataproducts&taskid=190405034
class PostDataproductsView(generics.CreateAPIView):
queryset = DataProduct.objects.all()
serializer_class = DataProductSerializer
pagination_class = DataProductsPagination
def post(self, request, *args, **kwargs):
try:
body_unicode = request.body.decode('utf-8')
dataproducts = json.loads(body_unicode)
except Exception as e:
print(e)
dataproducts = None
taskID = algorithms.add_dataproducts(dataproducts)
# return a response
return Response({
'taskID': taskID,
})
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment