diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py b/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py
index 5d3dbb98604d49b4a9aaf82dba99cc37bd3dcabe..4b979319b6d45e1b2ccd8faf40e8fdb075aea3ed 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py
@@ -1,4 +1,4 @@
-# Generated by Django 3.0.9 on 2021-01-29 16:33
+# Generated by Django 3.0.9 on 2021-02-11 13:51
from django.conf import settings
import django.contrib.postgres.fields
@@ -131,9 +131,6 @@ class Migration(migrations.Migration):
('size', models.BigIntegerField(help_text='Dataproduct size, in bytes. Used for accounting purposes. NULL if size is (yet) unknown (NULLable).', null=True)),
('feedback_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Dataproduct properties, as reported by the producing process.')),
],
- options={
- 'abstract': False,
- },
),
migrations.CreateModel(
name='DataproductArchiveInfo',
@@ -387,7 +384,6 @@ class Migration(migrations.Migration):
('private_data', models.BooleanField(default=True, help_text='True if data of this project is sensitive. Sensitive data is not made public.')),
('expert', models.BooleanField(default=False, help_text='Expert projects put more responsibility on the PI.')),
('filler', models.BooleanField(default=False, help_text='Use this project to fill up idle telescope time.')),
- ('archive_subdirectory', models.CharField(help_text='Subdirectory in which this project will store its data in the LTA. The full directory is constructed by prefixing with archive_location→directory.', max_length=1024)),
('auto_pin', models.BooleanField(default=False, help_text='True if the output_pinned flag of tasks in this project should be set True on creation.')),
],
options={
@@ -423,6 +419,12 @@ class Migration(migrations.Migration):
('value', models.FloatField(help_text='Resource Quota value')),
],
),
+ migrations.CreateModel(
+ name='ProjectQuotaArchiveLocation',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ],
+ ),
migrations.CreateModel(
name='ProjectRole',
fields=[
@@ -1205,6 +1207,16 @@ class Migration(migrations.Migration):
name='specifications_template',
field=models.ForeignKey(help_text='Schema used for specifications_doc.', on_delete=django.db.models.deletion.CASCADE, to='tmssapp.ReservationTemplate'),
),
+ migrations.AddField(
+ model_name='projectquotaarchivelocation',
+ name='archive_location',
+ field=models.ForeignKey(help_text='Location of an archive LTA cluster.', on_delete=django.db.models.deletion.PROTECT, to='tmssapp.Filesystem'),
+ ),
+ migrations.AddField(
+ model_name='projectquotaarchivelocation',
+ name='project_quota',
+ field=models.ForeignKey(help_text='Project to wich this quota belongs.', on_delete=django.db.models.deletion.PROTECT, related_name='project_quota', to='tmssapp.ProjectQuota'),
+ ),
migrations.AddField(
model_name='projectquota',
name='project',
@@ -1240,11 +1252,6 @@ class Migration(migrations.Migration):
name='PUT',
field=models.ManyToManyField(blank=True, related_name='can_PUT', to='tmssapp.ProjectRole'),
),
- migrations.AddField(
- model_name='project',
- name='archive_location',
- field=models.ForeignKey(help_text='Ingest data to this LTA cluster only (NULLable). NULL means: no preference.', null=True, on_delete=django.db.models.deletion.PROTECT, to='tmssapp.Filesystem'),
- ),
migrations.AddField(
model_name='project',
name='cycles',
@@ -1500,8 +1507,8 @@ class Migration(migrations.Migration):
model_name='dataproductarchiveinfo',
index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_dat_tags_ebf2ef_gin'),
),
- migrations.AddIndex(
+ migrations.AddConstraint(
model_name='dataproduct',
- index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_dat_tags_5932a3_gin'),
+ constraint=models.UniqueConstraint(fields=('directory', 'filename'), name='dataproduct_unique_path'),
),
]
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/models/specification.py b/SAS/TMSS/backend/src/tmss/tmssapp/models/specification.py
index 49754c503cb1d43976076d337ec735753280c475..44ecd759f2e655652fabe4a6f922af9665826b13 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/models/specification.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/models/specification.py
@@ -269,41 +269,13 @@ class Project(NamedCommonPK):
filler = BooleanField(default=False, help_text='Use this project to fill up idle telescope time.')
project_category = ForeignKey('ProjectCategory', null=True, on_delete=PROTECT, help_text='Project category.')
period_category = ForeignKey('PeriodCategory', null=True, on_delete=PROTECT, help_text='Period category.')
- archive_location = ForeignKey('Filesystem', null=True, on_delete=PROTECT, help_text='Ingest data to this LTA cluster only (NULLable). NULL means: no preference.')
- archive_subdirectory = CharField(max_length=1024, help_text='Subdirectory in which this project will store its data in the LTA. The full directory is constructed by prefixing with archive_location→directory.')
auto_pin = BooleanField(default=False, help_text='True if the output_pinned flag of tasks in this project should be set True on creation.')
- def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- if self.archive_subdirectory and not self.archive_subdirectory.endswith('/'):
- raise ValueError('directory value must end with a trailing slash!')
- if self.archive_subdirectory and self.archive_subdirectory.startswith('/'):
- raise ValueError('directory value must be a relative path (and not start with a slash)!')
-
- super().save(force_insert, force_update, using, update_fields)
-
- # JK, 29/07/20 - after discussion with Sander, it turns out that the ticket TMSS-277 was a misunderstanding.
- # 'default' does not refer to 'default values' that are supposed to be filled in by the backend.
- # It was meant to be 'resource_types displayed in the frontend by default', where the other resource_types are
- # optionally added to the set of quota. These can then be customized in the frontend and are created by the
- # frontend in the backend, but no quota are intended to be added automatically. So nothing is really needed in
- # the backend for this (apart from the set of predefined resource_types).
- # There was some open question on whether there may be a required subset of quota that have to be enforced. So
- # I'll leave this in for now, until that question is cleared up.
- #
- # # also create default project quotas when projects are created
- # def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- # creating = self._state.adding # True on create, False on update
- # super().save(force_insert, force_update, using, update_fields)
- # if creating:
- # # todo: review these defaults for being sensible
- # ProjectQuota.objects.create(resource_type=ResourceType.objects.get(name="lta_storage"), value=1024^4, project=self)
- # ProjectQuota.objects.create(resource_type=ResourceType.objects.get(name="cep_storage"), value=1024^4, project=self)
- # ProjectQuota.objects.create(resource_type=ResourceType.objects.get(name="cep_processing_time"), value=60*60*24, project=self)
- # ProjectQuota.objects.create(resource_type=ResourceType.objects.get(name="lofar_observing_time"), value=60*60*24, project=self)
- # ProjectQuota.objects.create(resource_type=ResourceType.objects.get(name="lofar_observing_time_prio_a"), value=60*60*12, project=self)
- # ProjectQuota.objects.create(resource_type=ResourceType.objects.get(name="lofar_observing_time_prio_b"), value=60*60*12, project=self)
- # ProjectQuota.objects.create(resource_type=ResourceType.objects.get(name="support_time"), value=60*60*6, project=self)
- # ProjectQuota.objects.create(resource_type=ResourceType.objects.get(name="number_of_triggers"), value=42, project=self)
+ @cached_property
+ def duration(self) -> datetime.timedelta:
+ '''return the overall duration of all tasks of this scheduling unit
+ '''
+ return self.relative_stop_time - self.relative_start_time
class ProjectQuota(Model):
@@ -312,6 +284,28 @@ class ProjectQuota(Model):
resource_type = ForeignKey('ResourceType', on_delete=PROTECT, help_text='Resource type.') # protected to avoid accidents
+class ProjectQuotaArchiveLocation(Model):
+ project_quota = ForeignKey('ProjectQuota', null=False, related_name="project_quota", on_delete=PROTECT, help_text='Project to wich this quota belongs.')
+ archive_location = ForeignKey('Filesystem', null=False, on_delete=PROTECT, help_text='Location of an archive LTA cluster.')
+
+ def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
+ if self.project_quota.resource_type.quantity.value != Quantity.Choices.BYTES.value:
+ raise ValueError("A ProjectQuotaArchiveLocation should have its project_quota defined in a 'bytes' quantity")
+ if not self.archive_location.cluster.archive_site:
+ raise ValueError("The archive_location of a ProjectQuotaArchiveLocation should be an archive site")
+ super().save(force_insert, force_update, using, update_fields)
+
+ @cached_property
+ def archive_subdirectory(self) -> str:
+ '''return name of the subdirectory where this project is stored. By default and convention this the the lower_case project name'''
+ return self.project_quota.project.name.lower()
+
+ @cached_property
+ def full_archive_uri(self) -> str:
+ '''return full uri where this project is stored. By default and convention this the the lower_case project name'''
+ return "%s/%s/" % (self.archive_location.directory.rstrip('/'), self.archive_subdirectory)
+
+
class ResourceType(NamedCommonPK):
quantity = ForeignKey('Quantity', null=False, on_delete=PROTECT, help_text='The quantity of this resource type.')
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/populate.py b/SAS/TMSS/backend/src/tmss/tmssapp/populate.py
index 1bab1d23a677f2a114c42c27ef601f41316da879..6eae000487adfd897cdfcc6a9491f9f343cf4d54 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/populate.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/populate.py
@@ -175,6 +175,10 @@ def populate_projects(apps, schema_editor):
# for convenience, create a schedulingset for each project
models.SchedulingSet.objects.create(**SchedulingSet_test_data(name="Test Scheduling Set", project=tmss_project))
+ project_quota = ProjectQuota.objects.create(project=tmss_project, value=1e12, resource_type=ResourceType.objects.get(name="lta_storage"))
+ sara_fs = Filesystem.objects.get(name="Lofar Storage (SARA)")
+ models.ProjectQuotaArchiveLocation.objects.create(project_quota=project_quota, archive_location=sara_fs)
+
def populate_resources(apps, schema_editor):
bytes_q = Quantity.objects.get(value=Quantity.Choices.BYTES.value)
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/serializers/specification.py b/SAS/TMSS/backend/src/tmss/tmssapp/serializers/specification.py
index d281087554ecbd86d4dd6c60753d9c977fab084e..6da77622176c08eec700950093c5a214a36c80bd 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/serializers/specification.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/serializers/specification.py
@@ -184,6 +184,18 @@ class ProjectQuotaSerializer(DynamicRelationalHyperlinkedModelSerializer):
extra_fields = ['resource_type']
+class ProjectQuotaArchiveLocationSerializer(DynamicRelationalHyperlinkedModelSerializer):
+ queryset = models.ProjectQuotaArchiveLocation.objects.all()
+
+ # performance boost: select the related models in a single db call.
+ queryset = queryset.select_related('project_quota', 'archive_location')
+
+ class Meta:
+ model = models.ProjectQuotaArchiveLocation
+ fields = '__all__'
+ extra_fields = ['archive_subdirectory', 'full_archive_uri']
+
+
class ResourceTypeSerializer(DynamicRelationalHyperlinkedModelSerializer):
class Meta:
model = models.ResourceType
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py
index b554ffdcef5395a2362c317718d75dc0946de8bf..506b44b8b18dc9b3969305d0c3ac1c326da23f54 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py
@@ -296,7 +296,22 @@ class ProjectQuotaViewSet(LOFARViewSet):
return queryset.filter(project=project)
return queryset
-
+
+
+class ProjectQuotaArchiveLocationViewSet(LOFARViewSet):
+ queryset = models.ProjectQuotaArchiveLocation.objects.all()
+ serializer_class = serializers.ProjectQuotaArchiveLocationSerializer
+
+ def get_queryset(self):
+ queryset = models.ProjectQuotaArchiveLocation.objects.all()
+
+ # query by project
+ project = self.request.query_params.get('project', None)
+ if project is not None:
+ return queryset.filter(project=project)
+
+ return queryset
+
class ResourceTypeViewSet(LOFARViewSet):
queryset = models.ResourceType.objects.all()
diff --git a/SAS/TMSS/backend/src/tmss/urls.py b/SAS/TMSS/backend/src/tmss/urls.py
index 8158bfc0267b5411129ebb7811160712f6a1b8f2..039b531a658e3bed589f131860f3d1193bfc3b39 100644
--- a/SAS/TMSS/backend/src/tmss/urls.py
+++ b/SAS/TMSS/backend/src/tmss/urls.py
@@ -149,6 +149,7 @@ router.register(r'cycle_quota', viewsets.CycleQuotaViewSet)
router.register(r'project', viewsets.ProjectViewSet)
router.register(r'resource_type', viewsets.ResourceTypeViewSet)
router.register(r'project_quota', viewsets.ProjectQuotaViewSet)
+router.register(r'project_quota_archive_location', viewsets.ProjectQuotaArchiveLocationViewSet)
router.register(r'setting', viewsets.SettingViewSet)
router.register(r'reservation', viewsets.ReservationViewSet)