diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/models/scheduling.py b/SAS/TMSS/backend/src/tmss/tmssapp/models/scheduling.py
index 06edc09e9e6a3782f71f7390ed390dc0ee4d6266..39165fae8e5bc0893f773be2a2116f221c1b461c 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/models/scheduling.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/models/scheduling.py
@@ -417,7 +417,7 @@ class Filesystem(NamedCommon):
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
if self.directory and not self.directory.endswith('/'):
- raise ValueError('directory value must end with a trailing slash!') # todo: ...and needs to start with slash?
+ self.directory += '/'
super().save(force_insert, force_update, using, update_fields)
diff --git a/SAS/TMSS/backend/test/t_complex_serializers.sh b/SAS/TMSS/backend/test/t_complex_serializers.sh
index 63ecb0c21ef0f6ae6d8ccc6d0c4d88d826a4189d..a1ebf8e5b3be2a3357f310b5baa582316304e21b 100755
--- a/SAS/TMSS/backend/test/t_complex_serializers.sh
+++ b/SAS/TMSS/backend/test/t_complex_serializers.sh
@@ -1,3 +1,3 @@
#!/bin/sh
-./runctest.sh t_scheduling
\ No newline at end of file
+./runctest.sh t_complex_serializers
\ No newline at end of file
diff --git a/SAS/TMSS/backend/test/t_scheduling.py b/SAS/TMSS/backend/test/t_scheduling.py
index 73892be8d0b9f8c95f1fa9aa0c0e28a2f39b85ef..c18ab360dc3f0e0d68f4d80ba558cef0ec2dca2a 100755
--- a/SAS/TMSS/backend/test/t_scheduling.py
+++ b/SAS/TMSS/backend/test/t_scheduling.py
@@ -555,14 +555,26 @@ class TestWithUC1Specifications(unittest.TestCase):
# clean all specs/tasks/claims in RADB (cascading delete)
for spec in tmss_test_env.ra_test_environment.radb.getSpecifications():
tmss_test_env.ra_test_environment.radb.deleteSpecification(spec['id'])
- # Set subtask back to 'defined', start_time to now (and no stoptime)
+ # Unschedule subtask, setting it back to 'defined', removing all dataproducts.
for tb in self.task_blueprints:
for subtask in tb.subtasks.all():
- subtask.state = models.SubtaskState.objects.get(value="defined")
+ if subtask.state.value == SubtaskState.Choices.SCHEDULED.value:
+ unschedule_subtask(subtask)
+ if subtask.state.value == SubtaskState.Choices.ERROR.value:
+ subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.DEFINED.value)
+
+ for output in subtask.outputs.all():
+ # delete all transforms (the producers of the output dataproducts), and the the dataproducts themselves
+ output.dataproducts.all().select_related('producers').delete()
+ output.dataproducts.all().delete()
+
+ # start_time to now (and no stoptime)
subtask.stop_time = None
subtask.start_time = datetime.utcnow()
subtask.save()
+
+
def _schedule_subtask_with_failure(self, station_reserved):
with tmss_test_env.create_tmss_client() as client:
with self.assertRaises(Exception) as context:
diff --git a/SAS/TMSS/backend/test/t_tmssapp_scheduling_django_API.py b/SAS/TMSS/backend/test/t_tmssapp_scheduling_django_API.py
index fe6cf69a5aa12bf6df9dd788a1f4b4de20732b87..682f22659885f52e3a3632ab288861efa19b3b5e 100755
--- a/SAS/TMSS/backend/test/t_tmssapp_scheduling_django_API.py
+++ b/SAS/TMSS/backend/test/t_tmssapp_scheduling_django_API.py
@@ -448,14 +448,9 @@ class FilesystemTest(unittest.TestCase):
self.assertLess(before, entry.updated_at)
self.assertGreater(after, entry.updated_at)
- def test_Filesystem_raises_ValueError_on_invalid_directory_name(self):
-
- # setup
- test_data = Filesystem_test_data(directory="/no/trailing/slash")
-
- # assert
- with self.assertRaises(ValueError):
- entry = models.Filesystem.objects.create(**test_data)
+ def test_Filesystem_appends_trailing_slash_to_dirname(self):
+ fs = models.Filesystem.objects.create(**Filesystem_test_data(directory="/no/trailing/slash"))
+ self.assertTrue(fs.directory.endswith('/'))
class ClusterTest(unittest.TestCase):
diff --git a/SAS/TMSS/backend/test/t_tmssapp_specification_django_API.py b/SAS/TMSS/backend/test/t_tmssapp_specification_django_API.py
index c07159c9116256563cbbabd1212a679604ecac75..fb612a0416d7b17b3e18e257a5775c6c501be218 100755
--- a/SAS/TMSS/backend/test/t_tmssapp_specification_django_API.py
+++ b/SAS/TMSS/backend/test/t_tmssapp_specification_django_API.py
@@ -341,18 +341,18 @@ class ProjectTest(unittest.TestCase):
self.assertLess(before, entry.updated_at)
self.assertGreater(after, entry.updated_at)
- def test_Project_raises_ValueError_on_invalid_archive_subdirectory_name(self):
-
+class FileSystemTest(unittest.TestCase):
+ def test_directory_always_ends_with_slash(self):
# setup
- test_data_1 = Project_test_data(archive_subdirectory="no/trailing/slash")
- test_data_2 = Project_test_data(archive_subdirectory="/with/leading/slash/")
+ test_data_1 = Filesystem_test_data(directory="/no/trailing/slash")
+ test_data_2 = Filesystem_test_data(directory="/with/trailing/slash/")
# assert
- with self.assertRaises(ValueError):
- entry = models.Project.objects.create(**test_data_1)
+ entry1 = models.Filesystem.objects.create(**test_data_1)
+ self.assertTrue(entry1.directory.endswith('/'))
- with self.assertRaises(ValueError):
- entry = models.Project.objects.create(**test_data_2)
+ entry2 = models.Filesystem.objects.create(**test_data_2)
+ self.assertTrue(entry2.directory.endswith('/'))
class ProjectQuotaTest(unittest.TestCase):
@@ -366,6 +366,36 @@ class ProjectQuotaTest(unittest.TestCase):
models.ProjectQuota.objects.create(**test_data)
+class ProjectQuotaArchiveLocationTest(unittest.TestCase):
+ def test_archive_location_must_be_archive_site(self):
+ with self.assertRaises(ValueError):
+ test_data = dict(ProjectQuotaArchiveLocation_test_data(archive_location=models.Filesystem.objects.create(**Filesystem_test_data(cluster=models.Cluster.objects.create(**Cluster_test_data(archive_site=False))))))
+ models.ProjectQuotaArchiveLocation.objects.create(**test_data)
+
+ test_data = dict(ProjectQuotaArchiveLocation_test_data(archive_location=models.Filesystem.objects.create(**Filesystem_test_data(cluster=models.Cluster.objects.create(**Cluster_test_data(archive_site=True))))))
+ models.ProjectQuotaArchiveLocation.objects.create(**test_data)
+
+ def test_quota_must_be_bytes(self):
+ with self.assertRaises(ValueError):
+ test_data = dict(ProjectQuotaArchiveLocation_test_data(project_quota = models.ProjectQuota.objects.create(**ProjectQuota_test_data(resource_type=models.ResourceType.objects.create(**ResourceType_test_data(quantity=models.Quantity.objects.get(value=models.Quantity.Choices.NUMBER.value)))))))
+ models.ProjectQuotaArchiveLocation.objects.create(**test_data)
+
+ test_data = dict(ProjectQuotaArchiveLocation_test_data(project_quota=models.ProjectQuota.objects.create(**ProjectQuota_test_data(resource_type=models.ResourceType.objects.create(**ResourceType_test_data(quantity=models.Quantity.objects.get(value=models.Quantity.Choices.BYTES.value)))))))
+ models.ProjectQuotaArchiveLocation.objects.create(**test_data)
+
+ def test_uri(self):
+ PROJECT_NAME = "TestProject"
+ SURL = "srm://my.srm.site:1234/path/to/data"
+ project = models.Project.objects.create(**Project_test_data(name=PROJECT_NAME))
+ archive_location = models.Filesystem.objects.create(**Filesystem_test_data(directory=SURL))
+ quota = models.ProjectQuota.objects.create(**ProjectQuota_test_data(project=project))
+
+ pqal = models.ProjectQuotaArchiveLocation.objects.create(**ProjectQuotaArchiveLocation_test_data(project_quota=quota, archive_location=archive_location))
+ self.assertEqual(PROJECT_NAME.lower(), pqal.archive_subdirectory)
+ self.assertEqual(SURL+'/'+PROJECT_NAME.lower()+'/', pqal.full_archive_uri)
+
+
+
class SchedulingSetTest(unittest.TestCase):
def test_SchedulingSet_gets_created_with_correct_creation_timestamp(self):
diff --git a/SAS/TMSS/backend/test/tmss_test_data_django_models.py b/SAS/TMSS/backend/test/tmss_test_data_django_models.py
index 8035afbcfc136067fe9d5da53e8362c4c4063742..7d3e065be36e697dd96a80777a6a9c9044fce46d 100644
--- a/SAS/TMSS/backend/test/tmss_test_data_django_models.py
+++ b/SAS/TMSS/backend/test/tmss_test_data_django_models.py
@@ -117,7 +117,7 @@ def Cycle_test_data() -> dict:
"start": datetime.utcnow().isoformat(),
"stop": datetime.utcnow().isoformat()}
-def Project_test_data(name: str=None, priority_rank: int = 1, archive_subdirectory="my_project/", auto_pin=False) -> dict:
+def Project_test_data(name: str=None, priority_rank: int = 1, auto_pin=False) -> dict:
if name is None:
name = 'my_project_' + str(uuid.uuid4())
@@ -132,24 +132,47 @@ def Project_test_data(name: str=None, priority_rank: int = 1, archive_subdirecto
"private_data": True,
"expert": True,
"filler": False,
- "archive_subdirectory": archive_subdirectory,
"auto_pin": auto_pin}
-def ResourceType_test_data() -> dict:
+def ResourceType_test_data(quantity: models.Quantity=None) -> dict:
+ if quantity is None:
+ quantity = models.Quantity.objects.get(value=models.Quantity.Choices.BYTES.value)
+
return {
"tags": [],
"description": 'my description ' + str(uuid.uuid4()),
"name": 'my_resource_type_' + str(uuid.uuid4()),
- "quantity": models.Quantity.objects.get(value=models.Quantity.Choices.NUMBER.value)
+ "quantity": quantity
}
-def ProjectQuota_test_data() -> dict:
- return {
- "value": '1000',
- "project": models.Project.objects.create(**Project_test_data()),
- "resource_type": models.ResourceType.objects.create(**ResourceType_test_data())
+
+def ProjectQuota_test_data(value: int=1000, project: models.Project=None, resource_type: models.ResourceType=None) -> dict:
+ if project is None:
+ project = models.Project.objects.create(**Project_test_data())
+
+ if resource_type is None:
+ resource_type = models.ResourceType.objects.create(**ResourceType_test_data())
+
+ return {
+ "value": value,
+ "project": project,
+ "resource_type": resource_type
}
-
+
+
+def ProjectQuotaArchiveLocation_test_data(project_quota: models.ProjectQuota=None, archive_location: models.Filesystem=None) -> dict:
+ if project_quota is None:
+ project_quota = models.ProjectQuota.objects.create(**ProjectQuota_test_data())
+
+ if archive_location is None:
+ archive_location = models.Filesystem.objects.create(**Filesystem_test_data())
+
+ return {
+ "project_quota": project_quota,
+ "archive_location": archive_location
+ }
+
+
def SchedulingSet_test_data(name="my_scheduling_set", project: models.Project=None) -> dict:
if project is None:
project = models.Project.objects.create(**Project_test_data())
@@ -404,8 +427,8 @@ def Subtask_test_data(task_blueprint: models.TaskBlueprint=None, subtask_templat
"global_identifier": models.SIPidentifier.objects.create(source="TMSS")}
def Dataproduct_test_data(producer: models.SubtaskOutput=None,
- filename: str="my_file.ext",
- directory: str="/data/test-projects",
+ filename: str=None,
+ directory: str=None,
dataformat: models.Dataformat=None,
datatype: models.Datatype=None,
specifications_doc: object=None,
@@ -413,6 +436,12 @@ def Dataproduct_test_data(producer: models.SubtaskOutput=None,
feedback_doc: object = None,
feedback_template: models.DataproductFeedbackTemplate = None) -> dict:
+ if filename is None:
+ filename = "my_file_%s.ext" % uuid.uuid4()
+
+ if directory is None:
+ directory = "/tmp/test_data/%s/" % uuid.uuid4()
+
if producer is None:
producer = models.SubtaskOutput.objects.create(**SubtaskOutput_test_data())
@@ -466,16 +495,19 @@ def DataproductTransform_test_data() -> dict:
"identity": True,
"tags": ['tmss', 'testing']}
-def Filesystem_test_data(directory="/") -> dict:
+def Filesystem_test_data(directory: str="/", cluster: models.Cluster=None) -> dict:
+ if cluster is None:
+ cluster = models.Cluster.objects.create(**Cluster_test_data())
+
return {"capacity": 1111111111,
- "cluster": models.Cluster.objects.create(**Cluster_test_data()),
+ "cluster": cluster,
"directory": directory,
"tags": ['tmss', 'testing']}
-def Cluster_test_data(name="default cluster") -> dict:
+def Cluster_test_data(name: str="default cluster", archive_site: bool=True) -> dict:
return {"name": name,
"location": "upstairs",
- "archive_site": True,
+ "archive_site": archive_site,
"tags": ['tmss', 'testing']}
def DataproductArchiveInfo_test_data() -> dict:
diff --git a/SAS/TMSS/backend/test/tmss_test_data_rest.py b/SAS/TMSS/backend/test/tmss_test_data_rest.py
index bc90c5ac03346f5226a834a7922545223a37b81d..44a4cf68156c45053fe0624cacfc7139b1fdb12a 100644
--- a/SAS/TMSS/backend/test/tmss_test_data_rest.py
+++ b/SAS/TMSS/backend/test/tmss_test_data_rest.py
@@ -251,7 +251,6 @@ class TMSSRESTTestDataCreator():
"can_trigger": False,
"private_data": True,
"cycles": cycle_urls,
- "archive_subdirectory": 'my_project/',
"auto_pin": auto_pin}
@property