From 5264087f2139777bfc83c8ebf0c75bada07f4011 Mon Sep 17 00:00:00 2001
From: Jorrit Schaap <schaap@astron.nl>
Date: Thu, 8 Nov 2018 15:37:16 +0000
Subject: [PATCH] SW-44: found bug during performance testing. added special
test test_20181108_bugfix_resource_usages to t_radb.py to prove it. Renamed
t_radb.* to t_radb_functionality.* to make a better distinction with
t_radb_performance.py
---
.gitattributes | 6 +-
.../tests/CMakeLists.txt | 2 +-
.../tests/t_radb.run | 5 -
.../tests/t_radb.sh | 3 -
.../{t_radb.py => t_radb_functionality.py} | 100 ++++++++++++++++++
.../tests/t_radb_functionality.run | 5 +
.../tests/t_radb_functionality.sh | 3 +
7 files changed, 112 insertions(+), 12 deletions(-)
delete mode 100755 SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.run
delete mode 100755 SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.sh
rename SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/{t_radb.py => t_radb_functionality.py} (96%)
create mode 100755 SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_functionality.run
create mode 100755 SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_functionality.sh
diff --git a/.gitattributes b/.gitattributes
index cb380abc268..cf79b26ea33 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -4717,9 +4717,9 @@ SAS/ResourceAssignment/ResourceAssignmentDatabase/radbpglistener.ini -text
SAS/ResourceAssignment/ResourceAssignmentDatabase/radbpglistener.py -text
SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/CMakeLists.txt -text
SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/radb_common_testing.py -text
-SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.py -text
-SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.run -text
-SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.sh -text
+SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_functionality.py -text
+SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_functionality.run -text
+SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_functionality.sh -text
SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_performance.py -text
SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_performance.run -text
SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_performance.sh -text
diff --git a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/CMakeLists.txt b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/CMakeLists.txt
index 53b213e6335..7bc26cd3333 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/CMakeLists.txt
+++ b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/CMakeLists.txt
@@ -5,6 +5,6 @@ include(FindPythonModule)
find_python_module(testing.postgresql)
find_python_module(mock)
-lofar_add_test(t_radb)
+lofar_add_test(t_radb_functionality)
lofar_add_test(t_radb_performance)
diff --git a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.run b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.run
deleted file mode 100755
index 7895b4e9902..00000000000
--- a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.run
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-
-# Run the unit test
-source python-coverage.sh
-python_coverage_test "ResourceAssignmentDatabase/*" t_radb.py
diff --git a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.sh b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.sh
deleted file mode 100755
index d7aa7a173a2..00000000000
--- a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-./runctest.sh t_radb
diff --git a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.py b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_functionality.py
similarity index 96%
rename from SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.py
rename to SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_functionality.py
index 6a409ec2e9f..92bf83e1250 100755
--- a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_functionality.py
@@ -2479,6 +2479,10 @@ class ResourceAssignmentDatabaseTest(radb_common_testing.RADBCommonTest):
def test_obsolete_claims_are_removed(self):
'''Test if obsolete claims from finished tasks are removed automatically'''
+ # start with clean database
+ for spec in self.radb.getSpecifications():
+ self.radb.deleteSpecification(spec['id']) # cascades into tasks and claims
+
base_time = datetime.utcnow()
# round to current full hour (for readability in logging)
base_time = base_time - timedelta(minutes=base_time.minute, seconds=base_time.second,
@@ -2527,6 +2531,102 @@ class ResourceAssignmentDatabaseTest(radb_common_testing.RADBCommonTest):
# ...and now claims should remain
self.assertEqual(0, len(self.radb.getResourceClaims(task_ids=task_id)))
+ def test_20181108_bugfix_resource_usages(self):
+ # start with clean database
+ for spec in self.radb.getSpecifications():
+ self.radb.deleteSpecification(spec['id']) # cascades into tasks and claims
+
+ now = datetime.utcnow()
+ now -= timedelta(minutes=now.minute, seconds=now.second, microseconds=now.microsecond) # round to full hour
+ now += timedelta(hours=3)
+
+ NUM_CLAIMS = 2
+ NUM_CLAIMS_PER_RESOURCE = 2
+ RESOURCE_ID = 0
+ resource_max_cap = long(self.radb.get_resource_claimable_capacity(RESOURCE_ID, now, now))
+
+ task1_id = self.radb.insertSpecificationAndTask(1, 1, 'approved', 'observation',
+ now+timedelta(hours=1),
+ now + timedelta(hours=2),
+ 'content', 'CEP4')['task_id']
+ task1 = self.radb.getTask(task1_id)
+
+ claims1 = [{'resource_id': RESOURCE_ID,
+ 'starttime': task1['starttime'],
+ 'endtime': task1['endtime'],
+ 'status': 'tentative',
+ 'claim_size': resource_max_cap/NUM_CLAIMS_PER_RESOURCE}
+ for _ in range(NUM_CLAIMS)]
+
+ self.radb.insertResourceClaims(task1_id, claims1, 'foo', 1, 1)
+
+ # there should be NUM_CLAIMS tentative claims,
+ # and usage should be one 'block' from start->endtime
+ self.assertEqual(NUM_CLAIMS, len(self.radb.getResourceClaims(task_ids=task1_id, status='tentative')))
+ self.assertEqual([{'as_of_timestamp': task1['starttime'], 'usage': resource_max_cap },
+ {'as_of_timestamp': task1['endtime'], 'usage': 0L}],
+ self.radb.getResourceUsages(task1['starttime'], task1['endtime'], RESOURCE_ID)[RESOURCE_ID]['tentative'])
+
+ # update the claims to 'claimed' status
+ self.radb.updateResourceClaims(where_task_ids=task1_id, status='claimed')
+
+ # now, there should be zero tentative claims, but NUM_CLAIMS 'claimed' claims
+ # and usage should be one 'block' from start->endtime for claimed status
+ self.assertEqual(0, len(self.radb.getResourceClaims(task_ids=task1_id, status='tentative')))
+ self.assertEqual(NUM_CLAIMS, len(self.radb.getResourceClaims(task_ids=task1_id, status='claimed')))
+ # self.assertEqual([],
+ # self.radb.getResourceUsages(task1['starttime'], task1['endtime'], RESOURCE_ID)[RESOURCE_ID]['tentative'])
+ self.assertEqual([{'as_of_timestamp': task1['starttime'], 'usage': resource_max_cap },
+ {'as_of_timestamp': task1['endtime'], 'usage': 0L}],
+ self.radb.getResourceUsages(task1['starttime'], task1['endtime'], RESOURCE_ID)[RESOURCE_ID]['claimed'])
+
+ # finish the task...
+ self.radb.updateTask(task_id=task1_id, task_status='finished')
+
+ # ... as a result there should be no more claims, and usages should be clean
+ self.assertEqual(0, len(self.radb.getResourceClaims(task_ids=task1_id)))
+ self.assertEqual([],
+ self.radb.getResourceUsages(task1['starttime'], task1['endtime'], RESOURCE_ID)[RESOURCE_ID]['tentative'])
+ self.assertEqual([],
+ self.radb.getResourceUsages(task1['starttime'], task1['endtime'], RESOURCE_ID)[RESOURCE_ID]['claimed'])
+
+ # insert second task after the first one (not overlapping)
+ task2_id = self.radb.insertSpecificationAndTask(2, 2, 'approved', 'observation',
+ now + timedelta(hours=3),
+ now + timedelta(hours=4),
+ 'content', 'CEP4')['task_id']
+ task2 = self.radb.getTask(task2_id)
+
+ # and insert claims for the second task
+ claims2 = [{'resource_id': RESOURCE_ID,
+ 'starttime': task2['starttime'],
+ 'endtime': task2['endtime'],
+ 'status': 'tentative',
+ 'claim_size': resource_max_cap / NUM_CLAIMS_PER_RESOURCE}
+ for _ in range(NUM_CLAIMS)]
+
+ self.radb.insertResourceClaims(task2_id, claims2, 'foo', 1, 1)
+
+ # there should be NUM_CLAIMS tentative claims,
+ # and usage should be one 'block' from start->endtime
+ self.assertEqual(NUM_CLAIMS, len(self.radb.getResourceClaims(task_ids=task2_id, status='tentative')))
+ self.assertEqual([{'as_of_timestamp': task2['starttime'], 'usage': resource_max_cap },
+ {'as_of_timestamp': task2['endtime'], 'usage': 0L}],
+ self.radb.getResourceUsages(task2['starttime'], task2['endtime'], RESOURCE_ID)[RESOURCE_ID]['tentative'])
+
+ # update the claims to 'claimed' status
+ self.radb.updateResourceClaims(where_task_ids=task2_id, status='claimed')
+
+ # now, there should be zero tentative claims, but NUM_CLAIMS 'claimed' claims
+ # and usage should be one 'block' from start->endtime for claimed status
+ self.assertEqual(0, len(self.radb.getResourceClaims(task_ids=task2_id, status='tentative')))
+ self.assertEqual(NUM_CLAIMS, len(self.radb.getResourceClaims(task_ids=task2_id, status='claimed')))
+ # self.assertEqual([],
+ # self.radb.getResourceUsages(task2['starttime'], task2['endtime'], RESOURCE_ID)[RESOURCE_ID]['tentative'])
+ self.assertEqual([{'as_of_timestamp': task2['starttime'], 'usage': resource_max_cap },
+ {'as_of_timestamp': task2['endtime'], 'usage': 0L}],
+ self.radb.getResourceUsages(task2['starttime'], task2['endtime'], RESOURCE_ID)[RESOURCE_ID]['claimed'])
+
if __name__ == "__main__":
os.environ['TZ'] = 'UTC'
diff --git a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_functionality.run b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_functionality.run
new file mode 100755
index 00000000000..ef3148622ed
--- /dev/null
+++ b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_functionality.run
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Run the unit test
+source python-coverage.sh
+python_coverage_test "ResourceAssignmentDatabase/*" t_radb_functionality.py
diff --git a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_functionality.sh b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_functionality.sh
new file mode 100755
index 00000000000..86b69a5b65c
--- /dev/null
+++ b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_functionality.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+./runctest.sh t_radb_functionality
--
GitLab