diff --git a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.py b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.py index c14737bf39c279c4c6391f5c8bef0cd490276cce..ad085d3ccb4182d7ced9fd0f037130fdabb819f3 100755 --- a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.py +++ b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.py @@ -1011,8 +1011,9 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): def test_task_and_claim_conflicts(self): # TODO: split up once the test setup is faster (not creating a new db for each test method) # for testing purposous let's give CEP4 storage a total size of 100 - self.assertTrue(self.radb.updateResourceAvailability(117, available_capacity=100, total_capacity=100)) - self.assertEqual(100, self.radb.getResources(117, include_availability=True)[0]['total_capacity']) + cep4_id = 117 + self.assertTrue(self.radb.updateResourceAvailability(cep4_id, available_capacity=100, total_capacity=100)) + self.assertEqual(100, self.radb.getResources(cep4_id, include_availability=True)[0]['total_capacity']) now = datetime.utcnow() now -= timedelta(minutes=now.minute, seconds=now.second, microseconds=now.microsecond) # round to full hour @@ -1026,7 +1027,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertTrue(task1) self.assertEqual(task_id1, task1['id']) - t1_claim1 = { 'resource_id': 117, + t1_claim1 = { 'resource_id': cep4_id, 'starttime': task1['starttime'], 'endtime': task1['endtime'], 'status': 'tentative', @@ -1051,7 +1052,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual(value, t1_claims[0][key]) # try to insert a claim with the wrong (already 'claimed') status. Should rollback, and return no ids. - t1_claim2 = { 'resource_id': 117, + t1_claim2 = { 'resource_id': cep4_id, 'starttime': task1['starttime'], 'endtime': task1['endtime'], 'status': 'claimed', @@ -1060,7 +1061,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual(1, len(self.radb.getResourceClaims(task_ids=task_id1))) #there should still be one (proper/non-faulty) claim for this task # try to insert a claim with the wrong (already 'conflict') status. Should rollback, and return no ids. - t1_claim3 = { 'resource_id': 117, + t1_claim3 = { 'resource_id': cep4_id, 'starttime': task1['starttime'], 'endtime': task1['endtime'], 'status': 'conflict', @@ -1084,8 +1085,8 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual(0, len(self.radb.get_overlapping_claims(t1_claim_ids[0]))) self.assertEqual(0, len(self.radb.get_overlapping_tasks(t1_claim_ids[0]))) - self.assertEqual(40, self.radb.get_max_resource_usage_between(117, task1['starttime'], task1['starttime'], 'claimed')['usage']) - self.assertEqual(0, self.radb.get_max_resource_usage_between(117, task1['starttime']-timedelta(hours=2), task1['starttime']-timedelta(hours=1), 'claimed')['usage']) + self.assertEqual(40, self.radb.get_max_resource_usage_between(cep4_id, task1['starttime'], task1['starttime'], 'claimed')['usage']) + self.assertEqual(0, self.radb.get_max_resource_usage_between(cep4_id, task1['starttime']-timedelta(hours=2), task1['starttime']-timedelta(hours=1), 'claimed')['usage']) logger.info('------------------------------ concludes task 1 ------------------------------') logger.info('-- now test with a 2nd task, and test resource availability, conflicts etc. --') @@ -1100,7 +1101,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertTrue(task2) # insert a claim which won't fit, claim status after insert should be 'conflict' instead of 'tentative' - t2_claim1 = { 'resource_id': 117, + t2_claim1 = { 'resource_id': cep4_id, 'starttime': task2['starttime'], 'endtime': task2['endtime'], 'status': 'tentative', @@ -1126,7 +1127,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): #try to connect this claim to other resource, should fail self.assertFalse(self.radb.updateResourceClaims(t2_claim_ids, resource_id=118)) - self.assertEqual(117, t2_claims[0]['resource_id']) + self.assertEqual(cep4_id, t2_claims[0]['resource_id']) # try to update the task status to scheduled, should not succeed, since it's claims are not 'claimed' yet. self.assertFalse(self.radb.updateTask(task_id2, task_status='scheduled')) @@ -1174,9 +1175,9 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # effect might be that a scheduled tasks goes to conflict # now, make simple endtime adjustment, task should stay scheduled - logger.info("resource usages:\n%s", pformat(self.radb.getResourceUsages(now-timedelta(days=1.0), now+timedelta(days=2.0), 117))) + logger.info("resource usages:\n%s", pformat(self.radb.getResourceUsages(now-timedelta(days=1.0), now+timedelta(days=2.0), cep4_id))) self.assertTrue(self.radb.updateTaskAndResourceClaims(task_id2, endtime=now+timedelta(hours=2.75))) - logger.info("resource usages:\n%s", pformat(self.radb.getResourceUsages(now-timedelta(days=1.0), now+timedelta(days=2.0), 117))) + logger.info("resource usages:\n%s", pformat(self.radb.getResourceUsages(now-timedelta(days=1.0), now+timedelta(days=2.0), cep4_id))) # now the task and claim status should still be scheduled/claimed self.assertEqual('scheduled', self.radb.getTask(task_id2)['status']) @@ -1218,7 +1219,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertTrue(task3) # insert a claim which won't fit, claim status after insert should be 'conflict' instead of 'tentative' - t3_claim1 = { 'resource_id': 117, + t3_claim1 = { 'resource_id': cep4_id, 'starttime': task3['starttime'], 'endtime': task3['endtime'], 'status': 'tentative', @@ -1289,17 +1290,18 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # break it first... self._execute_query('TRUNCATE TABLE resource_allocation.resource_usage;') #check that it's broken - self.assertNotEqual(40, self.radb.get_max_resource_usage_between(117, task1['starttime'], task1['starttime'], 'claimed')['usage']) + self.assertNotEqual(40, self.radb.get_max_resource_usage_between(cep4_id, task1['starttime'], task1['starttime'], 'claimed')['usage']) #fix it self.radb.rebuild_resource_usages_from_claims() #and test again that it's ok - self.assertEqual(40, self.radb.get_max_resource_usage_between(117, task1['starttime'], task1['starttime'], 'claimed')['usage']) - self.assertEqual(0, self.radb.get_max_resource_usage_between(117, task1['starttime']-timedelta(hours=2), task1['starttime']-timedelta(hours=1), 'claimed')['usage']) + self.assertEqual(40, self.radb.get_max_resource_usage_between(cep4_id, task1['starttime'], task1['starttime'], 'claimed')['usage']) + self.assertEqual(0, self.radb.get_max_resource_usage_between(cep4_id, task1['starttime']-timedelta(hours=2), task1['starttime']-timedelta(hours=1), 'claimed')['usage']) def test_resource_usages(self): # for testing purposous let's give CEP4 storage a total size of 100 - self.assertTrue(self.radb.updateResourceAvailability(117, available_capacity=100, total_capacity=100)) - self.assertEqual(100, self.radb.getResources(117, include_availability=True)[0]['total_capacity']) + cep4_id = 117 + self.assertTrue(self.radb.updateResourceAvailability(cep4_id, available_capacity=100, total_capacity=100)) + self.assertEqual(100, self.radb.getResources(cep4_id, include_availability=True)[0]['total_capacity']) now = datetime.utcnow() now -= timedelta(minutes=now.minute, seconds=now.second, microseconds=now.microsecond) # round to full hour @@ -1316,121 +1318,121 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # because there are various special cases coded below where claims overlap/touch/etc which all need to be checked. # insert a claim, and check the usages for various timestamps - claim1 = {'resource_id': 117, 'starttime': now+timedelta(minutes=0), 'endtime': now+timedelta(minutes=10), 'status': 'tentative', 'claim_size': 1} + claim1 = {'resource_id': cep4_id, 'starttime': now+timedelta(minutes=0), 'endtime': now+timedelta(minutes=10), 'status': 'tentative', 'claim_size': 1} self.radb.insertResourceClaims(task_id, [claim1], 'foo', 1, 1) # test usages twice, once to check the usages generated by insert-triggers, and then to check usages generated by rebuild_resource_usages_from_claims for i in range(2): if i == 1: # make sure the usage table is wiped, so asserts fail when rebuild_resource_usages_from_claims is erroneously roll'ed back. self.radb._executeQuery('TRUNCATE resource_allocation.resource_usage;') - self.radb.rebuild_resource_usages_from_claims(117, 'tentative') - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now-timedelta(minutes=10), 'tentative')['usage']) - self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(117, claim1['starttime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, claim1['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now+timedelta(minutes=1000), 'tentative')['usage']) + self.radb.rebuild_resource_usages_from_claims(cep4_id, 'tentative') + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(minutes=10), 'tentative')['usage']) + self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['starttime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['endtime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(minutes=1000), 'tentative')['usage']) # insert another non-overlapping claim, and check the usages for various timestamps - claim2 = {'resource_id': 117, 'starttime': now+timedelta(minutes=20), 'endtime': now+timedelta(minutes=30), 'status': 'tentative', 'claim_size': 1} + claim2 = {'resource_id': cep4_id, 'starttime': now+timedelta(minutes=20), 'endtime': now+timedelta(minutes=30), 'status': 'tentative', 'claim_size': 1} self.radb.insertResourceClaims(task_id, [claim2], 'foo', 1, 1) # test usages twice, once to check the usages generated by insert-triggers, and then to check usages generated by rebuild_resource_usages_from_claims for i in range(2): if i == 1: # make sure the usage table is wiped, so asserts fail when rebuild_resource_usages_from_claims is erroneously roll'ed back. self.radb._executeQuery('TRUNCATE resource_allocation.resource_usage;') - self.radb.rebuild_resource_usages_from_claims(117, 'tentative') - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now-timedelta(minutes=10), 'tentative')['usage']) - self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(117, claim1['starttime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, claim1['endtime'], 'tentative')['usage']) - self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(117, claim2['starttime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, claim2['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now+timedelta(minutes=1000), 'tentative')['usage']) + self.radb.rebuild_resource_usages_from_claims(cep4_id, 'tentative') + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(minutes=10), 'tentative')['usage']) + self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['starttime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['endtime'], 'tentative')['usage']) + self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['starttime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['endtime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(minutes=1000), 'tentative')['usage']) # insert another claim which overlaps with both claim1 and claim2, and check the usages for various timestamps - claim3 = {'resource_id': 117, 'starttime': now+timedelta(minutes=5), 'endtime': now+timedelta(minutes=25), 'status': 'tentative', 'claim_size': 1} + claim3 = {'resource_id': cep4_id, 'starttime': now+timedelta(minutes=5), 'endtime': now+timedelta(minutes=25), 'status': 'tentative', 'claim_size': 1} self.radb.insertResourceClaims(task_id, [claim3], 'foo', 1, 1) # test usages twice, once to check the usages generated by insert-triggers, and then to check usages generated by rebuild_resource_usages_from_claims for i in range(2): if i == 1: # make sure the usage table is wiped, so asserts fail when rebuild_resource_usages_from_claims is erroneously roll'ed back. self.radb._executeQuery('TRUNCATE resource_allocation.resource_usage;') - self.radb.rebuild_resource_usages_from_claims(117, 'tentative') - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now-timedelta(minutes=10), 'tentative')['usage']) - self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(117, claim1['starttime'], 'tentative')['usage']) - self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(117, claim3['starttime'], 'tentative')['usage']) - self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(117, claim1['endtime'], 'tentative')['usage']) - self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(117, claim2['starttime'], 'tentative')['usage']) - self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(117, claim3['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, claim2['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now+timedelta(minutes=1000), 'tentative')['usage']) + self.radb.rebuild_resource_usages_from_claims(cep4_id, 'tentative') + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(minutes=10), 'tentative')['usage']) + self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['starttime'], 'tentative')['usage']) + self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['starttime'], 'tentative')['usage']) + self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['endtime'], 'tentative')['usage']) + self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['starttime'], 'tentative')['usage']) + self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['endtime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['endtime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(minutes=1000), 'tentative')['usage']) # insert another claim which overlaps with claim1 and ends at the same endtime as claim3, and check the usages for various timestamps - claim4 = {'resource_id': 117, 'starttime': now+timedelta(minutes=7.5), 'endtime': claim3['endtime'], 'status': 'tentative', 'claim_size': 1} + claim4 = {'resource_id': cep4_id, 'starttime': now+timedelta(minutes=7.5), 'endtime': claim3['endtime'], 'status': 'tentative', 'claim_size': 1} self.radb.insertResourceClaims(task_id, [claim4], 'foo', 1, 1) # test usages twice, once to check the usages generated by insert-triggers, and then to check usages generated by rebuild_resource_usages_from_claims for i in range(2): if i == 1: # make sure the usage table is wiped, so asserts fail when rebuild_resource_usages_from_claims is erroneously roll'ed back. self.radb._executeQuery('TRUNCATE resource_allocation.resource_usage;') - self.radb.rebuild_resource_usages_from_claims(117, 'tentative') - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now-timedelta(minutes=10), 'tentative')['usage']) - self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(117, claim1['starttime'], 'tentative')['usage']) - self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(117, claim3['starttime'], 'tentative')['usage']) - self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(117, claim4['starttime'], 'tentative')['usage']) - self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(117, claim1['endtime'], 'tentative')['usage']) - self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(117, claim2['starttime'], 'tentative')['usage']) - self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(117, claim3['endtime'], 'tentative')['usage']) #c4_endtime should be equal to c3_endtime - self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(117, claim4['endtime'], 'tentative')['usage']) #so usage should drop by 2*1 at this timestamp - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, claim2['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now+timedelta(minutes=1000), 'tentative')['usage']) + self.radb.rebuild_resource_usages_from_claims(cep4_id, 'tentative') + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(minutes=10), 'tentative')['usage']) + self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['starttime'], 'tentative')['usage']) + self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['starttime'], 'tentative')['usage']) + self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(cep4_id, claim4['starttime'], 'tentative')['usage']) + self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['endtime'], 'tentative')['usage']) + self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['starttime'], 'tentative')['usage']) + self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['endtime'], 'tentative')['usage']) #c4_endtime should be equal to c3_endtime + self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim4['endtime'], 'tentative')['usage']) #so usage should drop by 2*1 at this timestamp + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['endtime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(minutes=1000), 'tentative')['usage']) # insert another claim which starts when claim1 ends and last 1 minute, and check the usages for various timestamps - claim5 = {'resource_id': 117, 'starttime': claim1['endtime'], 'endtime': claim1['endtime']+timedelta(minutes=1), 'status': 'tentative', 'claim_size': 1} + claim5 = {'resource_id': cep4_id, 'starttime': claim1['endtime'], 'endtime': claim1['endtime']+timedelta(minutes=1), 'status': 'tentative', 'claim_size': 1} self.radb.insertResourceClaims(task_id, [claim5], 'foo', 1, 1) # test usages twice, once to check the usages generated by insert-triggers, and then to check usages generated by rebuild_resource_usages_from_claims for i in range(2): if i == 1: # make sure the usage table is wiped, so asserts fail when rebuild_resource_usages_from_claims is erroneously roll'ed back. self.radb._executeQuery('TRUNCATE resource_allocation.resource_usage;') - self.radb.rebuild_resource_usages_from_claims(117, 'tentative') - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now-timedelta(minutes=10), 'tentative')['usage']) - self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(117, claim1['starttime'], 'tentative')['usage']) - self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(117, claim3['starttime'], 'tentative')['usage']) - self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(117, claim4['starttime'], 'tentative')['usage']) - self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(117, claim1['endtime'], 'tentative')['usage']) #drops by 1 because c1 ends, but climbs by 1 because c5 starts - self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(117, claim5['starttime'], 'tentative')['usage']) #drops by 1 because c1 ends, but climbs by 1 because c5 starts - self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(117, claim5['endtime'], 'tentative')['usage']) #drops by 1 because c5 ends - self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(117, claim2['starttime'], 'tentative')['usage']) - self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(117, claim3['endtime'], 'tentative')['usage']) - self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(117, claim4['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, claim2['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now+timedelta(minutes=1000), 'tentative')['usage']) + self.radb.rebuild_resource_usages_from_claims(cep4_id, 'tentative') + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(minutes=10), 'tentative')['usage']) + self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['starttime'], 'tentative')['usage']) + self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['starttime'], 'tentative')['usage']) + self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(cep4_id, claim4['starttime'], 'tentative')['usage']) + self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['endtime'], 'tentative')['usage']) #drops by 1 because c1 ends, but climbs by 1 because c5 starts + self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(cep4_id, claim5['starttime'], 'tentative')['usage']) #drops by 1 because c1 ends, but climbs by 1 because c5 starts + self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim5['endtime'], 'tentative')['usage']) #drops by 1 because c5 ends + self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['starttime'], 'tentative')['usage']) + self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['endtime'], 'tentative')['usage']) + self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim4['endtime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['endtime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(minutes=1000), 'tentative')['usage']) # last edge case, insert another claim which starts when first claim starts, and end when last claim ends. Should lift all usages by 1 (except outer ones). - claim6 = {'resource_id': 117, 'starttime': claim1['starttime'], 'endtime': claim2['endtime'], 'status': 'tentative', 'claim_size': 1} + claim6 = {'resource_id': cep4_id, 'starttime': claim1['starttime'], 'endtime': claim2['endtime'], 'status': 'tentative', 'claim_size': 1} self.radb.insertResourceClaims(task_id, [claim6], 'foo', 1, 1) # test usages twice, once to check the usages generated by insert-triggers, and then to check usages generated by rebuild_resource_usages_from_claims for i in range(2): if i == 1: # make sure the usage table is wiped, so asserts fail when rebuild_resource_usages_from_claims is erroneously roll'ed back. self.radb._executeQuery('TRUNCATE resource_allocation.resource_usage;') - self.radb.rebuild_resource_usages_from_claims(117, 'tentative') - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now-timedelta(minutes=10), 'tentative')['usage']) - self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(117, claim1['starttime'], 'tentative')['usage']) - self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(117, claim3['starttime'], 'tentative')['usage']) - self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(117, claim4['starttime'], 'tentative')['usage']) - self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(117, claim1['endtime'], 'tentative')['usage']) - self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(117, claim5['starttime'], 'tentative')['usage']) - self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(117, claim5['endtime'], 'tentative')['usage']) - self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(117, claim2['starttime'], 'tentative')['usage']) - self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(117, claim3['endtime'], 'tentative')['usage']) #c4_endtime should be equal to c3_endtime - self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(117, claim4['endtime'], 'tentative')['usage']) #so usage should drop by 2*1 at this timestamp - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, claim2['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now+timedelta(minutes=1000), 'tentative')['usage']) + self.radb.rebuild_resource_usages_from_claims(cep4_id, 'tentative') + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(minutes=10), 'tentative')['usage']) + self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['starttime'], 'tentative')['usage']) + self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['starttime'], 'tentative')['usage']) + self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(cep4_id, claim4['starttime'], 'tentative')['usage']) + self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['endtime'], 'tentative')['usage']) + self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(cep4_id, claim5['starttime'], 'tentative')['usage']) + self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(cep4_id, claim5['endtime'], 'tentative')['usage']) + self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['starttime'], 'tentative')['usage']) + self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['endtime'], 'tentative')['usage']) #c4_endtime should be equal to c3_endtime + self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim4['endtime'], 'tentative')['usage']) #so usage should drop by 2*1 at this timestamp + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['endtime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(minutes=1000), 'tentative')['usage']) # conclude with two simple cases, # first final simple case: insert another claim follows (non overlapping/non-touching) all others - claim7 = {'resource_id': 117, 'starttime': claim2['endtime']+timedelta(minutes=10), 'endtime': claim2['endtime']+timedelta(minutes=20), 'status': 'tentative', 'claim_size': 1} + claim7 = {'resource_id': cep4_id, 'starttime': claim2['endtime']+timedelta(minutes=10), 'endtime': claim2['endtime']+timedelta(minutes=20), 'status': 'tentative', 'claim_size': 1} self.radb.insertResourceClaims(task_id, [claim7], 'foo', 1, 1) # test usages twice, once to check the usages generated by insert-triggers, and then to check usages generated by rebuild_resource_usages_from_claims for i in range(2): @@ -1438,48 +1440,48 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # make sure the usage table is wiped, so asserts fail when rebuild_resource_usages_from_claims is erroneously roll'ed back. logger.info('') self.radb._executeQuery('TRUNCATE resource_allocation.resource_usage;') - self.radb.rebuild_resource_usages_from_claims(117, 'tentative') - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now-timedelta(minutes=10), 'tentative')['usage']) - self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(117, claim1['starttime'], 'tentative')['usage']) - self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(117, claim3['starttime'], 'tentative')['usage']) - self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(117, claim4['starttime'], 'tentative')['usage']) - self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(117, claim1['endtime'], 'tentative')['usage']) - self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(117, claim5['starttime'], 'tentative')['usage']) - self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(117, claim5['endtime'], 'tentative')['usage']) - self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(117, claim2['starttime'], 'tentative')['usage']) - self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(117, claim3['endtime'], 'tentative')['usage']) - self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(117, claim4['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, claim2['endtime'], 'tentative')['usage']) - self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(117, claim7['starttime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, claim7['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now+timedelta(minutes=1000), 'tentative')['usage']) + self.radb.rebuild_resource_usages_from_claims(cep4_id, 'tentative') + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(minutes=10), 'tentative')['usage']) + self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['starttime'], 'tentative')['usage']) + self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['starttime'], 'tentative')['usage']) + self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(cep4_id, claim4['starttime'], 'tentative')['usage']) + self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['endtime'], 'tentative')['usage']) + self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(cep4_id, claim5['starttime'], 'tentative')['usage']) + self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(cep4_id, claim5['endtime'], 'tentative')['usage']) + self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['starttime'], 'tentative')['usage']) + self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['endtime'], 'tentative')['usage']) + self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim4['endtime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['endtime'], 'tentative')['usage']) + self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim7['starttime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim7['endtime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(minutes=1000), 'tentative')['usage']) # second final simple case: insert another claim which precedes (non overlapping/non-touching) all the others - claim8 = {'resource_id': 117, 'starttime': claim1['starttime']-timedelta(minutes=20), 'endtime': claim1['starttime']-timedelta(minutes=10), 'status': 'tentative', 'claim_size': 1} + claim8 = {'resource_id': cep4_id, 'starttime': claim1['starttime']-timedelta(minutes=20), 'endtime': claim1['starttime']-timedelta(minutes=10), 'status': 'tentative', 'claim_size': 1} self.radb.insertResourceClaims(task_id, [claim8], 'foo', 1, 1) # test usages twice, once to check the usages generated by insert-triggers, and then to check usages generated by rebuild_resource_usages_from_claims for i in range(2): if i == 1: # make sure the usage table is wiped, so asserts fail when rebuild_resource_usages_from_claims is erroneously roll'ed back. self.radb._executeQuery('TRUNCATE resource_allocation.resource_usage;') - self.radb.rebuild_resource_usages_from_claims(117, 'tentative') - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now-timedelta(minutes=100), 'tentative')['usage']) - self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(117, claim8['starttime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, claim8['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, claim1['starttime']-timedelta(minutes=1), 'tentative')['usage']) - self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(117, claim1['starttime'], 'tentative')['usage']) - self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(117, claim3['starttime'], 'tentative')['usage']) - self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(117, claim4['starttime'], 'tentative')['usage']) - self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(117, claim1['endtime'], 'tentative')['usage']) - self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(117, claim5['starttime'], 'tentative')['usage']) - self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(117, claim5['endtime'], 'tentative')['usage']) - self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(117, claim2['starttime'], 'tentative')['usage']) - self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(117, claim3['endtime'], 'tentative')['usage']) - self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(117, claim4['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, claim2['endtime'], 'tentative')['usage']) - self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(117, claim7['starttime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, claim7['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now+timedelta(minutes=1000), 'tentative')['usage']) + self.radb.rebuild_resource_usages_from_claims(cep4_id, 'tentative') + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(minutes=100), 'tentative')['usage']) + self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim8['starttime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim8['endtime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['starttime']-timedelta(minutes=1), 'tentative')['usage']) + self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['starttime'], 'tentative')['usage']) + self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['starttime'], 'tentative')['usage']) + self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(cep4_id, claim4['starttime'], 'tentative')['usage']) + self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['endtime'], 'tentative')['usage']) + self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(cep4_id, claim5['starttime'], 'tentative')['usage']) + self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(cep4_id, claim5['endtime'], 'tentative')['usage']) + self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['starttime'], 'tentative')['usage']) + self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['endtime'], 'tentative')['usage']) + self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim4['endtime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['endtime'], 'tentative')['usage']) + self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim7['starttime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim7['endtime'], 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(minutes=1000), 'tentative')['usage']) def test_overlapping_claims(self): # this is a special testcase to prove a bug found at 2017-08-16 @@ -1488,8 +1490,9 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # and then, we'll fix the code, (so this test succeeds) # for testing purposous let's give CEP4 storage a total size of 100 - self.assertTrue(self.radb.updateResourceAvailability(117, available_capacity=100, total_capacity=100)) - self.assertEqual(100, self.radb.getResources(117, include_availability=True)[0]['total_capacity']) + cep4_id = 117 + self.assertTrue(self.radb.updateResourceAvailability(cep4_id, available_capacity=100, total_capacity=100)) + self.assertEqual(100, self.radb.getResources(cep4_id, include_availability=True)[0]['total_capacity']) now = datetime.utcnow() now -= timedelta(minutes=now.minute, seconds=now.second, microseconds=now.microsecond) # round to full hour @@ -1504,12 +1507,12 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual(task_id, task['id']) #create two overlapping claims - claims = [ { 'resource_id': 117, + claims = [ { 'resource_id': cep4_id, 'starttime': now, 'endtime': now+timedelta(hours=0.75), 'status': 'tentative', 'claim_size': 40 }, - {'resource_id': 117, + {'resource_id': cep4_id, 'starttime': now+timedelta(hours=0.25), 'endtime': now + timedelta(hours=1), 'status': 'tentative', @@ -1534,18 +1537,18 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual('claimed', claim['status']) # check the resource usage trend - logger.info("resource usages:\n%s", pformat(self.radb.getResourceUsages(now-timedelta(hours=1.0), now+timedelta(hours=2.0), 117))) - self.assertEqual(0, self.radb.get_max_resource_usage_between(117, now-timedelta(hours=1.0), now-timedelta(hours=0.01), 'claimed')['usage']) - self.assertEqual(40, self.radb.get_max_resource_usage_between(117, now+timedelta(hours=0.0), now+timedelta(hours=0.2), 'claimed')['usage']) - self.assertEqual(80, self.radb.get_max_resource_usage_between(117, now+timedelta(hours=0.3), now+timedelta(hours=0.6), 'claimed')['usage']) - self.assertEqual(40, self.radb.get_max_resource_usage_between(117, now+timedelta(hours=0.80), now+timedelta(hours=1.0), 'claimed')['usage']) + logger.info("resource usages:\n%s", pformat(self.radb.getResourceUsages(now-timedelta(hours=1.0), now+timedelta(hours=2.0), cep4_id))) + self.assertEqual(0, self.radb.get_max_resource_usage_between(cep4_id, now-timedelta(hours=1.0), now-timedelta(hours=0.01), 'claimed')['usage']) + self.assertEqual(40, self.radb.get_max_resource_usage_between(cep4_id, now+timedelta(hours=0.0), now+timedelta(hours=0.2), 'claimed')['usage']) + self.assertEqual(80, self.radb.get_max_resource_usage_between(cep4_id, now+timedelta(hours=0.3), now+timedelta(hours=0.6), 'claimed')['usage']) + self.assertEqual(40, self.radb.get_max_resource_usage_between(cep4_id, now+timedelta(hours=0.80), now+timedelta(hours=1.0), 'claimed')['usage']) #check for a time range encapsulating the full task - self.assertEqual(80, self.radb.get_max_resource_usage_between(117, now+timedelta(hours=-0.1), now+timedelta(hours=1.1), 'claimed')['usage']) + self.assertEqual(80, self.radb.get_max_resource_usage_between(cep4_id, now+timedelta(hours=-0.1), now+timedelta(hours=1.1), 'claimed')['usage']) #check for a time range not including the task - self.assertEqual(0, self.radb.get_max_resource_usage_between(117, now+timedelta(hours=1.1), now+timedelta(hours=2.0), 'claimed')['usage']) - self.assertEqual(0, self.radb.get_max_resource_usage_between(117, now-timedelta(hours=1.1), now-timedelta(hours=1.0), 'claimed')['usage']) + self.assertEqual(0, self.radb.get_max_resource_usage_between(cep4_id, now+timedelta(hours=1.1), now+timedelta(hours=2.0), 'claimed')['usage']) + self.assertEqual(0, self.radb.get_max_resource_usage_between(cep4_id, now-timedelta(hours=1.1), now-timedelta(hours=1.0), 'claimed')['usage']) # check that there are no overlapping conflicting claims/tasks for claim in claims: @@ -1558,22 +1561,22 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual(0, len(self.radb.get_overlapping_tasks(claim['id'], 'tentative'))) #check claimable_capacity for various timestamps - self.assertEqual(100, self.radb.get_resource_claimable_capacity(117, now-timedelta(hours=1.0), now-timedelta(hours=1.0))['claimable_capacity']) - self.assertEqual(60, self.radb.get_resource_claimable_capacity(117, now+timedelta(hours=0.0), now+timedelta(hours=0.0))['claimable_capacity']) - self.assertEqual(60, self.radb.get_resource_claimable_capacity(117, now+timedelta(hours=0.2), now+timedelta(hours=0.2))['claimable_capacity']) - self.assertEqual(20, self.radb.get_resource_claimable_capacity(117, now+timedelta(hours=0.3), now+timedelta(hours=0.3))['claimable_capacity']) - self.assertEqual(20, self.radb.get_resource_claimable_capacity(117, now+timedelta(hours=0.5), now+timedelta(hours=0.5))['claimable_capacity']) - self.assertEqual(60, self.radb.get_resource_claimable_capacity(117, now+timedelta(hours=0.75), now+timedelta(hours=0.75))['claimable_capacity']) - self.assertEqual(60, self.radb.get_resource_claimable_capacity(117, now+timedelta(hours=0.8), now+timedelta(hours=0.8))['claimable_capacity']) - self.assertEqual(100, self.radb.get_resource_claimable_capacity(117, now+timedelta(hours=1.0), now+timedelta(hours=1.0))['claimable_capacity']) - self.assertEqual(100, self.radb.get_resource_claimable_capacity(117, now+timedelta(hours=10.0), now+timedelta(hours=10.0))['claimable_capacity']) + self.assertEqual(100, self.radb.get_resource_claimable_capacity(cep4_id, now-timedelta(hours=1.0), now-timedelta(hours=1.0))['claimable_capacity']) + self.assertEqual(60, self.radb.get_resource_claimable_capacity(cep4_id, now+timedelta(hours=0.0), now+timedelta(hours=0.0))['claimable_capacity']) + self.assertEqual(60, self.radb.get_resource_claimable_capacity(cep4_id, now+timedelta(hours=0.2), now+timedelta(hours=0.2))['claimable_capacity']) + self.assertEqual(20, self.radb.get_resource_claimable_capacity(cep4_id, now+timedelta(hours=0.3), now+timedelta(hours=0.3))['claimable_capacity']) + self.assertEqual(20, self.radb.get_resource_claimable_capacity(cep4_id, now+timedelta(hours=0.5), now+timedelta(hours=0.5))['claimable_capacity']) + self.assertEqual(60, self.radb.get_resource_claimable_capacity(cep4_id, now+timedelta(hours=0.75), now+timedelta(hours=0.75))['claimable_capacity']) + self.assertEqual(60, self.radb.get_resource_claimable_capacity(cep4_id, now+timedelta(hours=0.8), now+timedelta(hours=0.8))['claimable_capacity']) + self.assertEqual(100, self.radb.get_resource_claimable_capacity(cep4_id, now+timedelta(hours=1.0), now+timedelta(hours=1.0))['claimable_capacity']) + self.assertEqual(100, self.radb.get_resource_claimable_capacity(cep4_id, now+timedelta(hours=10.0), now+timedelta(hours=10.0))['claimable_capacity']) #check claimable_capacity for full task's timewindow (+extra) - self.assertEqual(20, self.radb.get_resource_claimable_capacity(117, now-timedelta(hours=10.0), now+timedelta(hours=10.0))['claimable_capacity']) + self.assertEqual(20, self.radb.get_resource_claimable_capacity(cep4_id, now-timedelta(hours=10.0), now+timedelta(hours=10.0))['claimable_capacity']) #add an extra claim, overlapping with only the last claim of size 40. So it should fit (100-40=60 and 60>30). - extra_claim = { 'resource_id': 117, + extra_claim = { 'resource_id': cep4_id, 'starttime': now+timedelta(hours=0.8), 'endtime': now+timedelta(hours=0.9), 'status': 'tentative', @@ -1604,8 +1607,9 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # We'll keep this new test anyway, just to prove that these cases work as expected. # for testing purposous let's give CEP4 storage a total size of 100 - self.assertTrue(self.radb.updateResourceAvailability(117, available_capacity=100, total_capacity=100)) - self.assertEqual(100, self.radb.getResources(117, include_availability=True)[0]['total_capacity']) + cep4_id = 117 + self.assertTrue(self.radb.updateResourceAvailability(cep4_id, available_capacity=100, total_capacity=100)) + self.assertEqual(100, self.radb.getResources(cep4_id, include_availability=True)[0]['total_capacity']) now = datetime.utcnow() now -= timedelta(minutes=now.minute, seconds=now.second, microseconds=now.microsecond) # round to full hour @@ -1634,7 +1638,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual('second content', self.radb.getSpecification(task['specification_id'])['content']) #spec content should have been renewed # create and insert a claim - claim = {'resource_id': 117, + claim = {'resource_id': cep4_id, 'starttime': task['starttime'], 'endtime': task['endtime'], 'status': 'tentative', @@ -1653,9 +1657,9 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): for claim in self.radb.getResourceClaims(claim_ids=claim_ids): self.assertEqual('claimed', claim['status']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now-timedelta(hours=0.5), 'claimed')['usage']) - self.assertEqual(40, self.radb.get_resource_usage_at_or_before(117, now+timedelta(hours=0.5), 'claimed')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now+timedelta(hours=1.5), 'claimed')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(hours=0.5), 'claimed')['usage']) + self.assertEqual(40, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(hours=0.5), 'claimed')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(hours=1.5), 'claimed')['usage']) # prove again that we can re-insert the spec/task (now with claims), and that the new task is indeed inserted and new, # and that the claim(s) and usage(s) were actually deleted (via cascading deletes) @@ -1673,9 +1677,9 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # this newly inserted spec/task should have no claims anymore self.assertEqual( 0, len(self.radb.getResourceClaims())) # and all usages should now be 0 - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now-timedelta(hours=0.5), 'claimed')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now+timedelta(hours=0.5), 'claimed')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now+timedelta(hours=1.5), 'claimed')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(hours=0.5), 'claimed')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(hours=0.5), 'claimed')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(hours=1.5), 'claimed')['usage']) # 2017-08-29: ok, we could not reproduce the bug found on production, # so it seems there is a strange corner case which caused the usages table to become inconsistent. @@ -1684,7 +1688,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # so, let's do that in the remainder of this test. #insert a claim again (cause we don't have a claim anymore since we inserted the spec/taks above) - claim = {'resource_id': 117, + claim = {'resource_id': cep4_id, 'starttime': task['starttime'], 'endtime': task['endtime'], 'status': 'tentative', @@ -1694,9 +1698,9 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual(1, len(self.radb.getResourceClaims(claim_ids=claim_ids))) # usages should be ok - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now-timedelta(hours=0.5), 'tentative')['usage']) - self.assertEqual(40, self.radb.get_resource_usage_at_or_before(117, now+timedelta(hours=0.5), 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now+timedelta(hours=1.5), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(hours=0.5), 'tentative')['usage']) + self.assertEqual(40, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(hours=0.5), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(hours=1.5), 'tentative')['usage']) # now let's break the usages table (intentionally, to mimic the production inconsistenty) # we shift one entry (where the claim starts) by one minute @@ -1705,7 +1709,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.radb._executeQuery("UPDATE resource_allocation.resource_usage SET as_of_timestamp = %s WHERE as_of_timestamp = %s;", (now+timedelta(minutes=1), now)) # check that the usages were indeed changed (the first one shifted in time) - usages = self.radb.getResourceUsages()[117]['tentative'] + usages = self.radb.getResourceUsages()[cep4_id]['tentative'] self.assertEqual({'usage': 40, 'as_of_timestamp': now+timedelta(minutes=1)}, usages[0]) self.assertEqual({'usage': 0, 'as_of_timestamp': task['endtime']}, usages[1]) @@ -1726,10 +1730,144 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # this newly inserted spec/task should have no claims anymore self.assertEqual( 0, len(self.radb.getResourceClaims())) # and all usages should now be 0 - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now-timedelta(hours=0.5), 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now+timedelta(hours=0.5), 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(117, now+timedelta(hours=1.5), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(hours=0.5), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(hours=0.5), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(hours=1.5), 'tentative')['usage']) + def test_claims_on_partially_misc_filled_resource(self): + # this is a special testcase to prove a bug found at 2017-08-24 + # the bug was that a claim that should fit, does not fit according to the radb claim-methods. + # the reason that it (erroneously) does not fit is an error in the get_resource_claimable_capacity_between method in sql. + # first, we'll prove that the bug exists (and that this test fails), + # and then, we'll fix the code, (so this test succeeds) + + #first make sure that there are no specs/tasks/claims lingering around + for spec in self.radb.getSpecifications(): + self.radb.deleteSpecification(spec['id']) # cascades into tasks and claims + + # for testing purposous let's give CEP4 storage a total size of 100 + cep4_id = 117 + self.assertTrue(self.radb.updateResourceAvailability(cep4_id, available_capacity=100, total_capacity=100)) + self.assertEqual(100, self.radb.getResources(cep4_id, include_availability=True)[0]['total_capacity']) + self.assertEqual(100, self.radb.getResources(cep4_id, include_availability=True)[0]['available_capacity']) + self.assertEqual( 0, self.radb.getResources(cep4_id, include_availability=True)[0]['used_capacity']) + self.assertEqual( 0, self.radb.getResources(cep4_id, include_availability=True)[0]['misc_used_capacity']) + + now = datetime.utcnow() + start = now - timedelta(minutes=now.minute, seconds=now.second, microseconds=now.microsecond) # round to current full hour + + #insert a task + result = self.radb.insertSpecificationAndTask(0, 0, 'approved', 'observation', start, start+timedelta(hours=2), 'foo', 'CEP4') + self.assertTrue(result['inserted']) + task_id = result['task_id'] + + task = self.radb.getTask(task_id) + self.assertTrue(task) + self.assertEqual(task_id, task['id']) + + #check if there is indeed still 100 claimable_capacity, and that there is no resource_usage + #because there are no claims yet. + self.assertEqual(100, self.radb.get_resource_claimable_capacity(cep4_id, task['starttime'], task['endtime'])['claimable_capacity']) + self.assertEqual(0, self.radb.get_max_resource_usage_between(cep4_id, task['starttime'], task['endtime'], 'claimed')['usage']) + self.assertEqual(0, self.radb.get_current_resource_usage(cep4_id, 'claimed')['usage']) + + #add a claim for the task which should fit. + #there is 100 available + claim = { 'resource_id': cep4_id, + 'starttime': task['starttime'], + 'endtime': task['endtime'], + 'status': 'tentative', + 'claim_size': 50 } + claim_id = self.radb.insertResourceClaims(task_id, [claim], 'foo', 1, 1)[0] + claim = self.radb.getResourceClaims(claim_ids=[claim_id])[0] + self.assertEqual('tentative', claim['status']) + + # because the claim is still tentative, there should be 100 claimable_capacity left + self.assertEqual(100, self.radb.get_resource_claimable_capacity(cep4_id, task['starttime'], task['endtime'])['claimable_capacity']) + + # and the capacities of the resource should still be the same as in the beginning + self.assertEqual(100, self.radb.getResources(cep4_id, include_availability=True)[0]['total_capacity']) + self.assertEqual(100, self.radb.getResources(cep4_id, include_availability=True)[0]['available_capacity']) + self.assertEqual( 0, self.radb.getResources(cep4_id, include_availability=True)[0]['used_capacity']) + self.assertEqual( 0, self.radb.getResources(cep4_id, include_availability=True)[0]['misc_used_capacity']) + + # set the status to 'claimed', and check it. + self.assertTrue(self.radb.updateResourceClaims(claim_id, status='claimed')) + claim = self.radb.getResourceClaims(claim_ids=[claim_id])[0] + self.assertEqual('claimed', claim['status']) + + # now there should be 100-50=50 claimable_capacity left + self.assertEqual(50, self.radb.get_resource_claimable_capacity(cep4_id, task['starttime'], task['endtime'])['claimable_capacity']) + #and the resource_usage should be 50 + self.assertEqual(50, self.radb.get_max_resource_usage_between(cep4_id, task['starttime'], task['endtime'], 'claimed')['usage']) + self.assertEqual(50, self.radb.get_current_resource_usage(cep4_id, 'claimed')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, start-timedelta(hours=1), 'claimed')['usage']) + self.assertEqual(50, self.radb.get_resource_usage_at_or_before(cep4_id, claim['starttime'], 'claimed')['usage']) + self.assertEqual(50, self.radb.get_resource_usage_at_or_before(cep4_id, start+timedelta(hours=0.25), 'claimed')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim['endtime'], 'claimed')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, task['endtime'], 'claimed')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, task['endtime']+timedelta(hours=1), 'claimed')['usage']) + + # assume the data has been written, and that the claim of size 50 now occupies 50 on the resource + # that would be detected (by the storagequeryservice) and propagated into the radb + # so, let's update the available_capacity + self.assertTrue(self.radb.updateResourceAvailability(cep4_id, available_capacity=100-50)) + # check the capacities of the resource + # please note that the misc_used_capacity=0 because we used exactly the same amount of diskspace as was claimed (as it should be) + self.assertEqual(100, self.radb.getResources(cep4_id, include_availability=True)[0]['total_capacity']) + self.assertEqual( 50, self.radb.getResources(cep4_id, include_availability=True)[0]['available_capacity']) + self.assertEqual( 50, self.radb.getResources(cep4_id, include_availability=True)[0]['used_capacity']) + self.assertEqual( 0, self.radb.getResources(cep4_id, include_availability=True)[0]['misc_used_capacity']) + + # so far, so good... + # now onto the situation in practice.... + # suppose there is some additional (20) miscelaneous data on cep4, which is not known in claims (like backups/logs/other_data) + # this should be reflected in the available_capacity and misc_used_capacity + # available_capacity = 100-50-20 : 50 is claimed and in use, and 20 is other unaccounted for data. + self.assertTrue(self.radb.updateResourceAvailability(cep4_id, available_capacity=100-50-20)) + self.assertEqual(100, self.radb.getResources(cep4_id, include_availability=True)[0]['total_capacity']) + self.assertEqual( 30, self.radb.getResources(cep4_id, include_availability=True)[0]['available_capacity']) + self.assertEqual( 70, self.radb.getResources(cep4_id, include_availability=True)[0]['used_capacity']) + # and the used_capacity of 70 should be build up of the parts: resource_usage=50 and misc_used_capacity=20 + self.assertEqual( 50, self.radb.get_resource_usage_at_or_before(cep4_id, start+timedelta(hours=0.5), 'claimed')['usage']) + self.assertEqual( 20, self.radb.getResources(cep4_id, include_availability=True)[0]['misc_used_capacity']) + # and the resource_usage should still be 50 (cause no claims were added/changed) + self.assertEqual(50, self.radb.get_max_resource_usage_between(cep4_id, task['starttime'], task['endtime'], 'claimed')['usage']) + self.assertEqual(50, self.radb.get_current_resource_usage(cep4_id, 'claimed')['usage']) + # but, there should be less claimable capacity left: 100 -50 (claim) -20 (misc_data) = 30 claimable_capacity left + self.assertEqual(30, self.radb.get_resource_claimable_capacity(cep4_id, task['starttime'], task['endtime'])['claimable_capacity']) + # and for a new task (where there are no claims yet), there should be less claimable capacity left: 100-20 (misc_data) = 80 claimable_capacity left + self.assertEqual(80, self.radb.get_resource_claimable_capacity(cep4_id, task['endtime'] + timedelta(hours=1), task['endtime'] + timedelta(hours=2))['claimable_capacity']) + + #so, it should be possible to add an extra claim of 25 during this task (which should fit!) + claim2 = { 'resource_id': cep4_id, + 'starttime': task['starttime'], + 'endtime': task['endtime'], + 'status': 'tentative', + 'claim_size': 25 } + claim2_id = self.radb.insertResourceClaims(task_id, [claim2], 'foo', 1, 1)[0] + claim2 = self.radb.getResourceClaims(claim_ids=[claim2_id])[0] + self.assertEqual('tentative', claim2['status']) + + # and the claim should be able to have the status set to 'claimed'. check it. + self.assertTrue(self.radb.updateResourceClaims(claim2_id, status='claimed')) + claim2 = self.radb.getResourceClaims(claim_ids=[claim2_id])[0] + self.assertEqual('claimed', claim2['status']) + + #and, it should also be possible to add an extra claim of 75 after this task (where there is no claim yet) (which should fit!) + claim3 = { 'resource_id': cep4_id, + 'starttime': task['starttime'], + 'endtime': task['endtime'], + 'status': 'tentative', + 'claim_size': 75 } + claim3_id = self.radb.insertResourceClaims(task_id, [claim3], 'foo', 1, 1)[0] + claim3 = self.radb.getResourceClaims(claim_ids=[claim3_id])[0] + self.assertEqual('tentative', claim3['status']) + + # and the claim should be able to have the status set to 'claimed'. check it. + self.assertTrue(self.radb.updateResourceClaims(claim3_id, status='claimed')) + claim3 = self.radb.getResourceClaims(claim_ids=[claim3_id])[0] + self.assertEqual('claimed', claim3['status']) if __name__ == "__main__": os.environ['TZ'] = 'UTC'