From a53e2be56137203c7355c4b9f44c8b9b59a559bd Mon Sep 17 00:00:00 2001 From: Jorrit Schaap <schaap@astron.nl> Date: Mon, 11 Sep 2017 09:51:24 +0000 Subject: [PATCH] Task #11246: test was erroneously time-dependend. Fixed that. All radb tests pass, so no merge issues found. --- .../tests/t_radb.py | 169 ++++++++++-------- 1 file changed, 92 insertions(+), 77 deletions(-) diff --git a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.py b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.py index 8d3da661a76..99e8c0dacd7 100755 --- a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.py +++ b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb.py @@ -1018,7 +1018,10 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): now = datetime.utcnow() now -= timedelta(minutes=now.minute, seconds=now.second, microseconds=now.microsecond) # round to full hour - result = self.radb.insertSpecificationAndTask(0, 0, 'approved', 'observation', now, now+timedelta(hours=1), 'foo', 'CEP4') + # we'll schedule some tasks in the future... + future = now + timedelta(hours=2) + + result = self.radb.insertSpecificationAndTask(0, 0, 'approved', 'observation', future, future+timedelta(hours=1), 'foo', 'CEP4') self.assertTrue(result['inserted']) spec_id1 = result['specification_id'] task_id1 = result['task_id'] @@ -1092,7 +1095,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): logger.info('-- now test with a 2nd task, and test resource availability, conflicts etc. --') # another task, fully overlapping with task1 - result = self.radb.insertSpecificationAndTask(1, 1, 'approved', 'observation', now, now+timedelta(hours=1), 'foo', 'CEP4') + result = self.radb.insertSpecificationAndTask(1, 1, 'approved', 'observation', future, future+timedelta(hours=1), 'foo', 'CEP4') self.assertTrue(result['inserted']) spec_id2 = result['specification_id'] task_id2 = result['task_id'] @@ -1138,7 +1141,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual('conflict', self.radb.getResourceClaim(t2_claim_ids[0])['status']) # do conflict resolution, shift task and claims - self.assertTrue(self.radb.updateTaskAndResourceClaims(task_id2, starttime=now+timedelta(hours=2), endtime=now+timedelta(hours=3))) + self.assertTrue(self.radb.updateTaskAndResourceClaims(task_id2, starttime=future+timedelta(hours=2), endtime=future+timedelta(hours=3))) # now the task and claim status should not be conflict anymore self.assertEqual('tentative', self.radb.getResourceClaim(t2_claim_ids[0])['status']) self.assertEqual('approved', self.radb.getTask(task_id2)['status']) @@ -1165,7 +1168,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual('conflict', self.radb.getTask(task_id2)['status']) # again do conflict resolution, shift task and claims - self.assertTrue(self.radb.updateTaskAndResourceClaims(task_id2, starttime=now+timedelta(hours=2), endtime=now+timedelta(hours=3))) + self.assertTrue(self.radb.updateTaskAndResourceClaims(task_id2, starttime=future+timedelta(hours=2), endtime=future+timedelta(hours=3))) self.assertTrue(self.radb.updateTaskAndResourceClaims(task_id2, claim_status='claimed', task_status='scheduled')) # now the task and claim status should be scheduled/claimed self.assertEqual('scheduled', self.radb.getTask(task_id2)['status']) @@ -1175,9 +1178,9 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # effect might be that a scheduled tasks goes to conflict # now, make simple endtime adjustment, task should stay scheduled - logger.info("resource usages:\n%s", pformat(self.radb.getResourceUsages(now-timedelta(days=1.0), now+timedelta(days=2.0), cep4_id))) - self.assertTrue(self.radb.updateTaskAndResourceClaims(task_id2, endtime=now+timedelta(hours=2.75))) - logger.info("resource usages:\n%s", pformat(self.radb.getResourceUsages(now-timedelta(days=1.0), now+timedelta(days=2.0), cep4_id))) + logger.info("resource usages:\n%s", pformat(self.radb.getResourceUsages(future-timedelta(days=1.0), future+timedelta(days=2.0), cep4_id))) + self.assertTrue(self.radb.updateTaskAndResourceClaims(task_id2, endtime=future+timedelta(hours=2.75))) + logger.info("resource usages:\n%s", pformat(self.radb.getResourceUsages(future-timedelta(days=1.0), future+timedelta(days=2.0), cep4_id))) # now the task and claim status should still be scheduled/claimed self.assertEqual('scheduled', self.radb.getTask(task_id2)['status']) @@ -1194,7 +1197,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual('claimed', self.radb.getResourceClaim(t2_claim_ids[0])['status']) #ok, that works, now set the start/end time back to 'normal' for some later test cases - self.assertTrue(self.radb.updateTaskAndResourceClaims(task_id2, starttime=now+timedelta(hours=2), endtime=now+timedelta(hours=3))) + self.assertTrue(self.radb.updateTaskAndResourceClaims(task_id2, starttime=future+timedelta(hours=2), endtime=future+timedelta(hours=3))) self.assertEqual('finished', self.radb.getTask(task_id2)['status']) self.assertEqual('claimed', self.radb.getResourceClaim(t2_claim_ids[0])['status']) @@ -1306,8 +1309,11 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): now = datetime.utcnow() now -= timedelta(minutes=now.minute, seconds=now.second, microseconds=now.microsecond) # round to full hour + # we'll schedule some tasks in the future... + future = now + timedelta(hours=2) + # insert one task, and reuse that for multiple claims - result = self.radb.insertSpecificationAndTask(0, 0, 'approved', 'observation', now, now + timedelta(hours=1), + result = self.radb.insertSpecificationAndTask(0, 0, 'approved', 'observation', future, future + timedelta(hours=1), 'content', 'CEP4') self.assertTrue(result['inserted']) task_id = result['task_id'] @@ -1318,7 +1324,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # because there are various special cases coded below where claims overlap/touch/etc which all need to be checked. # insert a claim, and check the usages for various timestamps - claim1 = {'resource_id': cep4_id, 'starttime': now+timedelta(minutes=0), 'endtime': now+timedelta(minutes=10), 'status': 'tentative', 'claim_size': 1} + claim1 = {'resource_id': cep4_id, 'starttime': future+timedelta(minutes=0), 'endtime': future+timedelta(minutes=10), 'status': 'tentative', 'claim_size': 1} self.radb.insertResourceClaims(task_id, [claim1], 'foo', 1, 1) # test usages twice, once to check the usages generated by insert-triggers, and then to check usages generated by rebuild_resource_usages_from_claims for i in range(2): @@ -1326,13 +1332,13 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # make sure the usage table is wiped, so asserts fail when rebuild_resource_usages_from_claims is erroneously roll'ed back. self.radb._executeQuery('TRUNCATE resource_allocation.resource_usage;') self.radb.rebuild_resource_usages_from_claims(cep4_id, 'tentative') - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(minutes=10), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future-timedelta(minutes=10), 'tentative')['usage']) self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['starttime'], 'tentative')['usage']) self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(minutes=1000), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future+timedelta(minutes=1000), 'tentative')['usage']) # insert another non-overlapping claim, and check the usages for various timestamps - claim2 = {'resource_id': cep4_id, 'starttime': now+timedelta(minutes=20), 'endtime': now+timedelta(minutes=30), 'status': 'tentative', 'claim_size': 1} + claim2 = {'resource_id': cep4_id, 'starttime': future+timedelta(minutes=20), 'endtime': future+timedelta(minutes=30), 'status': 'tentative', 'claim_size': 1} self.radb.insertResourceClaims(task_id, [claim2], 'foo', 1, 1) # test usages twice, once to check the usages generated by insert-triggers, and then to check usages generated by rebuild_resource_usages_from_claims for i in range(2): @@ -1340,15 +1346,15 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # make sure the usage table is wiped, so asserts fail when rebuild_resource_usages_from_claims is erroneously roll'ed back. self.radb._executeQuery('TRUNCATE resource_allocation.resource_usage;') self.radb.rebuild_resource_usages_from_claims(cep4_id, 'tentative') - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(minutes=10), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future-timedelta(minutes=10), 'tentative')['usage']) self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['starttime'], 'tentative')['usage']) self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['endtime'], 'tentative')['usage']) self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['starttime'], 'tentative')['usage']) self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(minutes=1000), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future+timedelta(minutes=1000), 'tentative')['usage']) # insert another claim which overlaps with both claim1 and claim2, and check the usages for various timestamps - claim3 = {'resource_id': cep4_id, 'starttime': now+timedelta(minutes=5), 'endtime': now+timedelta(minutes=25), 'status': 'tentative', 'claim_size': 1} + claim3 = {'resource_id': cep4_id, 'starttime': future+timedelta(minutes=5), 'endtime': future+timedelta(minutes=25), 'status': 'tentative', 'claim_size': 1} self.radb.insertResourceClaims(task_id, [claim3], 'foo', 1, 1) # test usages twice, once to check the usages generated by insert-triggers, and then to check usages generated by rebuild_resource_usages_from_claims for i in range(2): @@ -1356,18 +1362,18 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # make sure the usage table is wiped, so asserts fail when rebuild_resource_usages_from_claims is erroneously roll'ed back. self.radb._executeQuery('TRUNCATE resource_allocation.resource_usage;') self.radb.rebuild_resource_usages_from_claims(cep4_id, 'tentative') - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(minutes=10), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future-timedelta(minutes=10), 'tentative')['usage']) self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['starttime'], 'tentative')['usage']) self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['starttime'], 'tentative')['usage']) self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['endtime'], 'tentative')['usage']) self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['starttime'], 'tentative')['usage']) self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['endtime'], 'tentative')['usage']) self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(minutes=1000), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future+timedelta(minutes=1000), 'tentative')['usage']) # insert another claim which overlaps with claim1 and ends at the same endtime as claim3, and check the usages for various timestamps - claim4 = {'resource_id': cep4_id, 'starttime': now+timedelta(minutes=7.5), 'endtime': claim3['endtime'], 'status': 'tentative', 'claim_size': 1} + claim4 = {'resource_id': cep4_id, 'starttime': future+timedelta(minutes=7.5), 'endtime': claim3['endtime'], 'status': 'tentative', 'claim_size': 1} self.radb.insertResourceClaims(task_id, [claim4], 'foo', 1, 1) # test usages twice, once to check the usages generated by insert-triggers, and then to check usages generated by rebuild_resource_usages_from_claims for i in range(2): @@ -1375,7 +1381,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # make sure the usage table is wiped, so asserts fail when rebuild_resource_usages_from_claims is erroneously roll'ed back. self.radb._executeQuery('TRUNCATE resource_allocation.resource_usage;') self.radb.rebuild_resource_usages_from_claims(cep4_id, 'tentative') - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(minutes=10), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future-timedelta(minutes=10), 'tentative')['usage']) self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['starttime'], 'tentative')['usage']) self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['starttime'], 'tentative')['usage']) self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(cep4_id, claim4['starttime'], 'tentative')['usage']) @@ -1384,7 +1390,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['endtime'], 'tentative')['usage']) #c4_endtime should be equal to c3_endtime self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim4['endtime'], 'tentative')['usage']) #so usage should drop by 2*1 at this timestamp self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(minutes=1000), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future+timedelta(minutes=1000), 'tentative')['usage']) # insert another claim which starts when claim1 ends and last 1 minute, and check the usages for various timestamps claim5 = {'resource_id': cep4_id, 'starttime': claim1['endtime'], 'endtime': claim1['endtime']+timedelta(minutes=1), 'status': 'tentative', 'claim_size': 1} @@ -1395,7 +1401,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # make sure the usage table is wiped, so asserts fail when rebuild_resource_usages_from_claims is erroneously roll'ed back. self.radb._executeQuery('TRUNCATE resource_allocation.resource_usage;') self.radb.rebuild_resource_usages_from_claims(cep4_id, 'tentative') - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(minutes=10), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future-timedelta(minutes=10), 'tentative')['usage']) self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['starttime'], 'tentative')['usage']) self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['starttime'], 'tentative')['usage']) self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(cep4_id, claim4['starttime'], 'tentative')['usage']) @@ -1406,7 +1412,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['endtime'], 'tentative')['usage']) self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim4['endtime'], 'tentative')['usage']) self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(minutes=1000), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future+timedelta(minutes=1000), 'tentative')['usage']) # last edge case, insert another claim which starts when first claim starts, and end when last claim ends. Should lift all usages by 1 (except outer ones). claim6 = {'resource_id': cep4_id, 'starttime': claim1['starttime'], 'endtime': claim2['endtime'], 'status': 'tentative', 'claim_size': 1} @@ -1417,7 +1423,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # make sure the usage table is wiped, so asserts fail when rebuild_resource_usages_from_claims is erroneously roll'ed back. self.radb._executeQuery('TRUNCATE resource_allocation.resource_usage;') self.radb.rebuild_resource_usages_from_claims(cep4_id, 'tentative') - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(minutes=10), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future-timedelta(minutes=10), 'tentative')['usage']) self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['starttime'], 'tentative')['usage']) self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['starttime'], 'tentative')['usage']) self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(cep4_id, claim4['starttime'], 'tentative')['usage']) @@ -1428,7 +1434,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['endtime'], 'tentative')['usage']) #c4_endtime should be equal to c3_endtime self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim4['endtime'], 'tentative')['usage']) #so usage should drop by 2*1 at this timestamp self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(minutes=1000), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future+timedelta(minutes=1000), 'tentative')['usage']) # conclude with two simple cases, # first final simple case: insert another claim follows (non overlapping/non-touching) all others @@ -1441,7 +1447,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): logger.info('') self.radb._executeQuery('TRUNCATE resource_allocation.resource_usage;') self.radb.rebuild_resource_usages_from_claims(cep4_id, 'tentative') - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(minutes=10), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future-timedelta(minutes=10), 'tentative')['usage']) self.assertEqual( 2, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['starttime'], 'tentative')['usage']) self.assertEqual( 3, self.radb.get_resource_usage_at_or_before(cep4_id, claim3['starttime'], 'tentative')['usage']) self.assertEqual( 4, self.radb.get_resource_usage_at_or_before(cep4_id, claim4['starttime'], 'tentative')['usage']) @@ -1454,7 +1460,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['endtime'], 'tentative')['usage']) self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim7['starttime'], 'tentative')['usage']) self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim7['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(minutes=1000), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future+timedelta(minutes=1000), 'tentative')['usage']) # second final simple case: insert another claim which precedes (non overlapping/non-touching) all the others claim8 = {'resource_id': cep4_id, 'starttime': claim1['starttime']-timedelta(minutes=20), 'endtime': claim1['starttime']-timedelta(minutes=10), 'status': 'tentative', 'claim_size': 1} @@ -1465,7 +1471,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # make sure the usage table is wiped, so asserts fail when rebuild_resource_usages_from_claims is erroneously roll'ed back. self.radb._executeQuery('TRUNCATE resource_allocation.resource_usage;') self.radb.rebuild_resource_usages_from_claims(cep4_id, 'tentative') - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(minutes=100), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future-timedelta(minutes=100), 'tentative')['usage']) self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim8['starttime'], 'tentative')['usage']) self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim8['endtime'], 'tentative')['usage']) self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim1['starttime']-timedelta(minutes=1), 'tentative')['usage']) @@ -1481,7 +1487,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim2['endtime'], 'tentative')['usage']) self.assertEqual( 1, self.radb.get_resource_usage_at_or_before(cep4_id, claim7['starttime'], 'tentative')['usage']) self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, claim7['endtime'], 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(minutes=1000), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future+timedelta(minutes=1000), 'tentative')['usage']) def test_overlapping_claims(self): # this is a special testcase to prove a bug found at 2017-08-16 @@ -1497,8 +1503,14 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): now = datetime.utcnow() now -= timedelta(minutes=now.minute, seconds=now.second, microseconds=now.microsecond) # round to full hour + #nothing is claimed yet, so the claimable capacity should be 100 as well + self.assertEqual(100, self.radb.get_resource_claimable_capacity(cep4_id, now-timedelta(days=1.0), now+timedelta(days=1.0))['claimable_capacity']) + + # we'll schedule some tasks in the future... + future = now + timedelta(hours=2) + #insert one task, and reuse that for multiple overlapping claims - result = self.radb.insertSpecificationAndTask(0, 0, 'approved', 'observation', now, now+timedelta(hours=1), 'foo', 'CEP4') + result = self.radb.insertSpecificationAndTask(0, 0, 'approved', 'observation', future, future+timedelta(hours=1), 'foo', 'CEP4') self.assertTrue(result['inserted']) task_id = result['task_id'] @@ -1508,13 +1520,13 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): #create two overlapping claims claims = [ { 'resource_id': cep4_id, - 'starttime': now, - 'endtime': now+timedelta(hours=0.75), + 'starttime': future, + 'endtime': future+timedelta(hours=0.75), 'status': 'tentative', 'claim_size': 40 }, {'resource_id': cep4_id, - 'starttime': now+timedelta(hours=0.25), - 'endtime': now + timedelta(hours=1), + 'starttime': future+timedelta(hours=0.25), + 'endtime': future + timedelta(hours=1), 'status': 'tentative', 'claim_size': 40} ] @@ -1537,18 +1549,18 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual('claimed', claim['status']) # check the resource usage trend - logger.info("resource usages:\n%s", pformat(self.radb.getResourceUsages(now-timedelta(hours=1.0), now+timedelta(hours=2.0), cep4_id))) - self.assertEqual(0, self.radb.get_max_resource_usage_between(cep4_id, now-timedelta(hours=1.0), now-timedelta(hours=0.01), 'claimed')['usage']) - self.assertEqual(40, self.radb.get_max_resource_usage_between(cep4_id, now+timedelta(hours=0.0), now+timedelta(hours=0.2), 'claimed')['usage']) - self.assertEqual(80, self.radb.get_max_resource_usage_between(cep4_id, now+timedelta(hours=0.3), now+timedelta(hours=0.6), 'claimed')['usage']) - self.assertEqual(40, self.radb.get_max_resource_usage_between(cep4_id, now+timedelta(hours=0.80), now+timedelta(hours=1.0), 'claimed')['usage']) + logger.info("resource usages:\n%s", pformat(self.radb.getResourceUsages(future-timedelta(hours=1.0), future+timedelta(hours=2.0), cep4_id))) + self.assertEqual(0, self.radb.get_max_resource_usage_between(cep4_id, future-timedelta(hours=1.0), future-timedelta(hours=0.01), 'claimed')['usage']) + self.assertEqual(40, self.radb.get_max_resource_usage_between(cep4_id, future+timedelta(hours=0.0), future+timedelta(hours=0.2), 'claimed')['usage']) + self.assertEqual(80, self.radb.get_max_resource_usage_between(cep4_id, future+timedelta(hours=0.3), future+timedelta(hours=0.6), 'claimed')['usage']) + self.assertEqual(40, self.radb.get_max_resource_usage_between(cep4_id, future+timedelta(hours=0.80), future+timedelta(hours=1.0), 'claimed')['usage']) #check for a time range encapsulating the full task - self.assertEqual(80, self.radb.get_max_resource_usage_between(cep4_id, now+timedelta(hours=-0.1), now+timedelta(hours=1.1), 'claimed')['usage']) + self.assertEqual(80, self.radb.get_max_resource_usage_between(cep4_id, future+timedelta(hours=-0.1), future+timedelta(hours=1.1), 'claimed')['usage']) #check for a time range not including the task - self.assertEqual(0, self.radb.get_max_resource_usage_between(cep4_id, now+timedelta(hours=1.1), now+timedelta(hours=2.0), 'claimed')['usage']) - self.assertEqual(0, self.radb.get_max_resource_usage_between(cep4_id, now-timedelta(hours=1.1), now-timedelta(hours=1.0), 'claimed')['usage']) + self.assertEqual(0, self.radb.get_max_resource_usage_between(cep4_id, future+timedelta(hours=1.1), future+timedelta(hours=2.0), 'claimed')['usage']) + self.assertEqual(0, self.radb.get_max_resource_usage_between(cep4_id, future-timedelta(hours=1.1), future-timedelta(hours=1.0), 'claimed')['usage']) # check that there are no overlapping conflicting claims/tasks for claim in claims: @@ -1561,24 +1573,24 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual(0, len(self.radb.get_overlapping_tasks(claim['id'], 'tentative'))) #check claimable_capacity for various timestamps - self.assertEqual(100, self.radb.get_resource_claimable_capacity(cep4_id, now-timedelta(hours=1.0), now-timedelta(hours=1.0))['claimable_capacity']) - self.assertEqual(60, self.radb.get_resource_claimable_capacity(cep4_id, now+timedelta(hours=0.0), now+timedelta(hours=0.0))['claimable_capacity']) - self.assertEqual(60, self.radb.get_resource_claimable_capacity(cep4_id, now+timedelta(hours=0.2), now+timedelta(hours=0.2))['claimable_capacity']) - self.assertEqual(20, self.radb.get_resource_claimable_capacity(cep4_id, now+timedelta(hours=0.3), now+timedelta(hours=0.3))['claimable_capacity']) - self.assertEqual(20, self.radb.get_resource_claimable_capacity(cep4_id, now+timedelta(hours=0.5), now+timedelta(hours=0.5))['claimable_capacity']) - self.assertEqual(60, self.radb.get_resource_claimable_capacity(cep4_id, now+timedelta(hours=0.75), now+timedelta(hours=0.75))['claimable_capacity']) - self.assertEqual(60, self.radb.get_resource_claimable_capacity(cep4_id, now+timedelta(hours=0.8), now+timedelta(hours=0.8))['claimable_capacity']) - self.assertEqual(100, self.radb.get_resource_claimable_capacity(cep4_id, now+timedelta(hours=1.0), now+timedelta(hours=1.0))['claimable_capacity']) - self.assertEqual(100, self.radb.get_resource_claimable_capacity(cep4_id, now+timedelta(hours=10.0), now+timedelta(hours=10.0))['claimable_capacity']) + self.assertEqual(100, self.radb.get_resource_claimable_capacity(cep4_id, future-timedelta(hours=1.0), future-timedelta(hours=1.0))['claimable_capacity']) + self.assertEqual(60, self.radb.get_resource_claimable_capacity(cep4_id, future+timedelta(hours=0.0), future+timedelta(hours=0.0))['claimable_capacity']) + self.assertEqual(60, self.radb.get_resource_claimable_capacity(cep4_id, future+timedelta(hours=0.2), future+timedelta(hours=0.2))['claimable_capacity']) + self.assertEqual(20, self.radb.get_resource_claimable_capacity(cep4_id, future+timedelta(hours=0.3), future+timedelta(hours=0.3))['claimable_capacity']) + self.assertEqual(20, self.radb.get_resource_claimable_capacity(cep4_id, future+timedelta(hours=0.5), future+timedelta(hours=0.5))['claimable_capacity']) + self.assertEqual(60, self.radb.get_resource_claimable_capacity(cep4_id, future+timedelta(hours=0.75), future+timedelta(hours=0.75))['claimable_capacity']) + self.assertEqual(60, self.radb.get_resource_claimable_capacity(cep4_id, future+timedelta(hours=0.8), future+timedelta(hours=0.8))['claimable_capacity']) + self.assertEqual(100, self.radb.get_resource_claimable_capacity(cep4_id, future+timedelta(hours=1.0), future+timedelta(hours=1.0))['claimable_capacity']) + self.assertEqual(100, self.radb.get_resource_claimable_capacity(cep4_id, future+timedelta(hours=10.0), future+timedelta(hours=10.0))['claimable_capacity']) #check claimable_capacity for full task's timewindow (+extra) - self.assertEqual(20, self.radb.get_resource_claimable_capacity(cep4_id, now-timedelta(hours=10.0), now+timedelta(hours=10.0))['claimable_capacity']) + self.assertEqual(20, self.radb.get_resource_claimable_capacity(cep4_id, future-timedelta(hours=10.0), future+timedelta(hours=10.0))['claimable_capacity']) #add an extra claim, overlapping with only the last claim of size 40. So it should fit (100-40=60 and 60>30). extra_claim = { 'resource_id': cep4_id, - 'starttime': now+timedelta(hours=0.8), - 'endtime': now+timedelta(hours=0.9), + 'starttime': future+timedelta(hours=0.8), + 'endtime': future+timedelta(hours=0.9), 'status': 'tentative', 'claim_size': 30 } extra_claim_ids = self.radb.insertResourceClaims(task_id, [extra_claim], 'foo', 1, 1) @@ -1614,8 +1626,11 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): now = datetime.utcnow() now -= timedelta(minutes=now.minute, seconds=now.second, microseconds=now.microsecond) # round to full hour + # we'll schedule some tasks in the future... + future = now + timedelta(hours=2) + # insert one task, and reuse that for multiple overlapping claims - result = self.radb.insertSpecificationAndTask(0, 0, 'approved', 'observation', now, now + timedelta(hours=1), 'first content', + result = self.radb.insertSpecificationAndTask(0, 0, 'approved', 'observation', future, future + timedelta(hours=1), 'first content', 'CEP4') self.assertTrue(result['inserted']) task_id = result['task_id'] @@ -1626,7 +1641,7 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual('first content', self.radb.getSpecification(task['specification_id'])['content']) # prove that we can re-insert the spec/task, and that the new task is indeed inserted and new - result = self.radb.insertSpecificationAndTask(0, 0, 'approved', 'observation', now, now + timedelta(hours=1), 'second content', + result = self.radb.insertSpecificationAndTask(0, 0, 'approved', 'observation', future, future + timedelta(hours=1), 'second content', 'CEP4') self.assertTrue(result['inserted']) self.assertNotEqual(task_id, result['task_id']) # we should have a new id because it was re-inserted @@ -1657,13 +1672,13 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): for claim in self.radb.getResourceClaims(claim_ids=claim_ids): self.assertEqual('claimed', claim['status']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(hours=0.5), 'claimed')['usage']) - self.assertEqual(40, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(hours=0.5), 'claimed')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(hours=1.5), 'claimed')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future-timedelta(hours=0.5), 'claimed')['usage']) + self.assertEqual(40, self.radb.get_resource_usage_at_or_before(cep4_id, future+timedelta(hours=0.5), 'claimed')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future+timedelta(hours=1.5), 'claimed')['usage']) - # prove again that we can re-insert the spec/task (now with claims), and that the new task is indeed inserted and new, + # prove again that we can re-insert the spec/task (future with claims), and that the new task is indeed inserted and new, # and that the claim(s) and usage(s) were actually deleted (via cascading deletes) - result = self.radb.insertSpecificationAndTask(0, 0, 'approved', 'observation', now, now + timedelta(hours=1), 'third content', + result = self.radb.insertSpecificationAndTask(0, 0, 'approved', 'observation', future, future + timedelta(hours=1), 'third content', 'CEP4') self.assertTrue(result['inserted']) self.assertNotEqual(task_id, result['task_id']) # we should have a new id because it was re-inserted @@ -1676,10 +1691,10 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # this newly inserted spec/task should have no claims anymore self.assertEqual( 0, len(self.radb.getResourceClaims())) - # and all usages should now be 0 - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(hours=0.5), 'claimed')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(hours=0.5), 'claimed')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(hours=1.5), 'claimed')['usage']) + # and all usages should future be 0 + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future-timedelta(hours=0.5), 'claimed')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future+timedelta(hours=0.5), 'claimed')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future+timedelta(hours=1.5), 'claimed')['usage']) # 2017-08-29: ok, we could not reproduce the bug found on production, # so it seems there is a strange corner case which caused the usages table to become inconsistent. @@ -1698,25 +1713,25 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): self.assertEqual(1, len(self.radb.getResourceClaims(claim_ids=claim_ids))) # usages should be ok - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(hours=0.5), 'tentative')['usage']) - self.assertEqual(40, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(hours=0.5), 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(hours=1.5), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future-timedelta(hours=0.5), 'tentative')['usage']) + self.assertEqual(40, self.radb.get_resource_usage_at_or_before(cep4_id, future+timedelta(hours=0.5), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future+timedelta(hours=1.5), 'tentative')['usage']) - # now let's break the usages table (intentionally, to mimic the production inconsistenty) + # now, let's break the usages table (intentionally, to mimic the production inconsistenty) # we shift one entry (where the claim starts) by one minute # result should be that when we change the claim (or delete the claim cause it's task/spec was deleted), # that the associated usage at the claim's starttime cannot be found anymore (which is the bug on production). - self.radb._executeQuery("UPDATE resource_allocation.resource_usage SET as_of_timestamp = %s WHERE as_of_timestamp = %s;", (now+timedelta(minutes=1), now)) + self.radb._executeQuery("UPDATE resource_allocation.resource_usage SET as_of_timestamp = %s WHERE as_of_timestamp = %s;", (future+timedelta(minutes=1), future)) # check that the usages were indeed changed (the first one shifted in time) usages = self.radb.getResourceUsages()[cep4_id]['tentative'] - self.assertEqual({'usage': 40, 'as_of_timestamp': now+timedelta(minutes=1)}, usages[0]) + self.assertEqual({'usage': 40, 'as_of_timestamp': future+timedelta(minutes=1)}, usages[0]) self.assertEqual({'usage': 0, 'as_of_timestamp': task['endtime']}, usages[1]) - # and prove again that we can re-insert the spec/task (now with claims and a corrupted usage table), and that the new task is indeed inserted and new, + # and prove again that we can re-insert the spec/task (future with claims and a corrupted usage table), and that the new task is indeed inserted and new, # and that the claim(s) and usage(s) were actually deleted (via cascading deletes) # 2017-08-29: YEAH! the insert fails just like on production. Now we can start making a fix! - result = self.radb.insertSpecificationAndTask(0, 0, 'approved', 'observation', now, now + timedelta(hours=1), 'fourth content', + result = self.radb.insertSpecificationAndTask(0, 0, 'approved', 'observation', future, future + timedelta(hours=1), 'fourth content', 'CEP4') self.assertTrue(result['inserted']) self.assertNotEqual(task_id, result['task_id']) # we should have a new id because it was re-inserted @@ -1729,10 +1744,10 @@ class ResourceAssignmentDatabaseTest(unittest.TestCase): # this newly inserted spec/task should have no claims anymore self.assertEqual( 0, len(self.radb.getResourceClaims())) - # and all usages should now be 0 - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now-timedelta(hours=0.5), 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(hours=0.5), 'tentative')['usage']) - self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, now+timedelta(hours=1.5), 'tentative')['usage']) + # and all usages should future be 0 + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future-timedelta(hours=0.5), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future+timedelta(hours=0.5), 'tentative')['usage']) + self.assertEqual( 0, self.radb.get_resource_usage_at_or_before(cep4_id, future+timedelta(hours=1.5), 'tentative')['usage']) def test_claims_on_partially_misc_filled_resource(self): # this is a special testcase to prove a bug found at 2017-08-24 -- GitLab