Skip to content
Snippets Groups Projects
Commit 5a1dc1cb authored by Jorrit Schaap's avatar Jorrit Schaap
Browse files

SW-41: reuse the common radb testing code with a fresh testing database for each run

parent b4cc17c3
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/radb_performance_test.py
+ 65
− 77
View file @ 5a1dc1cb
......@@ -18,96 +18,84 @@
# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
# $Id: $
from optparse import OptionParser
import unittest
import psycopg2
import os
from datetime import datetime, timedelta
import logging
from dateutil import parser
from pprint import pformat
from random import randint
from lofar.common.datetimeutils import totalSeconds
import logging
logger = logging.getLogger(__name__)
from lofar.common import dbcredentials
from lofar.sas.resourceassignment.database.radb import RADatabase
from lofar.common.datetimeutils import totalSeconds
def test_resource_usages_performance(radb):
radb.updateResourceAvailability(117, available_capacity=10000000, total_capacity=10000000)
now = datetime.utcnow()
now -= timedelta(minutes=now.minute, seconds=now.second, microseconds=now.microsecond) # round to full hour
spec_ids = []
filename = 'resource_usages_performance%s.csv' % (datetime.utcnow().strftime('%Y%m%dT%H%M%S'),)
with open(filename, 'w') as file:
file.write('#claims, elapsed_insert, elapsed_rebuild\n')
counter = 0
for k in range(20):
num_claims_to_insert = 20
num_insert_repeats = 10
elapsed_insert = 0
for i in range(num_insert_repeats):
counter += 1
result = radb.insertSpecificationAndTask(counter, counter, 'approved', 'observation',
now+timedelta(hours=3*counter),
now + timedelta(hours=1 + 3*counter),
'content', 'CEP4')
task_id = result['task_id']
task = radb.getTask(task_id)
spec_ids.append(task['specification_id'])
claims = [{'resource_id': 117,
'starttime': task['starttime']-timedelta(minutes=randint(0, 1800)),
'endtime': task['starttime']+timedelta(seconds=randint(1801, 3600)),
'status': 'tentative',
'claim_size': q}
for q in range(num_claims_to_insert)]
import radb_common_testing
def setUpModule():
return radb_common_testing.setUpModule()
def tearDownModule():
return radb_common_testing.tearDownModule()
class ResourceAssignmentDatabaseTest(radb_common_testing.RADBCommonTest):
def test_resource_usages_performance(self):
self.radb.updateResourceAvailability(117, available_capacity=10000000, total_capacity=10000000)
now = datetime.utcnow()
now -= timedelta(minutes=now.minute, seconds=now.second, microseconds=now.microsecond) # round to full hour
spec_ids = []
filename = 'resource_usages_performance%s.csv' % (datetime.utcnow().strftime('%Y%m%dT%H%M%S'),)
with open(filename, 'w') as file:
file.write('#claims, elapsed_insert, elapsed_rebuild\n')
counter = 0
for k in range(20):
num_claims_to_insert = 20
num_insert_repeats = 10
elapsed_insert = 0
for i in range(num_insert_repeats):
counter += 1
result = self.radb.insertSpecificationAndTask(counter, counter, 'approved', 'observation',
now+timedelta(hours=3*counter),
now + timedelta(hours=1 + 3*counter),
'content', 'CEP4')
task_id = result['task_id']
task = self.radb.getTask(task_id)
spec_ids.append(task['specification_id'])
claims = [{'resource_id': 117,
'starttime': task['starttime']-timedelta(minutes=randint(0, 1800)),
'endtime': task['starttime']+timedelta(seconds=randint(1801, 3600)),
'status': 'tentative',
'claim_size': q}
for q in range(num_claims_to_insert)]
start = datetime.utcnow()
self.radb.insertResourceClaims(task_id, claims, 'foo', 1, 1)
elapsed_insert += totalSeconds(datetime.utcnow() - start)
elapsed_insert /= num_insert_repeats
start = datetime.utcnow()
radb.insertResourceClaims(task_id, claims, 'foo', 1, 1)
elapsed_insert += totalSeconds(datetime.utcnow() - start)
elapsed_insert /= 10
start = datetime.utcnow()
# make sure the usage table is wiped, so asserts fail when rebuild_resource_usages_from_claims is erroneously roll'ed back.
radb.rebuild_resource_usages_from_claims(117, 'tentative')
elapsed_rebuild = totalSeconds(datetime.utcnow() - start)
logger.info('TEST RESULT: radb now contains %d claims, insert of %d claims takes on average %.3fsec and a rebuild of the whole usage table takes %.3fsec',
len(radb.getResourceClaims()), num_claims_to_insert, elapsed_insert, elapsed_rebuild)
file.write('%d, %.3f, %.3f\n' % (len(radb.getResourceClaims()), elapsed_insert, elapsed_rebuild))
file.flush()
logger.info('removing all test specs/tasks/claims from db')
for spec_id in spec_ids:
radb.deleteSpecification(spec_id)
logger.info('Done. Results can be found in file: %s', filename)
if __name__ == '__main__':
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',level=logging.INFO)
# make sure the usage table is wiped, so asserts fail when rebuild_resource_usages_from_claims is erroneously roll'ed back.
self.radb.rebuild_resource_usages_from_claims(117, 'tentative')
elapsed_rebuild = totalSeconds(datetime.utcnow() - start)
# Check the invocation arguments
parser = OptionParser("%prog [options]", description='runs some test queries on the radb')
parser.add_option('-V', '--verbose', dest='verbose', action='store_true', help='verbose logging')
parser.add_option_group(dbcredentials.options_group(parser))
parser.set_defaults(dbcredentials="RADB")
(options, args) = parser.parse_args()
logger.info('TEST RESULT: radb now contains %d claims, insert of %d claims takes on average %.3fsec and a rebuild of the whole usage table takes %.3fsec',
len(self.radb.getResourceClaims()), num_claims_to_insert, elapsed_insert, elapsed_rebuild)
file.write('%d, %.3f, %.3f\n' % (len(self.radb.getResourceClaims()), elapsed_insert, elapsed_rebuild))
file.flush()
dbcreds = dbcredentials.parse_options(options)
logger.info('removing all test specs/tasks/claims from db')
print
print 'Using dbcreds: %s' % dbcreds.stringWithHiddenPassword()
print 'Are you sure you want to run the performance tests on this database? Tables will be modified! Precious data might be lost!'
print 'This test gives the most reproducable results when run on a clean database.'
print
answer = raw_input('CONTINUE? y/<n>: ')
if 'y' not in answer.lower():
print 'Exiting without running the test...'
exit(1)
for spec_id in spec_ids:
self.radb.deleteSpecification(spec_id)
print 'Starting test....'
radb = RADatabase(dbcreds=dbcreds, log_queries=options.verbose)
logger.info('Done. Results can be found in file: %s', filename)
test_resource_usages_performance(radb)
if __name__ == "__main__":
os.environ['TZ'] = 'UTC'
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
unittest.main()
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment