Task #8721: SW-280: processed review comment: removed user input and...

Task #8721: SW-280: processed review comment: removed user input and possibility to wipe production database. performance test now runs on a fresh database test instance

LTA/ltastorageoverview/test/db_performance_test.py

@@ -29,59 +29,79 @@ logger = logging.getLogger()
 def main():
     from optparse import OptionParser
     from lofar.common import dbcredentials
+    import testing.postgresql
+    import psycopg2
 
     # Check the invocation arguments
-    parser = OptionParser("%prog [options]", description='wipes the lta storageoverview database, and executes a performance test by inserting many files.')
+    parser = OptionParser("%prog [options]", description='execute a performance test by inserting many files on an empty test database.')
     parser.add_option('-V', '--verbose', dest='verbose', action='store_true', help='verbose logging')
-    parser.add_option_group(dbcredentials.options_group(parser))
-    parser.set_defaults(dbcredentials="LTASO")
     (options, args) = parser.parse_args()
 
     logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
                         level=logging.DEBUG if options.verbose else logging.INFO)
 
-    dbcreds = dbcredentials.parse_options(options)
 
-    logger.info("Using dbcreds: %s" % dbcreds.stringWithHiddenPassword())
+    # create a test webservice.db
+    logger.info('  creating test postgres server')
+    with testing.postgresql.Postgresql() as test_psql:
+        dsn = test_psql.dsn()
+        logger.info('  created test postgres server, dsn=%s', dsn)
 
-    base_date = datetime.utcnow()
+        dbcreds = dbcredentials.Credentials()
+        dbcreds.user = 'test_user'
+        dbcreds.password = 'test_password'
 
-    db = store.LTAStorageDb(dbcreds, options.verbose)
-    if raw_input('Are you sure you want to wipe the database? <y>/<n>:').lower() not in ['yes', 'y']:
-        return
+        with psycopg2.connect(**dsn) as conn:
+            cursor = conn.cursor()
+            #use same user/pass as stored in local webservice.dbcreds
+            query = "CREATE USER %s WITH SUPERUSER PASSWORD '%s'" % (dbcreds.user, dbcreds.password)
+            cursor.execute(query)
 
-    db.executeQuery('TRUNCATE lta.site CASCADE;', store.FETCH_NONE)
-    db.insertSite('sara', 'srm://srm.siteA.nl:8444')
-    rootdir_id = db.insertRootDirectory('sara', '/pnfs/grid.siteA.nl/data/lofar/ops')
-    projects_dir_id = db.insertSubDirectory(rootdir_id, '/pnfs/grid.siteA.nl/data/lofar/ops/projects')
+            create_script_path = os.path.normpath(os.path.join(os.environ['LOFARROOT'], 'share', 'ltaso', 'create_db_ltastorageoverview.sql'))
+            logger.info('  running ltaso create script create_script=%s', create_script_path)
+            with open(create_script_path, 'r') as script:
+                cursor.execute(script.read())
+            logger.info('  completed ltaso create script')
 
-    # junk_dir_id = db.insertSubDirectory(rootdir_id, '/pnfs/grid.siteA.nl/data/lofar/ops/misc')
-    # fileinfos = [('testfile%d' % (i,), i+100, base_date, junk_dir_id) for i in range(0, 43)]
-    # file_ids = db.insertFileInfos(fileinfos)
-    # total_num_files_inserted = len(file_ids)
-    total_num_files_inserted = 0
+        # copy the test postgres server settings into webservice.dbcreds
+        # we can use these webservice.dbcreds in each test method to connect to the testing ltaso database
+        dbcreds.host = dsn['host']
+        dbcreds.database = dsn['database']
+        dbcreds.port = dsn['port']
 
-    with open('db_perf.csv', 'w') as file:
-        for cycle_nr in range(1, 10):
-            for project_nr in range(1, 10):
-                # project_name = 'lc%d_%03d/%d' % (cycle_nr, project_nr, os.getpid())
-                project_name = 'lc%d_%03d' % (cycle_nr, project_nr)
-                projectdir_id = db.insertSubDirectory(projects_dir_id, '/pnfs/grid.siteA.nl/data/lofar/ops/projects/%s' % (project_name,))
+        logger.info('finished setting up test LTASO database')
 
-                obs_base_id = cycle_nr*100000+project_nr*1000
-                for obs_nr, obsId in enumerate(range(obs_base_id, obs_base_id+20)):
-                    obsName = 'L%s' % obsId
+        base_date = datetime.utcnow()
 
-                    obsdir_id = db.insertSubDirectory(projectdir_id, '/pnfs/grid.siteA.nl/data/lofar/ops/projects/%s/%s' % (project_name, obsName))
+        db = store.LTAStorageDb(dbcreds, options.verbose)
 
-                    fileinfos = [('%s_SB%3d' % (obsName, sbNr), 1000+sbNr+project_nr*cycle_nr, base_date + timedelta(days=10*cycle_nr+project_nr, minutes=obs_nr, seconds=sbNr), obsdir_id) for sbNr in range(0, 2)]
-                    now = datetime.utcnow()
-                    file_ids = db.insertFileInfos(fileinfos)
-                    total_num_files_inserted += len(file_ids)
-                    elapsed = totalSeconds(datetime.utcnow() - now)
-                    line = '%s,%s' % (total_num_files_inserted, elapsed)
-                    print line
-                    file.write(line + '\n')
+        db.insertSite('sara', 'srm://srm.siteA.nl:8444')
+        rootdir_id = db.insertRootDirectory('sara', '/pnfs/grid.siteA.nl/data/lofar/ops')
+        projects_dir_id = db.insertSubDirectory(rootdir_id, '/pnfs/grid.siteA.nl/data/lofar/ops/projects')
+
+        total_num_files_inserted = 0
+
+        with open('db_perf.csv', 'w') as file:
+            for cycle_nr in range(1, 10):
+                for project_nr in range(1, 10):
+                    # project_name = 'lc%d_%03d/%d' % (cycle_nr, project_nr, os.getpid())
+                    project_name = 'lc%d_%03d' % (cycle_nr, project_nr)
+                    projectdir_id = db.insertSubDirectory(projects_dir_id, '/pnfs/grid.siteA.nl/data/lofar/ops/projects/%s' % (project_name,))
+
+                    obs_base_id = cycle_nr*100000+project_nr*1000
+                    for obs_nr, obsId in enumerate(range(obs_base_id, obs_base_id+20)):
+                        obsName = 'L%s' % obsId
+
+                        obsdir_id = db.insertSubDirectory(projectdir_id, '/pnfs/grid.siteA.nl/data/lofar/ops/projects/%s/%s' % (project_name, obsName))
+
+                        fileinfos = [('%s_SB%3d' % (obsName, sbNr), 1000+sbNr+project_nr*cycle_nr, base_date + timedelta(days=10*cycle_nr+project_nr, minutes=obs_nr, seconds=sbNr), obsdir_id) for sbNr in range(0, 2)]
+                        now = datetime.utcnow()
+                        file_ids = db.insertFileInfos(fileinfos)
+                        total_num_files_inserted += len(file_ids)
+                        elapsed = totalSeconds(datetime.utcnow() - now)
+                        line = '%s,%s' % (total_num_files_inserted, elapsed)
+                        print line
+                        file.write(line + '\n')
 
 if __name__ == "__main__":
     main()