#!/usr/bin/python with open("add_virtual_instrument.sql", 'w+') as output: output.write("-- This file was generated by the \'create_add_virtual_instrument.sql.py\' script.\n") output.write("-- Please do not modify this sql file, but modify and the python script if you want a different virtual instrument.\n\n") output.write("-- resourceassignment password for testing on mcu005 is the same as the password on the president's luggage +6\n") output.write("-- psql resourceassignment -U resourceassignment -f add_virtual_instrument.sql -W\n") output.write("BEGIN;\n") output.write("-- start with clearing the old virtual instrument model\n") output.write("-- for some reason truncate takes a looooong time to complete, so use delete from\n") output.write("DELETE FROM virtual_instrument.resource_group_to_resource_group CASCADE;\n") output.write("DELETE FROM virtual_instrument.resource_to_resource_group CASCADE;\n") output.write("DELETE FROM virtual_instrument.resource_group CASCADE;\n") output.write("DELETE FROM virtual_instrument.resource_group_type CASCADE;\n") output.write("DELETE FROM virtual_instrument.resource CASCADE;\n") output.write("DELETE FROM virtual_instrument.resource_type CASCADE;\n") output.write("DELETE FROM virtual_instrument.unit CASCADE;\n") output.write("-- end of initial cleanup\n\n") #----- resource unit ----- output.write("INSERT INTO virtual_instrument.unit VALUES \n") output.write("(0, 'rsp_channel_bit'),(1, 'bytes'),(2, 'rcu_board'),(3, 'bytes/second'),(4, 'cores');\n") #----- resource type ----- output.write("INSERT INTO virtual_instrument.resource_type VALUES \n") rsp_index = 0 tbb_index = 1 rcu_index = 2 bandwidth_index = 3 processor_index = 4 storage_index = 5 output.write("(0, 'rsp', 0), (1, 'tbb', 1), (2, 'rcu', 2), (3, 'bandwidth', 3), (4, 'processor', 4), (5, 'storage', 1);\n") #----- resource_group type ----- output.write("INSERT INTO virtual_instrument.resource_group_type VALUES \n") output.write("(0, 'instrument'),(1, 'cluster'),(2, 'station_group'),(3, 'station'),(4, 'node_group'),(5, 'node');\n") #----- resource groups ----- output.write("INSERT INTO virtual_instrument.resource_group VALUES \n") ##instrument output.write("(0, 'LOFAR', 0)") lastGroupNumber = 0 ##cluster #clusters = ['CEP2', 'CEP3', 'CEP4', 'Cobalt', 'DragNet'] clusters = ['CEP4', 'Cobalt'] clusternumbers = range(lastGroupNumber+1, lastGroupNumber+1+len(clusters)) if clusters: lastGroupNumber += len(clusters) output.write(', ') output.write(',\n'.join(["(%i, '%s', 1)" % (nr, cluster) for nr, cluster in zip(clusternumbers, clusters)])) nodegroups = ['computenodes', 'gpunodes'] nodegroupnumbers = range(lastGroupNumber+1, lastGroupNumber+1+len(nodegroups)) if nodegroups: lastGroupNumber += len(nodegroups) output.write(', ') output.write(',\n'.join(["(%i, '%s', 4)" % (nr, nodegroup) for nr, nodegroup in zip(nodegroupnumbers, nodegroups)])) ##station group #stationgroups = ['STATIONS', 'SUPERTERP', 'CORE', 'REMOTE', 'INTERNATIONAL', 'LORA', 'AARTFAAC'] stationgroups = [] stationgroupnumbers = range(lastGroupNumber+1, lastGroupNumber+1+len(stationgroups)) if stationgroups: lastGroupNumber += len(stationgroups) if clusters: output.write(', ') output.write(',\n'.join(["(%i, '%s', 2)" % (nr, stationgroup) for nr, stationgroup in zip(stationgroupnumbers, stationgroups)])) ##station this list should be imported from a central location? ##We might need to switch to CS001HBA1 CS003LBA notation at some point? This could also be a specification vs resource allocation problem. #stations = ['CS001', 'CS002', 'CS003', 'CS004', 'CS005', 'CS006', 'CS007', 'CS011', 'CS013', 'CS017', #'CS021', 'CS024', 'CS026', 'CS028', 'CS030', 'CS031', 'CS032', 'CS101', 'CS103', 'CS201', 'CS301', #'CS302', 'CS401', 'CS501', 'RS106', 'RS205', 'RS208', 'RS210', 'RS305', 'RS306', 'RS307', 'RS310', 'RS406', #'RS407', 'RS408', 'RS409', 'RS503', 'RS508', 'RS509', 'DE601', 'DE602', 'DE603', 'DE604', 'DE605', 'FR606', #'SE607', 'UK608','DE609','PL610','PL611','PL612'] stations = [] stationnumbers = range(lastGroupNumber+1, lastGroupNumber+1+len(stations)) if stations: lastGroupNumber += len(stations) if clusters or stationgroups: output.write(', ') output.write(',\n'.join(["(%i, '%s', 3)" % (nr, station) for nr, station in zip(stationnumbers, stations)])) ##cep4 nodes numCep4Nodes = 50 cep4numbers = range(lastGroupNumber+1, lastGroupNumber+1+numCep4Nodes) if cep4numbers: lastGroupNumber += len(cep4numbers) if clusters or stationgroups or stations: output.write(', ') output.write(',\n'.join(["(%i, 'cpunode%02d', 5)" % (nr, node) for nr, node in zip(cep4numbers, range(1, numCep4Nodes+1))])) ##cobalt nodes numCobaltNodes = 8 cobaltnumbers = range(lastGroupNumber+1, lastGroupNumber+1+numCobaltNodes) if cobaltnumbers: lastGroupNumber += len(cobaltnumbers) if clusters or stationgroups or stations or cep4numbers: output.write(', ') output.write(',\n'.join(["(%i, 'cbt%03d', 5)" % (nr, node) for nr, node in zip(cobaltnumbers, range(1, numCobaltNodes+1))])) if clusters or stationgroups or stations or cobaltnumbers: output.write(';\n') ##node_group #output.write("(64, 'cep4_gpu', 4);\n") ## unknown which nodes are in here? Last entry for resource groups, to make the syntax work FIXME #----- resource & resource_to_resource_group & resource_capacity & resource_availability ----- resource_count = 0 resources = "" resource_to_resource_group = "" resource_capacity = "" resource_available = "" ## stations for station in stationnumbers: resources += "(%i, 'rsp', %i), " % (resource_count, rsp_index) resource_to_resource_group += "(DEFAULT, %i, %i), " % (resource_count, station) resource_capacity += "(DEFAULT, %i, %i, %i), " % (resource_count, 61*16*4, 61*16*4) ##magic numbers FIXME resource_available += "(DEFAULT, %i, TRUE), " % (resource_count,) resource_count += 1 resources += "(%i, 'tbb', %i), " % (resource_count, tbb_index) resource_to_resource_group += "(DEFAULT, %i, %i), " % (resource_count, station) resource_capacity += "(DEFAULT, %i, %i, %i), " % (resource_count, 96*8*1024*1024*1024, 96*8*1024*1024*1024) ##magic numbers FIXME resource_available += "(DEFAULT, %i, TRUE), " % (resource_count,) resource_count += 1 resources += "(%i, 'rcu', %i), " % (resource_count, rcu_index) resource_to_resource_group += "(DEFAULT, %i, %i), " % (resource_count, station) resource_capacity += "(DEFAULT, %i, %i, %i), " % (resource_count, 96, 96) ##magic numbers FIXME, should be twice as high for INTERNATIONAL resource_available += "(DEFAULT, %i, TRUE), " % (resource_count,) resource_count += 1 resources += "(%i, 'bandwidth', %i), " % (resource_count, bandwidth_index) resource_to_resource_group += "(DEFAULT, %i, %i), " % (resource_count, station) resource_capacity += "(DEFAULT, %i, %i, %i), " % (resource_count, 3*1024*1024*1024, 3*1024*1024*1024) ##magic numbers FIXME resource_available += "(DEFAULT, %i, TRUE), " % (resource_count,) resource_count += 1 ## nodes for node in cep4numbers + cobaltnumbers: resources += "(%i, 'bandwidth', %i), " % (resource_count, bandwidth_index) resource_to_resource_group += "(DEFAULT, %i, %i), " % (resource_count, node) resource_capacity += "(DEFAULT, %i, %i, %i), " % (resource_count, 2000000000, 2000000000) ##magic numbers FIXME resource_available += "(DEFAULT, %i, TRUE), " % (resource_count,) resource_count += 1 resources += "(%i, 'processor', %i), " % (resource_count, processor_index) resource_to_resource_group += "(DEFAULT, %i, %i), " % (resource_count, node) resource_capacity += "(DEFAULT, %i, %i, %i), " % (resource_count, 24, 24) ##magic numbers FIXME resource_available += "(DEFAULT, %i, TRUE), " % (resource_count,) resource_count += 1 #resources += "(%i, 'storage', %i), " % (resource_count, storage_index) #resource_to_resource_group += "(DEFAULT, %i, %i), " % (resource_count, node) #resource_capacity += "(DEFAULT, %i, %i, %i), " % (resource_count, 0, 0) ##magic numbers FIXME #resource_available += "(DEFAULT, %i, TRUE), " % (resource_count,) #resource_count += 1 ## CEP4 bandwidth resources += "(%i, 'cep4bandwidth', %i), " % (resource_count, bandwidth_index) resource_to_resource_group += "(DEFAULT, %i, %i), " % (resource_count, 1) ##magic numbers FIXME resource_capacity += "(DEFAULT, %i, %i, %i), " % (resource_count, 160000000000, 160000000000) ##magic numbers FIXME resource_available += "(DEFAULT, %i, TRUE), " % (resource_count,) resource_count += 1 ## CEP4 storage resources += "(%i, 'cep4storage', %i), " % (resource_count, storage_index) resource_to_resource_group += "(DEFAULT, %i, %i), " % (resource_count, 1) ##magic numbers FIXME resource_capacity += "(DEFAULT, %i, %i, %i), " % (resource_count, 2100*1024*1024*1024, 2100*1024*1024*1024) ##magic numbers FIXME resource_available += "(DEFAULT, %i, TRUE), " % (resource_count,) resource_count += 1 output.write("INSERT INTO virtual_instrument.resource VALUES %s;\n" % resources[:-2]) output.write("INSERT INTO virtual_instrument.resource_to_resource_group VALUES %s;\n" % resource_to_resource_group[:-2]) output.write("INSERT INTO resource_monitoring.resource_capacity VALUES %s;\n" % resource_capacity[:-2]) output.write("INSERT INTO resource_monitoring.resource_availability VALUES %s;\n" % resource_available[:-2]) #----- resource_group_to_resource_group ----- resource_group_to_resource_group = "(DEFAULT, 0, NULL), " ##LOFAR does not have parent, magic numbers FIXME for i, station in enumerate(stations): resource_group_to_resource_group += "(DEFAULT, %i, 6), " % (stationnumbers[i],) ## STATIONS if "CS" in station: resource_group_to_resource_group += "(DEFAULT, %i, 8), " % (stationnumbers[i],) ## CORE elif "RS" in station: resource_group_to_resource_group += "(DEFAULT, %i, 9), " % (stationnumbers[i],) ## REMOTE else: resource_group_to_resource_group += "(DEFAULT, %i, 10), " % (stationnumbers[i],) ## INTERNATIONAL resource_group_to_resource_group += "(DEFAULT, 3, 1), " ## CEP4 computenodes for node in cep4numbers: resource_group_to_resource_group += "(DEFAULT, %i, 3), " % (node,) ## CEP4 nodes for node in cobaltnumbers: resource_group_to_resource_group += "(DEFAULT, %i, 2), " % (node,) ## Cobalt nodes for cluster in clusternumbers: resource_group_to_resource_group += "(DEFAULT, %i, 0), " % (cluster,) # clusters to LOFAR for stationgroup in stationgroupnumbers: resource_group_to_resource_group += "(DEFAULT, %i, 0), " % (stationgroup,) # station groups to LOFAR output.write("INSERT INTO virtual_instrument.resource_group_to_resource_group VALUES %s;\n" % (resource_group_to_resource_group[:-2],)) output.write("COMMIT;\n")