-
Jorrit Schaap authoredJorrit Schaap authored
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
create_add_virtual_instrument.sql.py 10.79 KiB
#!/usr/bin/python
with open("add_virtual_instrument.sql", 'w+') as output:
output.write("-- This file was generated by the \'create_add_virtual_instrument.sql.py\' script.\n")
output.write("-- Please do not modify this sql file, but modify and the python script if you want a different virtual instrument.\n\n")
output.write("-- resourceassignment password for testing on mcu005 is the same as the password on the president's luggage +6\n")
output.write("-- psql resourceassignment -U resourceassignment -f add_virtual_instrument.sql -W\n")
output.write("BEGIN;\n")
output.write("-- start with clearing the old virtual instrument model\n")
output.write("-- for some reason truncate takes a looooong time to complete, so use delete from\n")
output.write("DELETE FROM virtual_instrument.resource_group_to_resource_group CASCADE;\n")
output.write("DELETE FROM virtual_instrument.resource_to_resource_group CASCADE;\n")
output.write("DELETE FROM virtual_instrument.resource_group CASCADE;\n")
output.write("DELETE FROM virtual_instrument.resource_group_type CASCADE;\n")
output.write("DELETE FROM virtual_instrument.resource CASCADE;\n")
output.write("DELETE FROM virtual_instrument.resource_type CASCADE;\n")
output.write("DELETE FROM virtual_instrument.unit CASCADE;\n")
output.write("-- end of initial cleanup\n\n")
#----- resource unit -----
output.write("INSERT INTO virtual_instrument.unit VALUES \n")
output.write("(0, 'rsp_channel_bit'),(1, 'bytes'),(2, 'rcu_board'),(3, 'bytes/second'),(4, 'cores');\n")
#----- resource type -----
output.write("INSERT INTO virtual_instrument.resource_type VALUES \n")
rsp_index = 0
tbb_index = 1
rcu_index = 2
bandwidth_index = 3
processor_index = 4
storage_index = 5
output.write("(0, 'rsp', 0), (1, 'tbb', 1), (2, 'rcu', 2), (3, 'bandwidth', 3), (4, 'processor', 4), (5, 'storage', 1);\n")
#----- resource_group type -----
output.write("INSERT INTO virtual_instrument.resource_group_type VALUES \n")
output.write("(0, 'instrument'),(1, 'cluster'),(2, 'station_group'),(3, 'station'),(4, 'node_group'),(5, 'node');\n")
#----- resource groups -----
output.write("INSERT INTO virtual_instrument.resource_group VALUES \n")
##instrument
output.write("(0, 'LOFAR', 0)")
lastGroupNumber = 0
##cluster
#clusters = ['CEP2', 'CEP3', 'CEP4', 'Cobalt', 'DragNet']
clusters = ['CEP4', 'Cobalt']
clusternumbers = range(lastGroupNumber+1, lastGroupNumber+1+len(clusters))
if clusters:
lastGroupNumber += len(clusters)
output.write(', ')
output.write(',\n'.join(["(%i, '%s', 1)" % (nr, cluster) for nr, cluster in zip(clusternumbers, clusters)]))
nodegroups = ['computenodes', 'gpunodes']
nodegroupnumbers = range(lastGroupNumber+1, lastGroupNumber+1+len(nodegroups))
if nodegroups:
lastGroupNumber += len(nodegroups)
output.write(', ')
output.write(',\n'.join(["(%i, '%s', 4)" % (nr, nodegroup) for nr, nodegroup in zip(nodegroupnumbers, nodegroups)]))
##station group
#stationgroups = ['STATIONS', 'SUPERTERP', 'CORE', 'REMOTE', 'INTERNATIONAL', 'LORA', 'AARTFAAC']
stationgroups = []
stationgroupnumbers = range(lastGroupNumber+1, lastGroupNumber+1+len(stationgroups))
if stationgroups:
lastGroupNumber += len(stationgroups)
if clusters:
output.write(', ')
output.write(',\n'.join(["(%i, '%s', 2)" % (nr, stationgroup) for nr, stationgroup in zip(stationgroupnumbers, stationgroups)]))
##station this list should be imported from a central location?
##We might need to switch to CS001HBA1 CS003LBA notation at some point? This could also be a specification vs resource allocation problem.
#stations = ['CS001', 'CS002', 'CS003', 'CS004', 'CS005', 'CS006', 'CS007', 'CS011', 'CS013', 'CS017',
#'CS021', 'CS024', 'CS026', 'CS028', 'CS030', 'CS031', 'CS032', 'CS101', 'CS103', 'CS201', 'CS301',
#'CS302', 'CS401', 'CS501', 'RS106', 'RS205', 'RS208', 'RS210', 'RS305', 'RS306', 'RS307', 'RS310', 'RS406',
#'RS407', 'RS408', 'RS409', 'RS503', 'RS508', 'RS509', 'DE601', 'DE602', 'DE603', 'DE604', 'DE605', 'FR606',
#'SE607', 'UK608','DE609','PL610','PL611','PL612']
stations = []
stationnumbers = range(lastGroupNumber+1, lastGroupNumber+1+len(stations))
if stations:
lastGroupNumber += len(stations)
if clusters or stationgroups:
output.write(', ')
output.write(',\n'.join(["(%i, '%s', 3)" % (nr, station) for nr, station in zip(stationnumbers, stations)]))
##cep4 nodes
numCep4Nodes = 50
cep4numbers = range(lastGroupNumber+1, lastGroupNumber+1+numCep4Nodes)
if cep4numbers:
lastGroupNumber += len(cep4numbers)
if clusters or stationgroups or stations:
output.write(', ')
output.write(',\n'.join(["(%i, 'cpunode%02d', 5)" % (nr, node) for nr, node in zip(cep4numbers, range(1, numCep4Nodes+1))]))
##cobalt nodes
numCobaltNodes = 8
cobaltnumbers = range(lastGroupNumber+1, lastGroupNumber+1+numCobaltNodes)
if cobaltnumbers:
lastGroupNumber += len(cobaltnumbers)
if clusters or stationgroups or stations or cep4numbers:
output.write(', ')
output.write(',\n'.join(["(%i, 'cbt%03d', 5)" % (nr, node) for nr, node in zip(cobaltnumbers, range(1, numCobaltNodes+1))]))
if clusters or stationgroups or stations or cobaltnumbers:
output.write(';\n')
##node_group
#output.write("(64, 'cep4_gpu', 4);\n") ## unknown which nodes are in here? Last entry for resource groups, to make the syntax work FIXME
#----- resource & resource_to_resource_group & resource_capacity & resource_availability -----
resource_count = 0
resources = ""
resource_to_resource_group = ""
resource_capacity = ""
resource_available = ""
## stations
for station in stationnumbers:
resources += "(%i, 'rsp', %i), " % (resource_count, rsp_index)
resource_to_resource_group += "(DEFAULT, %i, %i), " % (resource_count, station)
resource_capacity += "(DEFAULT, %i, %i, %i), " % (resource_count, 61*16*4, 61*16*4) ##magic numbers FIXME
resource_available += "(DEFAULT, %i, TRUE), " % (resource_count,)
resource_count += 1
resources += "(%i, 'tbb', %i), " % (resource_count, tbb_index)
resource_to_resource_group += "(DEFAULT, %i, %i), " % (resource_count, station)
resource_capacity += "(DEFAULT, %i, %i, %i), " % (resource_count, 96*8*1024*1024*1024, 96*8*1024*1024*1024) ##magic numbers FIXME
resource_available += "(DEFAULT, %i, TRUE), " % (resource_count,)
resource_count += 1
resources += "(%i, 'rcu', %i), " % (resource_count, rcu_index)
resource_to_resource_group += "(DEFAULT, %i, %i), " % (resource_count, station)
resource_capacity += "(DEFAULT, %i, %i, %i), " % (resource_count, 96, 96) ##magic numbers FIXME, should be twice as high for INTERNATIONAL
resource_available += "(DEFAULT, %i, TRUE), " % (resource_count,)
resource_count += 1
resources += "(%i, 'bandwidth', %i), " % (resource_count, bandwidth_index)
resource_to_resource_group += "(DEFAULT, %i, %i), " % (resource_count, station)
resource_capacity += "(DEFAULT, %i, %i, %i), " % (resource_count, 3*1024*1024*1024, 3*1024*1024*1024) ##magic numbers FIXME
resource_available += "(DEFAULT, %i, TRUE), " % (resource_count,)
resource_count += 1
## nodes
for node in cep4numbers + cobaltnumbers:
resources += "(%i, 'bandwidth', %i), " % (resource_count, bandwidth_index)
resource_to_resource_group += "(DEFAULT, %i, %i), " % (resource_count, node)
resource_capacity += "(DEFAULT, %i, %i, %i), " % (resource_count, 2000000000, 2000000000) ##magic numbers FIXME
resource_available += "(DEFAULT, %i, TRUE), " % (resource_count,)
resource_count += 1
resources += "(%i, 'processor', %i), " % (resource_count, processor_index)
resource_to_resource_group += "(DEFAULT, %i, %i), " % (resource_count, node)
resource_capacity += "(DEFAULT, %i, %i, %i), " % (resource_count, 24, 24) ##magic numbers FIXME
resource_available += "(DEFAULT, %i, TRUE), " % (resource_count,)
resource_count += 1
#resources += "(%i, 'storage', %i), " % (resource_count, storage_index)
#resource_to_resource_group += "(DEFAULT, %i, %i), " % (resource_count, node)
#resource_capacity += "(DEFAULT, %i, %i, %i), " % (resource_count, 0, 0) ##magic numbers FIXME
#resource_available += "(DEFAULT, %i, TRUE), " % (resource_count,)
#resource_count += 1
## CEP4 bandwidth
resources += "(%i, 'cep4bandwidth', %i), " % (resource_count, bandwidth_index)
resource_to_resource_group += "(DEFAULT, %i, %i), " % (resource_count, 1) ##magic numbers FIXME
resource_capacity += "(DEFAULT, %i, %i, %i), " % (resource_count, 160000000000, 160000000000) ##magic numbers FIXME
resource_available += "(DEFAULT, %i, TRUE), " % (resource_count,)
resource_count += 1
## CEP4 storage
resources += "(%i, 'cep4storage', %i), " % (resource_count, storage_index)
resource_to_resource_group += "(DEFAULT, %i, %i), " % (resource_count, 1) ##magic numbers FIXME
resource_capacity += "(DEFAULT, %i, %i, %i), " % (resource_count, 2100*1024*1024*1024, 2100*1024*1024*1024) ##magic numbers FIXME
resource_available += "(DEFAULT, %i, TRUE), " % (resource_count,)
resource_count += 1
output.write("INSERT INTO virtual_instrument.resource VALUES %s;\n" % resources[:-2])
output.write("INSERT INTO virtual_instrument.resource_to_resource_group VALUES %s;\n" % resource_to_resource_group[:-2])
output.write("INSERT INTO resource_monitoring.resource_capacity VALUES %s;\n" % resource_capacity[:-2])
output.write("INSERT INTO resource_monitoring.resource_availability VALUES %s;\n" % resource_available[:-2])
#----- resource_group_to_resource_group -----
resource_group_to_resource_group = "(DEFAULT, 0, NULL), " ##LOFAR does not have parent, magic numbers FIXME
for i, station in enumerate(stations):
resource_group_to_resource_group += "(DEFAULT, %i, 6), " % (stationnumbers[i],) ## STATIONS
if "CS" in station:
resource_group_to_resource_group += "(DEFAULT, %i, 8), " % (stationnumbers[i],) ## CORE
elif "RS" in station:
resource_group_to_resource_group += "(DEFAULT, %i, 9), " % (stationnumbers[i],) ## REMOTE
else:
resource_group_to_resource_group += "(DEFAULT, %i, 10), " % (stationnumbers[i],) ## INTERNATIONAL
resource_group_to_resource_group += "(DEFAULT, 3, 1), " ## CEP4 computenodes
for node in cep4numbers:
resource_group_to_resource_group += "(DEFAULT, %i, 3), " % (node,) ## CEP4 nodes
for node in cobaltnumbers:
resource_group_to_resource_group += "(DEFAULT, %i, 2), " % (node,) ## Cobalt nodes
for cluster in clusternumbers:
resource_group_to_resource_group += "(DEFAULT, %i, 0), " % (cluster,) # clusters to LOFAR
for stationgroup in stationgroupnumbers:
resource_group_to_resource_group += "(DEFAULT, %i, 0), " % (stationgroup,) # station groups to LOFAR
output.write("INSERT INTO virtual_instrument.resource_group_to_resource_group VALUES %s;\n" % (resource_group_to_resource_group[:-2],))
output.write("COMMIT;\n")