diff --git a/SAS/ResourceAssignment/ResourceAssignmentDatabase/radb.py b/SAS/ResourceAssignment/ResourceAssignmentDatabase/radb.py
index d11e2c17d7e83f4208ae356fd00d20218b247389..908d6a59b4d7b651b72b38a52bba9f213aaa7ba9 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentDatabase/radb.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentDatabase/radb.py
@@ -636,8 +636,26 @@ class RADatabase:
raise KeyError('No such resource_claim_property_type: %s Valid values are: %s' % (type_name, ', '.join(self.getResourceClaimPropertyTypeNames())))
+ def getResourceClaimPropertyIOTypes(self):
+ query = '''SELECT * from resource_allocation.resource_claim_property_io_type;'''
+
+ return list(self._executeQuery(query, fetch=_FETCH_ALL))
+
+ def getResourceClaimPropertyIOTypeNames(self):
+ return [x['name'] for x in self.getResourceClaimPropertyIOTypes()]
+
+ def getResourceClaimPropertyIOTypeId(self, io_type_name):
+ query = '''SELECT id from resource_allocation.resource_claim_property_io_type
+ WHERE name = %s;'''
+ result = self._executeQuery(query, [io_type_name], fetch=_FETCH_ONE)
+
+ if result:
+ return result['id']
+
+ raise KeyError('No such resource_claim_property_io_type: %s Valid values are: %s' % (io_type_name, ', '.join(self.getResourceClaimPropertyIOTypeNames())))
+
def getResourceClaimProperties(self, claim_ids=None, task_id=None):
- query = '''SELECT rcpv.id, rcpv.resource_claim_id, rcpv.value, rcpv.type_id, rcpv.type_name, sap.number as sap_nr
+ query = '''SELECT rcpv.id, rcpv.resource_claim_id, rcpv.value, rcpv.type_id, rcpv.type_name, rcpv.io_type_id, rcpv.io_type_name, sap.number as sap_nr
FROM resource_allocation.resource_claim_property_view rcpv
LEFT JOIN resource_allocation.sap sap on rcpv.sap_id = sap.id'''
@@ -667,15 +685,19 @@ class RADatabase:
return properties
- def insertResourceClaimProperty(self, claim_id, property_type, value, commit=True):
- return self.insertResourceClaimProperties([(claim_id, property_type, value)], commit)
+ def insertResourceClaimProperty(self, claim_id, property_type, value, io_type, commit=True):
+ return self.insertResourceClaimProperties([(claim_id, property_type, value, io_type)], commit)
def insertResourceClaimProperties(self, props, commit=True):
if not props:
return []
+ # props is a list of tuples
+ # each tuple prop is encoded as: (claim_id, type, value, io_type, sap_nr)
+ # index: (0 , 1 , 2 , 3 , 4 )
+
# first insert unique sap numbers
- claim_sap_nrs = list(set([(p[0], p[3]) for p in props if p[3] is not None]))
+ claim_sap_nrs = list(set([(p[0], p[4]) for p in props if p[4] is not None]))
sap_ids = self.insertSAPNumbers(claim_sap_nrs, False)
if sap_ids == None:
@@ -696,19 +718,25 @@ class RADatabase:
type_strings = set([p[1] for p in props if isinstance(p[1], basestring)])
type_string2id = {t:self.getResourceClaimPropertyTypeId(t) for t in type_strings}
+ # convert all property io_type strings to id's
+ io_type_strings = set([p[3] for p in props if isinstance(p[3], basestring)])
+ io_type_string2id = {t:self.getResourceClaimPropertyIOTypeId(t) for t in io_type_strings}
+
# finally we have all the info we need,
# so we can build the bulk property insert query
- insert_values = ','.join(self.cursor.mogrify('(%s, %s, %s, %s)',
+ insert_values = ','.join(self.cursor.mogrify('(%s, %s, %s, %s, %s)',
(p[0],
type_string2id[p[1]] if
isinstance(p[1], basestring) else p[1],
p[2],
- claim_id2sap_nr2sap_id[p[0]].get(p[3]) if
+ io_type_string2id[p[3]] if
+ isinstance(p[3], basestring) else p[3],
+ claim_id2sap_nr2sap_id[p[0]].get(p[4]) if
p[0] in claim_id2sap_nr2sap_id else None))
for p in props)
query = '''INSERT INTO resource_allocation.resource_claim_property
- (resource_claim_id, type_id, value, sap_id)
+ (resource_claim_id, type_id, value, io_type_id, sap_id)
VALUES {values}
RETURNING id;'''.format(values=insert_values)
@@ -929,7 +957,7 @@ class RADatabase:
properties = []
for claim_id, claim in zip(claimIds, claims):
if 'properties' in claim and len(claim['properties']) > 0:
- claim_props = [(claim_id, p['type'], p['value'], p.get('sap_nr')) for p in claim['properties']]
+ claim_props = [(claim_id, p['type'], p['value'], p.get('io_type', 0), p.get('sap_nr')) for p in claim['properties']]
properties += claim_props
if properties:
@@ -1396,25 +1424,6 @@ if __name__ == '__main__':
print '\n-- ' + str(method.__name__) + ' --'
print '\n'.join([str(x) for x in method()])
- resultPrint(db.getTasks)
-
- for t in db.getTasks():
- print db.getTask(t['id'])
-
- exit()
-
- #print db.getResourceClaims(task_id=440)
- #print
- #print db.getResourceClaims(lower_bound=datetime.utcnow() + timedelta(days=9))
- #print
- #print db.getResourceClaims(upper_bound=datetime.utcnow() + timedelta(days=19))
- #print
- #print db.getResourceClaims(status='allocated')
- #print
- #print db.getResourceClaims(status='claimed')
- #print
- #print db.getResourceClaims(resource_type='storage')
-
#resultPrint(db.getTaskStatuses)
#resultPrint(db.getTaskStatusNames)
#resultPrint(db.getTaskTypes)
@@ -1435,82 +1444,57 @@ if __name__ == '__main__':
#print db.getTaskSuccessorIds()
#resultPrint(db.getSpecifications)
#resultPrint(db.getResourceClaims)
-
- #claims = db.getResourceClaims()
- #db.updateTaskAndResourceClaims(claims[0]['task_id'], starttime=claims[1]['starttime'], endtime=claims[1]['endtime'])
- #print
- #print db.getResourceClaims()
-
- #resultPrint(db.getResourceClaims)
-
-
- db.updateResourceAvailability(0, available_capacity=2)
- exit(0)
-
- import pprint
- pprint.pprint(db.getTaskConflictReasons())
- db.updateTask(21, task_status='conflict')
- db.insertTaskConflicts(21, [1, 2, 3])
- pprint.pprint(db.getTaskConflictReasons())
- db.updateTask(21, task_status='scheduled')
- pprint.pprint(db.getTaskConflictReasons())
- db.insertTaskConflicts(21, [1, 2, 3])
- pprint.pprint(db.getTaskConflictReasons())
-
- pprint.pprint(db.getResourceClaimConflictReasons(task_ids=22))
- #pprint.pprint(db.getResourceUsages())
-
- exit(0)
-
- for s in db.getSpecifications():
- db.deleteSpecification(s['id'])
+ resultPrint(db.getResourceClaimProperties)
resources = db.getResources()
- #task_id = db.insertSpecificationAndTask(1234, 5678, 600, 0, datetime.utcnow(), datetime.utcnow() + timedelta(hours=1), "", False)['task_id']
- #task = db.getTask(task_id)
-
- #claim = {'resource_id':resources[0]['id'],
- #'starttime':task['starttime'],
- #'endtime':task['endtime'],
- #'status':'claimed',
- #'claim_size':1}
- #db.insertResourceClaims(task_id, [claim], 1, 'anonymous', -1, False)
-
- #claim = {'resource_id':resources[1]['id'],
- #'starttime':task['starttime'],
- #'endtime':task['endtime'],
- #'status':'claimed',
- #'claim_size':1,
- #'properties':[{'type':'nr_of_is_files', 'value':10},{'type':'nr_of_cs_files', 'value':20}]}
- #db.insertResourceClaims(task_id, [claim], 1, 'anonymous', -1, False)
-
- #claim = {'resource_id':resources[2]['id'],
- #'starttime':task['starttime'],
- #'endtime':task['endtime'],
- #'status':'claimed',
- #'claim_size':1,
- #'properties':[{'type':'nr_of_is_files', 'value':10, 'sap_nr':0 },
- #{'type':'nr_of_cs_files', 'value':20, 'sap_nr':0},
- #{'type':'nr_of_uv_files', 'value':30, 'sap_nr':1},]}
- #db.insertResourceClaims(task_id, [claim], 1, 'anonymous', -1, False)
-
- #claim = {'resource_id':resources[3]['id'],
- #'starttime':task['starttime'],
- #'endtime':task['endtime'],
- #'status':'claimed',
- #'claim_size':1,
- #'properties':[{'type':'nr_of_is_files', 'value':15 },
- #{'type':'nr_of_cs_files', 'value':25 },
- #{'type':'nr_of_is_files', 'value':10, 'sap_nr':0 },
- #{'type':'nr_of_cs_files', 'value':20, 'sap_nr':0},
- #{'type':'nr_of_uv_files', 'value':30, 'sap_nr':1},]}
- #db.insertResourceClaims(task_id, [claim], 1, 'anonymous', -1, False)
+ task_id = db.insertSpecificationAndTask(1234, 5678, 600, 0, datetime.utcnow(), datetime.utcnow() + timedelta(hours=1), "", False)['task_id']
+ task = db.getTask(task_id)
+
+ claim = {'resource_id':resources[0]['id'],
+ 'starttime':task['starttime'],
+ 'endtime':task['endtime'],
+ 'status':'claimed',
+ 'claim_size':1}
+ db.insertResourceClaims(task_id, [claim], 1, 'anonymous', -1, False)
+
+ claim = {'resource_id':resources[1]['id'],
+ 'starttime':task['starttime'],
+ 'endtime':task['endtime'],
+ 'status':'claimed',
+ 'claim_size':1,
+ 'properties':[{'type':'nr_of_is_files', 'value':10},{'type':'nr_of_cs_files', 'value':20}]}
+ db.insertResourceClaims(task_id, [claim], 1, 'anonymous', -1, False)
+
+ claim = {'resource_id':resources[2]['id'],
+ 'starttime':task['starttime'],
+ 'endtime':task['endtime'],
+ 'status':'claimed',
+ 'claim_size':1,
+ 'properties':[{'type':'nr_of_is_files', 'value':10, 'sap_nr':0 },
+ {'type':'nr_of_cs_files', 'value':20, 'sap_nr':0},
+ {'type':'nr_of_uv_files', 'value':30, 'sap_nr':1},]}
+ db.insertResourceClaims(task_id, [claim], 1, 'anonymous', -1, False)
+
+ claim = {'resource_id':resources[3]['id'],
+ 'starttime':task['starttime'],
+ 'endtime':task['endtime'],
+ 'status':'claimed',
+ 'claim_size':1,
+ 'properties':[{'type':'nr_of_is_files', 'value':15 },
+ {'type':'nr_of_cs_files', 'value':25 },
+ {'type':'nr_of_is_files', 'value':10, 'sap_nr':0 },
+ {'type':'nr_of_cs_files', 'value':20, 'sap_nr':0},
+ {'type':'nr_of_uv_files', 'value':30, 'sap_nr':1},]}
+ db.insertResourceClaims(task_id, [claim], 1, 'anonymous', -1, False)
+
+ db.commit()
+ import pprint
+ pprint.pprint(db.getResourceClaims(include_properties=True))
+ print '\n'.join(str(x) for x in db.getResourceClaims(include_properties=True))
- #db.commit()
- #import pprint
- #pprint.pprint(db.getResourceClaims(include_properties=True))
- #print '\n'.join(str(x) for x in db.getResourceClaims(include_properties=True))
+ db.deleteSpecification(task['specification_id'])
+ exit()
#c = db.cursor
diff --git a/SAS/ResourceAssignment/ResourceAssignmentDatabase/sql/add_resource_allocation_statics.sql b/SAS/ResourceAssignment/ResourceAssignmentDatabase/sql/add_resource_allocation_statics.sql
index 47919fcdfa31b10631461d470ddad578bc7e6752..8c2ba9cfcaf08bc082d19cc560245431aa9e3962 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentDatabase/sql/add_resource_allocation_statics.sql
+++ b/SAS/ResourceAssignment/ResourceAssignmentDatabase/sql/add_resource_allocation_statics.sql
@@ -8,6 +8,7 @@ INSERT INTO resource_allocation.task_status VALUES (200, 'prepared'), (300, 'app
INSERT INTO resource_allocation.task_type VALUES (0, 'observation'),(1, 'pipeline'); -- We'll need more types
INSERT INTO resource_allocation.resource_claim_status VALUES (0, 'claimed'), (1, 'allocated'), (2, 'conflict');
INSERT INTO resource_allocation.resource_claim_property_type VALUES (0, 'nr_of_is_files'),(1, 'nr_of_cs_files'),(2, 'nr_of_uv_files'),(3, 'nr_of_im_files'),(4, 'nr_of_img_files'),(5, 'nr_of_pulp_files'),(6, 'nr_of_cs_stokes'),(7, 'nr_of_is_stokes'),(8, 'is_file_size'),(9, 'cs_file_size'),(10, 'uv_file_size'),(11, 'im_file_size'),(12, 'img_file_size'),(13, 'nr_of_pulp_files'),(14, 'nr_of_tabs');
+INSERT INTO resource_allocation.resource_claim_property_io_type VALUES (0, 'output'),(1, 'input');
INSERT INTO resource_allocation.config VALUES (0, 'max_fill_percentage_cep4', '85.00'), (1, 'claim_timeout', '172800'), (2, 'min_inter_task_delay', '60'); -- Just some values 172800 is two days in seconds
INSERT INTO resource_allocation.conflict_reason
VALUES
diff --git a/SAS/ResourceAssignment/ResourceAssignmentDatabase/sql/create_database.sql b/SAS/ResourceAssignment/ResourceAssignmentDatabase/sql/create_database.sql
index 5002a2e19c13d8db906f5acef3b7dc43f558c274..be6f730c23a8878340a83754540b4c6ede44e081 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentDatabase/sql/create_database.sql
+++ b/SAS/ResourceAssignment/ResourceAssignmentDatabase/sql/create_database.sql
@@ -31,6 +31,7 @@ DROP TABLE IF EXISTS resource_monitoring.resource_availability CASCADE;
DROP TABLE IF EXISTS resource_monitoring.resource_capacity CASCADE;
DROP TABLE IF EXISTS resource_allocation.resource_claim_property CASCADE;
DROP TABLE IF EXISTS resource_allocation.resource_claim_property_type CASCADE;
+DROP TABLE IF EXISTS resource_allocation.resource_claim_property_io_type CASCADE;
DROP TABLE IF EXISTS resource_allocation.sap CASCADE;
DROP TABLE IF EXISTS resource_allocation.conflict_reason CASCADE;
DROP TABLE IF EXISTS resource_allocation.resource_claim_conflict_reason CASCADE;
@@ -242,11 +243,20 @@ CREATE TABLE resource_allocation.resource_claim_property_type (
ALTER TABLE resource_allocation.resource_claim_property_type
OWNER TO resourceassignment;
+CREATE TABLE resource_allocation.resource_claim_property_io_type (
+ id serial NOT NULL,
+ name text NOT NULL,
+ PRIMARY KEY (id)
+) WITH (OIDS=FALSE);
+ALTER TABLE resource_allocation.resource_claim_property_io_type
+ OWNER TO resourceassignment;
+
CREATE TABLE resource_allocation.resource_claim_property (
id serial NOT NULL,
resource_claim_id integer NOT NULL REFERENCES resource_allocation.resource_claim ON DELETE CASCADE DEFERRABLE INITIALLY IMMEDIATE,
sap_id integer REFERENCES resource_allocation.sap ON DELETE CASCADE DEFERRABLE INITIALLY IMMEDIATE,
type_id integer NOT NULL REFERENCES resource_allocation.resource_claim_property_type DEFERRABLE INITIALLY IMMEDIATE,
+ io_type_id integer NOT NULL DEFAULT 0 REFERENCES resource_allocation.resource_claim_property_io_type DEFERRABLE INITIALLY IMMEDIATE,
value bigint NOT NULL DEFAULT 1,
PRIMARY KEY (id)
) WITH (OIDS=FALSE);
@@ -345,14 +355,16 @@ COMMENT ON VIEW resource_allocation.resource_claim_extended_view
IS 'extended view on resource_claim table, including resource_claim_status.name and the resource itself';
CREATE OR REPLACE VIEW resource_allocation.resource_claim_property_view AS
- SELECT rcp.id, rcp.resource_claim_id, rcp.value, rcp.type_id,
- rcpt.name AS type_name, rcp.sap_id
+ SELECT rcp.id, rcp.resource_claim_id, rcp.value, rcp.sap_id,
+ rcp.type_id, rcpt.name AS type_name,
+ rcp.io_type_id, rcpiot.name AS io_type_name
FROM resource_allocation.resource_claim_property rcp
- JOIN resource_allocation.resource_claim_property_type rcpt ON rcpt.id = rcp.type_id;
+ JOIN resource_allocation.resource_claim_property_type rcpt ON rcpt.id = rcp.type_id
+ JOIN resource_allocation.resource_claim_property_io_type rcpiot ON rcpiot.id = rcp.io_type_id;
ALTER VIEW resource_allocation.resource_claim_property_view
OWNER TO resourceassignment;
COMMENT ON VIEW resource_allocation.resource_claim_property_view
- IS 'plain view on resource_claim_property table, including resource_claim_property_type.name';
+ IS 'plain view on resource_claim_property table, including resource_claim_property_type.name and resource_claim_property_io_type.name';
CREATE OR REPLACE VIEW resource_monitoring.resource_view AS
SELECT rv.*,