diff --git a/LTA/ltastorageoverview/lib/scraper.py b/LTA/ltastorageoverview/lib/scraper.py
index 94c890666668d01229e54414a99a6a8ed238b901..96c2a39e6a4405500c9a6a84b3e5000f6d3947e5 100755
--- a/LTA/ltastorageoverview/lib/scraper.py
+++ b/LTA/ltastorageoverview/lib/scraper.py
@@ -40,7 +40,7 @@ from random import random, randint
logger = logging.getLogger()
VISIT_INTERVAL = datetime.timedelta(days=3)
-LEXAR_HOST = 'ingest@lexar004'
+LEXAR_HOST = 'ingest@lexar004.offline.lofar'
class FileInfo:
'''Simple struct to hold filename and size'''
@@ -388,20 +388,54 @@ class ResultGetterThread(threading.Thread):
db.updateDirectoryLastVisitTime(self.dir_id, datetime.datetime.utcnow() - VISIT_INTERVAL)
def populateDbWithLTASitesAndRootDirs(db):
+ """
+ Helper method to fill empty database with (hardcoded) information about our LTA partners/sites/quotas
+ """
if not db.sites():
#db.insertSite('nikhef', 'srm://tbn18.nikhef.nl:8446')
sara_id = db.insertSiteIfNotExists('sara', 'srm://srm.grid.sara.nl:8443')
juelich_id = db.insertSiteIfNotExists('juelich', 'srm://lofar-srm.fz-juelich.de:8443')
poznan_id = db.insertSiteIfNotExists('poznan', 'srm://lta-head.lofar.psnc.pl:8443')
- for site_id in [sara_id, juelich_id, poznan_id]:
- for i in range(1, 8):
- db.executeQuery('insert into lta.site_quota(site_id, quota, valid_until_date) values (%s, %s, %s);', (site_id, long(i*1e15), '201%d-12-31 23:59:59'%i))
-
- #db.insertRootDirectory('nikhef', '/dpm/nikhef.nl/home/lofar')
- db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar')
+ # insert the LTA site root dir(s)
+ db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/software')
+ db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/ops')
+ db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/storage')
+ db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/eor')
+ db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/pulsar')
+ db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/cosmics')
+ db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/surveys')
+ db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/user')
+ db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/proc')
+ db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/trans')
+ db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/lotest')
db.insertRootDirectory('juelich', '/pnfs/fz-juelich.de/data/lofar/ops')
db.insertRootDirectory('poznan', '/lofar/ops')
+ #db.insertRootDirectory('nikhef', '/dpm/nikhef.nl/home/lofar')
+
+ def end_of_year(year):
+ '''little helper function which returns a datetime timestamp for the end of the given year'''
+ return datetime.datetime(year, 12, 31, 23, 59, 59)
+
+ # insert quota as given by our LTA partners
+ db.insertSiteQuota(sara_id, 5e15, end_of_year(2012))
+ db.insertSiteQuota(sara_id, 8e15, end_of_year(2013))
+ db.insertSiteQuota(sara_id, 11e15, end_of_year(2014))
+ db.insertSiteQuota(sara_id, 14e15, end_of_year(2015))
+ db.insertSiteQuota(sara_id, 17e15, end_of_year(2016))
+ db.insertSiteQuota(sara_id, 20e15, end_of_year(2017))
+ db.insertSiteQuota(sara_id, 23e15, end_of_year(2018))
+
+ db.insertSiteQuota(juelich_id, 2.5e15, end_of_year(2013))
+ db.insertSiteQuota(juelich_id, 4.5e15, end_of_year(2014))
+ db.insertSiteQuota(juelich_id, 6.5e15, end_of_year(2015))
+ db.insertSiteQuota(juelich_id, 8.5e15, end_of_year(2016))
+ db.insertSiteQuota(juelich_id, 10.5e15, end_of_year(2017))
+ db.insertSiteQuota(juelich_id, 12.5e15, end_of_year(2018))
+
+ db.insertSiteQuota(poznan_id, 0.5e15, end_of_year(2016))
+ db.insertSiteQuota(poznan_id, 3.5e15, end_of_year(2017))
+ db.insertSiteQuota(poznan_id, 5.5e15, end_of_year(2018))
def main():
diff --git a/LTA/ltastorageoverview/lib/store.py b/LTA/ltastorageoverview/lib/store.py
index d233e5b0807644215306395797a5a9649de21f77..2db7e5d6a67c4aad91c8b9cf7ec6cd437f2dbda4 100644
--- a/LTA/ltastorageoverview/lib/store.py
+++ b/LTA/ltastorageoverview/lib/store.py
@@ -238,6 +238,24 @@ class LTAStorageDb(PostgresDatabaseConnection):
'''returns list of quota tuples (site_id, site_name, quota, valid_until_date)'''
return self.executeQuery('SELECT * FROM lta.site_quota;', FETCH_All)
+ def insertSiteQuota(self, site_id, quota, valid_until_date, commit=True):
+ """
+ insert the quota for a given site with a date until which this quota is valid.
+ :param int site_id: the id of the site for which you want to set the quota.
+ :param int quota: the quota in number of bytes.
+ :param datetime valid_until_date: the timestamp until which this given quota is valid.
+ :param bool commit: do/don't commit immediately.
+ :return: the id of the new quota
+ """
+ id = self.executeQuery('INSERT INTO lta.site_quota(site_id, quota, valid_until_date) values (%s, %s, %s) RETURNING id;',
+ (site_id, quota, valid_until_date))
+ if commit:
+ self.commit()
+ return id
+
+ '''returns list of quota tuples (site_id, site_name, quota, valid_until_date)'''
+ return self.executeQuery('SELECT * FROM lta.site_quota;', FETCH_All)
+
def directory(self, dir_id):
'''returns lta.directory (id, name, site_id, site_name) for the given dir_id'''
return self.executeQuery('''SELECT dir.id as dir_id, dir.name as dir_name, site.id as site_id, site.name as site_name