SW-285: insert known lta site quota

LTA/ltastorageoverview/lib/scraper.py

@@ -40,7 +40,7 @@ from random import random, randint
 logger = logging.getLogger()
 
 VISIT_INTERVAL = datetime.timedelta(days=3)
-LEXAR_HOST = 'ingest@lexar004'
+LEXAR_HOST = 'ingest@lexar004.offline.lofar'
 
 class FileInfo:
     '''Simple struct to hold filename and size'''
@@ -388,20 +388,54 @@ class ResultGetterThread(threading.Thread):
                 db.updateDirectoryLastVisitTime(self.dir_id, datetime.datetime.utcnow() - VISIT_INTERVAL)
 
 def populateDbWithLTASitesAndRootDirs(db):
+    """
+    Helper method to fill empty database with (hardcoded) information about our LTA partners/sites/quotas
+    """
     if not db.sites():
         #db.insertSite('nikhef', 'srm://tbn18.nikhef.nl:8446')
         sara_id = db.insertSiteIfNotExists('sara', 'srm://srm.grid.sara.nl:8443')
         juelich_id = db.insertSiteIfNotExists('juelich', 'srm://lofar-srm.fz-juelich.de:8443')
         poznan_id = db.insertSiteIfNotExists('poznan', 'srm://lta-head.lofar.psnc.pl:8443')
 
-        for site_id in [sara_id, juelich_id, poznan_id]:
-            for i in range(1, 8):
-                db.executeQuery('insert into lta.site_quota(site_id, quota, valid_until_date) values (%s, %s, %s);', (site_id, long(i*1e15), '201%d-12-31 23:59:59'%i))
-
-        #db.insertRootDirectory('nikhef', '/dpm/nikhef.nl/home/lofar')
-        db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar')
+        # insert the LTA site root dir(s)
+        db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/software')
+        db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/ops')
+        db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/storage')
+        db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/eor')
+        db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/pulsar')
+        db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/cosmics')
+        db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/surveys')
+        db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/user')
+        db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/proc')
+        db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/trans')
+        db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/lotest')
         db.insertRootDirectory('juelich', '/pnfs/fz-juelich.de/data/lofar/ops')
         db.insertRootDirectory('poznan', '/lofar/ops')
+        #db.insertRootDirectory('nikhef', '/dpm/nikhef.nl/home/lofar')
+
+        def end_of_year(year):
+            '''little helper function which returns a datetime timestamp for the end of the given year'''
+            return datetime.datetime(year, 12, 31, 23, 59, 59)
+
+        # insert quota as given by our LTA partners
+        db.insertSiteQuota(sara_id, 5e15, end_of_year(2012))
+        db.insertSiteQuota(sara_id, 8e15, end_of_year(2013))
+        db.insertSiteQuota(sara_id, 11e15, end_of_year(2014))
+        db.insertSiteQuota(sara_id, 14e15, end_of_year(2015))
+        db.insertSiteQuota(sara_id, 17e15, end_of_year(2016))
+        db.insertSiteQuota(sara_id, 20e15, end_of_year(2017))
+        db.insertSiteQuota(sara_id, 23e15, end_of_year(2018))
+
+        db.insertSiteQuota(juelich_id, 2.5e15, end_of_year(2013))
+        db.insertSiteQuota(juelich_id, 4.5e15, end_of_year(2014))
+        db.insertSiteQuota(juelich_id, 6.5e15, end_of_year(2015))
+        db.insertSiteQuota(juelich_id, 8.5e15, end_of_year(2016))
+        db.insertSiteQuota(juelich_id, 10.5e15, end_of_year(2017))
+        db.insertSiteQuota(juelich_id, 12.5e15, end_of_year(2018))
+
+        db.insertSiteQuota(poznan_id, 0.5e15, end_of_year(2016))
+        db.insertSiteQuota(poznan_id, 3.5e15, end_of_year(2017))
+        db.insertSiteQuota(poznan_id, 5.5e15, end_of_year(2018))
 
 
 def main():

LTA/ltastorageoverview/lib/store.py

@@ -238,6 +238,24 @@ class LTAStorageDb(PostgresDatabaseConnection):
         '''returns list of quota tuples (site_id, site_name, quota, valid_until_date)'''
         return self.executeQuery('SELECT * FROM lta.site_quota;', FETCH_All)
 
+    def insertSiteQuota(self, site_id, quota, valid_until_date, commit=True):
+        """
+        insert the quota for a given site with a date until which this quota is valid.
+        :param int site_id: the id of the site for which you want to set the quota.
+        :param int quota: the quota in number of bytes.
+        :param datetime valid_until_date: the timestamp until which this given quota is valid.
+        :param bool commit: do/don't commit immediately.
+        :return: the id of the new quota
+        """
+        id =  self.executeQuery('INSERT INTO lta.site_quota(site_id, quota, valid_until_date) values (%s, %s, %s) RETURNING id;',
+                                (site_id, quota, valid_until_date))
+        if commit:
+            self.commit()
+        return id
+
+        '''returns list of quota tuples (site_id, site_name, quota, valid_until_date)'''
+        return self.executeQuery('SELECT * FROM lta.site_quota;', FETCH_All)
+
     def directory(self, dir_id):
         '''returns lta.directory (id, name, site_id, site_name) for the given dir_id'''
         return self.executeQuery('''SELECT dir.id as dir_id, dir.name as dir_name, site.id as site_id, site.name as site_name