Task #8721: added triggers tables and view to gather project and observation info

LTA/ltastorageoverview/lib/create_db_ltastorageoverview.sql

@@ -271,6 +271,7 @@ project_id int;
 project_dir_name text;
 project_dir_id int;
 obs_id int;
+obs_dir_name text;
 BEGIN
     new_dir_name := trim(trailing '/' from NEW.name);
     project_pos := strpos(new_dir_name, '/projects');
@@ -306,8 +307,12 @@ BEGIN
                         dir_name_tail := substring(dir_name_tail from length(project_name)+2);
                         next_slash_pos := strpos(dir_name_tail, '/');
                         IF next_slash_pos > 0 THEN
+                            obs_dir_name := substring(dir_name_tail from 0 for next_slash_pos);
+                        ELSE
+                            obs_dir_name := dir_name_tail;
+                        END IF;
                             BEGIN
-                                obs_id := substring(dir_name_tail from 0 for next_slash_pos)::integer;
+                                obs_id := obs_dir_name::integer;
 
                                 INSERT INTO metainfo.observation(id) VALUES (obs_id) ON CONFLICT DO NOTHING;
 
@@ -315,7 +320,6 @@ BEGIN
                                 VALUES (project_id, obs_id) ON CONFLICT DO NOTHING;
                             EXCEPTION WHEN invalid_text_representation THEN
                             END;
-                        END IF;
                     END IF;
                 END IF;
             END IF;
@@ -491,12 +495,20 @@ CREATE VIEW metainfo.site_directory_stats as
 
 
 CREATE VIEW metainfo.project_observation_dataproducts as
-    select dp.id as dataproduct_id, dp.name as dataproduct_name, dp.observation_id as observation_id,
-    p.id as project_id, p.name as project_name
-    from metainfo.dataproducts dp
-    left join metainfo.project_observation po on po.observation_id = dp.observation_id
-    left join metainfo.project p on p.id = po.project_id
-    order by project_name, observation_id, dataproduct_name;
+ SELECT p.id AS project_id,
+    p.name AS project_name,
+    dp.observation_id,
+    dp.id AS dataproduct_id,
+    dp.name AS dataproduct_name,
+    dp.fileinfo_id AS fileinfo_id
+   FROM metainfo.dataproducts dp
+     LEFT JOIN metainfo.project_observation po ON po.observation_id = dp.observation_id
+     LEFT JOIN metainfo.project p ON p.id = po.project_id;
+
+CREATE VIEW metainfo.dataproducts_all as
+ SELECT pod.*, sdf.*
+   FROM metainfo.project_observation_dataproducts pod
+     INNER JOIN lta.site_directory_file sdf on sdf.file_id = pod.fileinfo_id;
 
 -- END VIEWS
 

LTA/ltastorageoverview/lib/scraper.py

@@ -37,11 +37,8 @@ from ltastorageoverview import store
 from lofar.common.util import humanreadablesize
 from random import random
 
-#logging.basicConfig(filename='scraper.' + time.strftime("%Y-%m-%d") + '.log', level=logging.DEBUG, format="%(asctime)-15s %(levelname)s %(message)s")
-logging.basicConfig(level=logging.DEBUG, format="%(asctime)-15s %(levelname)s %(message)s")
 logger = logging.getLogger()
 
-
 class FileInfo:
     '''Simple struct to hold filename and size'''
     def __init__(self, filename, size, created_at):
@@ -298,7 +295,7 @@ class ResultGetterThread(threading.Thread):
 
 def populateDbWithLTASitesAndRootDirs(db):
     if not db.sites():
-        #db.insertSite('target', 'srm://srm.target.rug.nl:8444')
+        db.insertSite('target', 'srm://srm.target.rug.nl:8444')
         #db.insertSite('nikhef', 'srm://tbn18.nikhef.nl:8446')
         #db.insertSite('sara', 'srm://srm.grid.sara.nl:8443')
         #db.insertSite('juelich', 'srm://lofar-srm.fz-juelich.de:8443')
@@ -307,14 +304,13 @@ def populateDbWithLTASitesAndRootDirs(db):
         #db.insertRootDirectory('target', '/lofar/ops')
         #db.insertRootDirectory('target', '/lofar/ops/disk')
         #db.insertRootDirectory('nikhef', '/dpm/nikhef.nl/home/lofar')
-        #db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/ops')
-        #db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/user')
-        #db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/software')
-        #db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/storage')
-        #db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/pulsar')
-        #db.insertRootDirectory('juelich', '/pnfs/fz-juelich.de/data/lofar/ops')
-        #db.insertRootDirectory('poznan', '/lofar/ops/projects')
-        db.insertRootDirectory('poznan', '/lofar/ops/projects/lt5_009')
+        db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/ops')
+        db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/user')
+        db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/software')
+        db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/storage')
+        db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/pulsar')
+        db.insertRootDirectory('juelich', '/pnfs/fz-juelich.de/data/lofar/ops')
+        db.insertRootDirectory('poznan', '/lofar/ops/projects')
 
         for dir_id in [x['dir_id'] for x in db.rootDirectories()]:
             db.updateDirectoryLastVisitTime(dir_id, datetime.datetime.utcnow() - datetime.timedelta(days=1000))
@@ -380,7 +376,7 @@ def main():
             #while numLocationsInQueues() > 0 and (totalNumGetters() <= 4 or
                                                   #(os.getloadavg()[0] < 3*multiprocessing.cpu_count() and
                                                   #totalNumGetters() < 2.5*multiprocessing.cpu_count())):
-            while numLocationsInQueues() > 0 and (totalNumGetters() < 1):
+            while numLocationsInQueues() > 0 and (totalNumGetters() < 8):
 
                 with lock:
                     sitesStats = db.visitStats(datetime.datetime.utcnow() - datetime.timedelta(days=1))

LTA/ltastorageoverview/lib/store.py

@@ -206,11 +206,11 @@ class LTAStorageDb(PostgresDatabaseConnection):
         result_query = query
         result_args = args
         if from_date:
-            result_query += ' and %s >= %s' % table_column
+            result_query += ' and {column} >= %s'.format(column=table_column)
             result_args += (from_date,)
 
         if to_date:
-            result_query += ' and %s  <= %s' % table_column
+            result_query += ' and {column} <= %s'.format(column=table_column)
             result_args += (to_date,)
 
         return result_query, result_args