diff --git a/LTA/ltastorageoverview/lib/create_db_ltastorageoverview.sql b/LTA/ltastorageoverview/lib/create_db_ltastorageoverview.sql
index 958922909537547a9c350d82fd39cef76352501f..dc438cf3162b24f944cdb5c09975a249b622a2d5 100644
--- a/LTA/ltastorageoverview/lib/create_db_ltastorageoverview.sql
+++ b/LTA/ltastorageoverview/lib/create_db_ltastorageoverview.sql
@@ -271,6 +271,7 @@ project_id int;
project_dir_name text;
project_dir_id int;
obs_id int;
+obs_dir_name text;
BEGIN
new_dir_name := trim(trailing '/' from NEW.name);
project_pos := strpos(new_dir_name, '/projects');
@@ -306,8 +307,12 @@ BEGIN
dir_name_tail := substring(dir_name_tail from length(project_name)+2);
next_slash_pos := strpos(dir_name_tail, '/');
IF next_slash_pos > 0 THEN
+ obs_dir_name := substring(dir_name_tail from 0 for next_slash_pos);
+ ELSE
+ obs_dir_name := dir_name_tail;
+ END IF;
BEGIN
- obs_id := substring(dir_name_tail from 0 for next_slash_pos)::integer;
+ obs_id := obs_dir_name::integer;
INSERT INTO metainfo.observation(id) VALUES (obs_id) ON CONFLICT DO NOTHING;
@@ -315,7 +320,6 @@ BEGIN
VALUES (project_id, obs_id) ON CONFLICT DO NOTHING;
EXCEPTION WHEN invalid_text_representation THEN
END;
- END IF;
END IF;
END IF;
END IF;
@@ -491,12 +495,20 @@ CREATE VIEW metainfo.site_directory_stats as
CREATE VIEW metainfo.project_observation_dataproducts as
- select dp.id as dataproduct_id, dp.name as dataproduct_name, dp.observation_id as observation_id,
- p.id as project_id, p.name as project_name
- from metainfo.dataproducts dp
- left join metainfo.project_observation po on po.observation_id = dp.observation_id
- left join metainfo.project p on p.id = po.project_id
- order by project_name, observation_id, dataproduct_name;
+ SELECT p.id AS project_id,
+ p.name AS project_name,
+ dp.observation_id,
+ dp.id AS dataproduct_id,
+ dp.name AS dataproduct_name,
+ dp.fileinfo_id AS fileinfo_id
+ FROM metainfo.dataproducts dp
+ LEFT JOIN metainfo.project_observation po ON po.observation_id = dp.observation_id
+ LEFT JOIN metainfo.project p ON p.id = po.project_id;
+
+CREATE VIEW metainfo.dataproducts_all as
+ SELECT pod.*, sdf.*
+ FROM metainfo.project_observation_dataproducts pod
+ INNER JOIN lta.site_directory_file sdf on sdf.file_id = pod.fileinfo_id;
-- END VIEWS
diff --git a/LTA/ltastorageoverview/lib/scraper.py b/LTA/ltastorageoverview/lib/scraper.py
index 3a4b304b5122fb7c50ad774cce8a3ba390b095ae..d20820f92d5223f51c060d624a2edca94973584b 100755
--- a/LTA/ltastorageoverview/lib/scraper.py
+++ b/LTA/ltastorageoverview/lib/scraper.py
@@ -37,11 +37,8 @@ from ltastorageoverview import store
from lofar.common.util import humanreadablesize
from random import random
-#logging.basicConfig(filename='scraper.' + time.strftime("%Y-%m-%d") + '.log', level=logging.DEBUG, format="%(asctime)-15s %(levelname)s %(message)s")
-logging.basicConfig(level=logging.DEBUG, format="%(asctime)-15s %(levelname)s %(message)s")
logger = logging.getLogger()
-
class FileInfo:
'''Simple struct to hold filename and size'''
def __init__(self, filename, size, created_at):
@@ -298,7 +295,7 @@ class ResultGetterThread(threading.Thread):
def populateDbWithLTASitesAndRootDirs(db):
if not db.sites():
- #db.insertSite('target', 'srm://srm.target.rug.nl:8444')
+ db.insertSite('target', 'srm://srm.target.rug.nl:8444')
#db.insertSite('nikhef', 'srm://tbn18.nikhef.nl:8446')
#db.insertSite('sara', 'srm://srm.grid.sara.nl:8443')
#db.insertSite('juelich', 'srm://lofar-srm.fz-juelich.de:8443')
@@ -307,14 +304,13 @@ def populateDbWithLTASitesAndRootDirs(db):
#db.insertRootDirectory('target', '/lofar/ops')
#db.insertRootDirectory('target', '/lofar/ops/disk')
#db.insertRootDirectory('nikhef', '/dpm/nikhef.nl/home/lofar')
- #db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/ops')
- #db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/user')
- #db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/software')
- #db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/storage')
- #db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/pulsar')
- #db.insertRootDirectory('juelich', '/pnfs/fz-juelich.de/data/lofar/ops')
- #db.insertRootDirectory('poznan', '/lofar/ops/projects')
- db.insertRootDirectory('poznan', '/lofar/ops/projects/lt5_009')
+ db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/ops')
+ db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/user')
+ db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/software')
+ db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/storage')
+ db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/pulsar')
+ db.insertRootDirectory('juelich', '/pnfs/fz-juelich.de/data/lofar/ops')
+ db.insertRootDirectory('poznan', '/lofar/ops/projects')
for dir_id in [x['dir_id'] for x in db.rootDirectories()]:
db.updateDirectoryLastVisitTime(dir_id, datetime.datetime.utcnow() - datetime.timedelta(days=1000))
@@ -380,7 +376,7 @@ def main():
#while numLocationsInQueues() > 0 and (totalNumGetters() <= 4 or
#(os.getloadavg()[0] < 3*multiprocessing.cpu_count() and
#totalNumGetters() < 2.5*multiprocessing.cpu_count())):
- while numLocationsInQueues() > 0 and (totalNumGetters() < 1):
+ while numLocationsInQueues() > 0 and (totalNumGetters() < 8):
with lock:
sitesStats = db.visitStats(datetime.datetime.utcnow() - datetime.timedelta(days=1))
diff --git a/LTA/ltastorageoverview/lib/store.py b/LTA/ltastorageoverview/lib/store.py
index 5790413a4e8c8c558c3abd3fe39ae18884b24011..0ed4e7b079568813889a90e033d581b8c003f9f5 100644
--- a/LTA/ltastorageoverview/lib/store.py
+++ b/LTA/ltastorageoverview/lib/store.py
@@ -206,11 +206,11 @@ class LTAStorageDb(PostgresDatabaseConnection):
result_query = query
result_args = args
if from_date:
- result_query += ' and %s >= %s' % table_column
+ result_query += ' and {column} >= %s'.format(column=table_column)
result_args += (from_date,)
if to_date:
- result_query += ' and %s <= %s' % table_column
+ result_query += ' and {column} <= %s'.format(column=table_column)
result_args += (to_date,)
return result_query, result_args