diff --git a/LTA/ltastorageoverview/lib/ltaso/create_db_ltastorageoverview.sql b/LTA/ltastorageoverview/lib/ltaso/create_db_ltastorageoverview.sql
index 16c182af109eff1f5feae7680da2c5eed1efd1b7..62b0891c7d778f8e021ca6a5968db49e6f918e26 100644
--- a/LTA/ltastorageoverview/lib/ltaso/create_db_ltastorageoverview.sql
+++ b/LTA/ltastorageoverview/lib/ltaso/create_db_ltastorageoverview.sql
@@ -648,8 +648,11 @@ DECLARE
stats_row metainfo.stats%ROWTYPE;
dir_num_files bigint;
dir_total_file_size bigint;
+ subdir_tree_num_files bigint;
+ subdir_tree_total_file_size bigint;
subdirs_tree_num_files bigint;
subdirs_tree_total_file_size bigint;
+ rec record;
BEGIN
-- we need to provide the requested tree_root_dir_id also as an output, so we can join on it
dir_id := tree_root_dir_id;
@@ -693,21 +696,18 @@ BEGIN
-- recurse into subdirectories, and accumulate subdir results
IF stats_row.tree_min_file_creation_date <= upper_ts OR stats_row.tree_max_file_creation_date >= lower_ts THEN
--sum all results from the subdirs which have at least partial overlap
- SELECT SUM(gts.tree_num_files), SUM(gts.tree_total_file_size)
- FROM lta.directory d
- INNER JOIN metainfo.stats s ON s.dir_id = d.id
- INNER JOIN metainfo.get_tree_stats(d.id, lower_ts, upper_ts) gts ON gts.dir_id = d.id
- WHERE d.parent_dir_id = tree_root_dir_id
- AND NOT (s.tree_min_file_creation_date > upper_ts OR s.tree_max_file_creation_date < lower_ts)
- INTO subdirs_tree_num_files, subdirs_tree_total_file_size;
-
- IF subdirs_tree_num_files IS NULL THEN
- subdirs_tree_num_files := 0;
- END IF;
+ subdirs_tree_num_files := 0;
+ subdirs_tree_total_file_size := 0;
- IF subdirs_tree_total_file_size IS NULL THEN
- subdirs_tree_total_file_size := 0;
- END IF;
+ -- TODO: replace slow for loop with recusrive query
+ FOR rec in (SELECT * FROM lta.directory d WHERE d.parent_dir_id = tree_root_dir_id) LOOP
+ SELECT gts.tree_num_files, gts.tree_total_file_size
+ FROM metainfo.get_tree_stats(rec.id, lower_ts, upper_ts) gts
+ INTO subdir_tree_num_files, subdir_tree_total_file_size;
+
+ subdirs_tree_num_files := subdirs_tree_num_files + subdir_tree_num_files;
+ subdirs_tree_total_file_size := subdirs_tree_total_file_size + subdir_tree_total_file_size;
+ END LOOP;
-- and add the num_files and total_file_size in this dir...
IF stats_row.dir_num_files > 0 THEN
@@ -787,6 +787,22 @@ BEGIN
END;
$$ LANGUAGE plpgsql;
+--TODO: see remarks at get_site_stats and get_tree_stats for optimizations.
+-- WARNING: SLOW!! Needs to be replaced by recursive select statements
+CREATE OR REPLACE FUNCTION metainfo.get_sites_quota_usage(OUT site_id integer, OUT site_name text, OUT site_quota_id integer, OUT quota bigint, OUT total_file_size bigint, OUT space_left bigint, OUT num_files bigint, OUT valid_until_date timestamp without time zone)
+ RETURNS SETOF record AS $$
+DECLARE rec record;
+BEGIN
+ FOR rec in SELECT * FROM lta.site_quota LOOP
+ SELECT rec.id INTO site_quota_id;
+ SELECT squ.site_id, squ.site_name, squ.quota, squ.total_file_size, squ.space_left, squ.num_files, squ.valid_until_date
+ FROM metainfo.get_site_quota_usage(site_quota_id) squ
+ INTO site_id, site_name, quota, total_file_size, space_left, num_files, valid_until_date;
+ RETURN NEXT;
+ END LOOP;
+END;
+$$ LANGUAGE plpgsql;
+
-- END NORMAL FUNCTIONS
@@ -795,24 +811,24 @@ $$ LANGUAGE plpgsql;
--
-- -- VIEWS
-CREATE VIEW lta.site_root_directory as
+CREATE OR REPLACE VIEW lta.site_root_directory as
select ss.id as site_id, ss.name as site_name, srd.root_dir_id, dir.name as dir_name
from lta.site_root_dir srd
join lta.directory dir on dir.id = srd.root_dir_id
join lta.site ss on ss.id = srd.site_id ;
-CREATE VIEW lta.site_quota_view as
- select ss.id as site_id, ss.name as site_name, ssq.quota, ssq.valid_until_date
+CREATE OR REPLACE VIEW lta.site_quota_view as
+ select ss.id as site_id, ss.name as site_name, ssq.id as site_quota_id, ssq.quota, ssq.valid_until_date
from lta.site ss
- left join lta.site_quota ssq on ssq.site_id = ss.id;
+ join lta.site_quota ssq on ssq.site_id = ss.id;
-CREATE VIEW lta.site_quota_root_directory as
+CREATE OR REPLACE VIEW lta.site_quota_root_directory as
SELECT s.id AS site_id, s.name AS site_name, d.id AS dir_id, d.name AS dir_name
FROM lta.quota_root_dirs qrd
JOIN lta.site s ON s.id = qrd.site_id
JOIN lta.directory d ON d.id = qrd.root_dir_id;
-CREATE VIEW lta.site_directory_tree as
+CREATE OR REPLACE VIEW lta.site_directory_tree as
select rd.site_id as site_id,
rd.site_name as site_name,
rd.root_dir_id as root_dir_id,
@@ -825,7 +841,7 @@ CREATE VIEW lta.site_directory_tree as
inner join lta.directory_closure dc on dc.ancestor_id = rd.root_dir_id
inner join lta.directory dir on dc.descendant_id = dir.id;
-CREATE VIEW scraper.site_scraper_last_directory_visit as
+CREATE OR REPLACE VIEW scraper.site_scraper_last_directory_visit as
select rd.site_id as site_id,
rd.site_name as site_name,
dir.id as dir_id,
@@ -836,7 +852,7 @@ CREATE VIEW scraper.site_scraper_last_directory_visit as
inner join lta.directory dir on dc.descendant_id = dir.id
inner join scraper.last_directory_visit sldv on sldv.dir_id = dir.id ;
-CREATE VIEW lta.site_directory_file as
+CREATE OR REPLACE VIEW lta.site_directory_file as
select site.id as site_id,
site.name as site_name,
dir.id as dir_id,
@@ -851,7 +867,7 @@ CREATE VIEW lta.site_directory_file as
inner join lta.directory dir on dc.descendant_id = dir.id
inner join lta.fileinfo on fileinfo.dir_id = dir.id ;
-CREATE VIEW metainfo.project_directory as
+CREATE OR REPLACE VIEW metainfo.project_directory as
select
project.id as project_id,
project.name as project_name,
@@ -862,7 +878,7 @@ CREATE VIEW metainfo.project_directory as
inner join lta.directory_closure dc on dc.ancestor_id = ptld.dir_id
inner join lta.directory dir on dc.descendant_id = dir.id ;
-CREATE VIEW metainfo.site_directory_stats as
+CREATE OR REPLACE VIEW metainfo.site_directory_stats as
select sdt.site_id,
sdt.site_name,
sdt.dir_id,
@@ -887,12 +903,12 @@ CREATE OR REPLACE VIEW metainfo.project_directory_stats AS
FROM metainfo.project_directory pd
JOIN metainfo.site_directory_stats sds ON sds.dir_id = pd.dir_id;
-CREATE VIEW metainfo.observation_dataproduct_file as
+CREATE OR REPLACE VIEW metainfo.observation_dataproduct_file as
SELECT sdf.site_id, sdf.site_name, dp.observation_id, dp.id as dataproduct_id, dp.name as dataproduct_name, sdf.dir_id, sdf.dir_name, sdf.file_id, sdf.file_name, sdf.file_size, sdf.file_creation_date
FROM metainfo.dataproduct dp
JOIN lta.site_directory_file sdf ON sdf.file_id = dp.fileinfo_id;
-CREATE VIEW metainfo.project_observation_dataproduct as
+CREATE OR REPLACE VIEW metainfo.project_observation_dataproduct as
SELECT p.id AS project_id,
p.name AS project_name,
dp.observation_id,
@@ -903,43 +919,42 @@ CREATE VIEW metainfo.project_observation_dataproduct as
INNER JOIN metainfo.project_observation po ON po.observation_id = dp.observation_id
INNER JOIN metainfo.project p ON p.id = po.project_id;
-CREATE VIEW metainfo.dataproduct_all as
+CREATE OR REPLACE VIEW metainfo.dataproduct_all as
SELECT pod.*, sdf.*
FROM metainfo.project_observation_dataproduct pod
INNER JOIN lta.site_directory_file sdf on sdf.file_id = pod.fileinfo_id;
-CREATE VIEW metainfo.site_project_stats as
+CREATE OR REPLACE VIEW metainfo.site_project_stats as
select ptld.project_id, p.name as project_name, site_id, site_name, sds.dir_id, sds.dir_name, tree_num_files, tree_total_file_size, tree_min_file_creation_date, tree_max_file_creation_date
from metainfo.project_top_level_directory ptld
inner join metainfo.project p on p.id = ptld.project_id
inner join metainfo.site_directory_stats sds on sds.dir_id = ptld.dir_id
where tree_num_files IS NOT NULL;
-CREATE VIEW metainfo.project_stats AS
+CREATE OR REPLACE VIEW metainfo.project_stats AS
SELECT project_id, project_name, COUNT(site_id) num_sites, SUM(tree_num_files) total_num_files, SUM(tree_total_file_size) total_file_size, MIN(tree_min_file_creation_date) min_file_creation_date, MAX(tree_max_file_creation_date) max_file_creation_date
FROM metainfo.site_project_stats
group by project_id, project_name;
-CREATE VIEW metainfo.site_project_observation_dataproduct_dir_file AS
+CREATE OR REPLACE VIEW metainfo.site_project_observation_dataproduct_dir_file AS
SELECT sdf.site_id, sdf.site_name, pod.project_id, pod.project_name, pod.observation_id, pod.dataproduct_id, pod.dataproduct_name, sdf.dir_id, sdf.dir_name, sdf.file_id, sdf.file_name, sdf.file_size, sdf.file_creation_date
FROM metainfo.project_observation_dataproduct pod
JOIN lta.site_directory_file sdf ON sdf.file_id = pod.fileinfo_id;
-CREATE VIEW metainfo.site_root_dir_tree_stats AS
+CREATE OR REPLACE VIEW metainfo.site_root_dir_tree_stats AS
SELECT srd.site_id, srd.site_name, srd.root_dir_id as root_dir_id, srd.dir_name as root_dir_name,
sds.tree_num_files, sds.tree_total_file_size, sds.tree_min_file_size, sds.tree_max_file_size, sds.tree_min_file_creation_date, sds.tree_max_file_creation_date
FROM lta.site_root_directory srd
INNER JOIN metainfo.site_directory_stats sds ON sds.dir_id = srd.root_dir_id;
-CREATE VIEW metainfo.site_stats as
+CREATE OR REPLACE VIEW metainfo.site_stats as
SELECT site_id, site_name, SUM(tree_num_files) total_num_files, SUM(tree_total_file_size) total_file_size, MIN(tree_min_file_size) min_file_size, MAX(tree_max_file_size) max_file_size, MIN(tree_min_file_creation_date) min_file_creation_date, MAX(tree_max_file_creation_date) max_file_creation_date
from metainfo.site_root_dir_tree_stats
group by site_id, site_name;
-CREATE VIEW metainfo.site_quota_usage AS
- select gsqu.*
- from lta.site_quota
- join metainfo.get_site_quota_usage(lta.site_quota.id) gsqu on gsqu.site_id = lta.site_quota.site_id;
+-- WARNING: SLOW!! Needs to be replaced by recursive select statements
+CREATE OR REPLACE VIEW metainfo.site_quota_usage AS
+ select * from metainfo.get_sites_quota_usage();
CREATE OR REPLACE VIEW metainfo.site_quota_root_dir_stats AS
SELECT sds.site_id, sds.site_name, sds.dir_id, sds.dir_name, sds.tree_num_files, sds.tree_total_file_size
diff --git a/LTA/ltastorageoverview/test/test_store.py b/LTA/ltastorageoverview/test/test_store.py
index bacd9e212389f157c86026a710597ba0c193acfe..d7c5bda6e7d3e2461df14625e8b367818d55fd78 100755
--- a/LTA/ltastorageoverview/test/test_store.py
+++ b/LTA/ltastorageoverview/test/test_store.py
@@ -208,7 +208,7 @@ class TestLTAStorageDb(CommonLTAStorageDbTest):
with self.assertRaises(LookupError) as context:
incorrect_dir_path = '/fdjsalfja5h43535h3oiu/5u905u3f'
db.insert_missing_directory_tree_if_needed(incorrect_dir_path, site_id)
- self.assertTrue('Could not find parent root dir' in context.exception.message)
+ self.assertTrue('Could not find parent root dir' in str(context.exception))
def testProjectsAndObservations(self):
with store.LTAStorageDb(self.dbcreds, True) as db: