Task #8721: SW-280: processed review comment: moved big test into seperate...

Task #8721: SW-280: processed review comment: moved big test into seperate integration test file. added explanation for parts of the bigger test.

.gitattributes

@@ -2080,7 +2080,11 @@ LTA/ltastorageoverview/lib/webservice/templates/index.html -text
 LTA/ltastorageoverview/lib/webservice/webservice.py -text
 LTA/ltastorageoverview/ltastorageoverview_build.sh -text
 LTA/ltastorageoverview/test/CMakeLists.txt -text
+LTA/ltastorageoverview/test/common_test_ltastoragedb.py -text
 LTA/ltastorageoverview/test/db_performance_test.py -text
+LTA/ltastorageoverview/test/integration_test_store.py -text
+LTA/ltastorageoverview/test/integration_test_store.run -text
+LTA/ltastorageoverview/test/integration_test_store.sh -text
 LTA/ltastorageoverview/test/test_lso_webservice.py -text
 LTA/ltastorageoverview/test/test_lso_webservice.run -text
 LTA/ltastorageoverview/test/test_lso_webservice.sh -text

LTA/ltastorageoverview/test/CMakeLists.txt

@@ -3,3 +3,5 @@ include(LofarCTest)
 
 lofar_add_test(test_store)
 lofar_add_test(test_lso_webservice)
+
+lofar_add_test(integration_test_store)

LTA/ltastorageoverview/test/common_test_ltastoragedb.py

+# Copyright (C) 2012-2015    ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.    See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+# $Id$
+
+import unittest
+import logging
+import os, os.path
+import psycopg2
+import lofar.common.dbcredentials as dbc
+
+try:
+    import testing.postgresql
+except ImportError as e:
+    print str(e)
+    print 'Please install python package testing.test_psql: sudo pip install testing.test_psql'
+    exit(3)  # special lofar test exit code: skipped test
+
+logger = logging.getLogger(__name__)
+
+class CommonLTAStorageDbTest(unittest.TestCase):
+    def setUp(self):
+        logger.info('setting up test LTASO database server...')
+
+        # create a test db
+        logger.info('  creating test postgres server')
+        self.test_psql = testing.postgresql.Postgresql()
+        dsn = self.test_psql.dsn()
+        logger.info('  created test postgres server, dsn=%s', dsn)
+
+        self.dbcreds = dbc.Credentials()
+        self.dbcreds.user = 'test_user'
+        self.dbcreds.password = 'test_password'
+
+        with psycopg2.connect(**dsn) as conn:
+            cursor = conn.cursor()
+            #use same user/pass as stored in local dbcreds
+            query = "CREATE USER %s WITH SUPERUSER PASSWORD '%s'" % (self.dbcreds.user, self.dbcreds.password)
+            cursor.execute(query)
+
+            create_script_path = os.path.normpath(os.path.join(os.environ['LOFARROOT'], 'share', 'ltaso', 'create_db_ltastorageoverview.sql'))
+            logger.info('  running ltaso create script create_script=%s', create_script_path)
+            with open(create_script_path, 'r') as script:
+                cursor.execute(script.read())
+            logger.info('  completed ltaso create script')
+
+        # copy the test postgres server settings into dbcreds
+        # we can use these dbcreds in each test method to connect to the testing ltaso database
+        self.dbcreds.host = dsn['host']
+        self.dbcreds.database = dsn['database']
+        self.dbcreds.port = dsn['port']
+
+        logger.info('finished setting up test LTASO database')
+
+    def tearDown(self):
+        logger.info('removing test LTASO database server...')
+        self.test_psql.stop()
+        logger.info('removed test LTASO database server')
+

LTA/ltastorageoverview/test/integration_test_store.py

+#!/usr/bin/python
+
+# Copyright (C) 2012-2015    ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.    See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+# $Id$
+
+import logging
+from datetime import datetime, timedelta
+import time
+from common_test_ltastoragedb import *
+from lofar.lta.ltastorageoverview import store
+
+logger = logging.getLogger(__name__)
+
+class IntegrationTestLTAStorageDb(CommonLTAStorageDbTest):
+    """
+    Bigger tests for the lofar.lta.ltastorageoverview.store.LTAStorageDb
+    which test more complex behaviour with bigger amounts of data.
+    """
+
+    def testDirectoryTreesAndStats(self):
+        """Quite a big test, almost an integration test.
+        It consists of two stages/phases:
+        1) inserts a tree of directories and files in various sites and projects,
+        2) test if the automatically computed tree- and dirstats are correct.
+        """
+
+        with store.LTAStorageDb(self.dbcreds, True) as db:
+            base_time = datetime.utcnow()
+            base_time -= timedelta(seconds=base_time.second, microseconds=base_time.microsecond)
+
+            ###########################################################
+            # STAGE 1: insertion and check phase.
+            # insert the sites, directories, and files
+            # and check the dir- and tree stats directly after insertion
+            ###########################################################
+            NUM_SITES = 2
+            NUM_PROJECTS = 3
+            NUM_PROJECT_SUBDIRS = 4
+            NUM_SUB_SUBDIRS = 5
+
+            # helper dict to store all subdir id's for each dir.
+            dir2subdir = {}
+
+            for site_nr in range(NUM_SITES):
+                site_name = 'site%d' % site_nr
+                site_url = 'srm://%s.org' % site_name
+                db.insertSite(site_name, site_url)
+
+                for project_nr in range(NUM_PROJECTS):
+                    rootDir_id = db.insertRootDirectory(site_name, 'rootDir_%d' % project_nr)
+                    dir2subdir[rootDir_id] = []
+
+                    for subdir_nr in range(NUM_PROJECT_SUBDIRS):
+                        subDir_id = db.insertSubDirectory(rootDir_id, 'subDir_%d' % subdir_nr)
+                        dir2subdir[subDir_id] = []
+                        dir2subdir[rootDir_id].append(subDir_id)
+                        for file_nr in range(project_nr*subdir_nr):
+                            db.insertFileInfo('file_%d' % file_nr, 271*(file_nr+1), base_time + timedelta(days=10*site_nr+project_nr, hours=subdir_nr, seconds=file_nr), subDir_id)
+
+                            dir_files = db.filesInDirectory(subDir_id)
+                            dir_stats = db.directoryTreeStats(subDir_id)
+
+                            self.assertEqual(sum(f['size'] for f in dir_files), dir_stats['dir_total_file_size'])
+                            self.assertEqual(len(dir_files), dir_stats['dir_num_files'])
+                            if dir_files:
+                                self.assertEqual(min(f['size'] for f in dir_files), dir_stats['dir_min_file_size'])
+                                self.assertEqual(max(f['size'] for f in dir_files), dir_stats['dir_max_file_size'])
+                                self.assertEqual(min(f['creation_date'] for f in dir_files), dir_stats['dir_min_file_creation_date'])
+                                self.assertEqual(max(f['creation_date'] for f in dir_files), dir_stats['dir_max_file_creation_date'])
+
+                        for subsubdir_nr in range(NUM_SUB_SUBDIRS):
+                            subsubDir_id = db.insertSubDirectory(subDir_id, 'subsubDir_%d' % subsubdir_nr)
+                            dir2subdir[subsubDir_id] = []
+                            dir2subdir[subDir_id].append(subsubDir_id)
+                            for kk in range(project_nr*subdir_nr*subsubdir_nr):
+                                db.insertFileInfo('file_%d_%d' % (subdir_nr,kk), 314*(kk+1), base_time + timedelta(days=10*site_nr+project_nr, hours=10*subdir_nr+subsubdir_nr+2, seconds=kk), subsubDir_id)
+
+                                dir_files = db.filesInDirectory(subsubDir_id)
+                                dir_stats = db.directoryTreeStats(subsubDir_id)
+
+                                self.assertEqual(sum(f['size'] for f in dir_files), dir_stats['dir_total_file_size'])
+                                self.assertEqual(len(dir_files), dir_stats['dir_num_files'])
+                                if dir_files:
+                                    self.assertEqual(min(f['size'] for f in dir_files), dir_stats['dir_min_file_size'])
+                                    self.assertEqual(max(f['size'] for f in dir_files), dir_stats['dir_max_file_size'])
+                                    self.assertEqual(min(f['creation_date'] for f in dir_files), dir_stats['dir_min_file_creation_date'])
+                                    self.assertEqual(max(f['creation_date'] for f in dir_files), dir_stats['dir_max_file_creation_date'])
+
+                                    tree_totals = db.totalFileSizeAndNumFilesInTree(subDir_id, dir_stats['dir_min_file_creation_date'], dir_stats['dir_max_file_creation_date'])
+                                    self.assertEqual(tree_totals['tree_num_files'], dir_stats['dir_num_files'])
+                                    self.assertEqual(tree_totals['tree_total_file_size'], dir_stats['dir_total_file_size'])
+
+                        # test 1st level subdir again, and also check inclusion of 2nd level subdirs in tree stats
+                        dir_files = db.filesInDirectory(subDir_id)
+                        dir_stats = db.directoryTreeStats(subDir_id)
+                        # this dir only...
+                        self.assertEqual(sum(f['size'] for f in dir_files), dir_stats['dir_total_file_size'])
+                        self.assertEqual(len(dir_files), dir_stats['dir_num_files'])
+                        if dir_files:
+                            self.assertEqual(min(f['size'] for f in dir_files), dir_stats['dir_min_file_size'])
+                            self.assertEqual(max(f['size'] for f in dir_files), dir_stats['dir_max_file_size'])
+                            self.assertEqual(min(f['creation_date'] for f in dir_files), dir_stats['dir_min_file_creation_date'])
+                            self.assertEqual(max(f['creation_date'] for f in dir_files), dir_stats['dir_max_file_creation_date'])
+
+                        # including subdirs in tree...
+                        self.assertEqual(sum(f['file_size'] for f in db.filesInTree(subDir_id)), dir_stats['tree_total_file_size'])
+                        self.assertEqual(len(db.filesInTree(subDir_id)), dir_stats['tree_num_files'])
+
+            ####################################################################################
+            # STAGE 2: reporting phase.
+            # loop over the sites, directories, and files now that the database has been filled.
+            # and check the dir- and tree stats totals
+            ####################################################################################
+            for site in db.sites():
+                site_id = site['id']
+
+                rootDirs = db.rootDirectoriesForSite(site_id)
+                self.assertEquals(NUM_PROJECTS, len(rootDirs))
+
+                for root_dir_id in [x['root_dir_id'] for x in rootDirs]:
+                    subDirs = db.subDirectories(root_dir_id, 1, False)
+                    self.assertEquals(NUM_PROJECT_SUBDIRS, len(subDirs))
+
+                    for subDir in subDirs:
+                        subDir_parent_id = subDir['parent_dir_id']
+                        self.assertEquals(root_dir_id, subDir_parent_id)
+                        self.assertTrue(subDir['id'] in dir2subdir[root_dir_id])
+
+                        subsubDirs = db.subDirectories(subDir['id'], 1, False)
+                        self.assertEquals(NUM_SUB_SUBDIRS, len(subsubDirs))
+
+                        for subsubDir in subsubDirs:
+                            subsubDir_parent_id = subsubDir['parent_dir_id']
+                            self.assertEquals(subDir['id'], subsubDir_parent_id)
+                            self.assertTrue(subsubDir['id'] in dir2subdir[subDir['id']])
+
+                    # check various selects of files in the tree, for each file
+                    tree_files = sorted(db.filesInTree(root_dir_id), key=lambda f: f['file_creation_date'])
+                    for file in tree_files:
+                        # check if filesInTree return this one file when time delimited for this specific file_creation_date
+                        file_creation_date = file['file_creation_date']
+                        selected_tree_files = db.filesInTree(root_dir_id, file_creation_date, file_creation_date)
+                        self.assertEqual(1, len(selected_tree_files))
+                        self.assertEqual(file['file_creation_date'], selected_tree_files[0]['file_creation_date'])
+                        self.assertEqual(file['file_size'], selected_tree_files[0]['file_size'])
+
+                        # get the 'totals' for this root_dir, but select only this file by date.
+                        # should return 1 file.
+                        tree_totals = db.totalFileSizeAndNumFilesInTree(root_dir_id, file_creation_date, file_creation_date)
+                        self.assertEqual(1, tree_totals['tree_num_files'])
+                        self.assertEqual(file['file_size'], tree_totals['tree_total_file_size'])
+
+                    # check some ranges files/times
+                    for idx, file in enumerate(tree_files):
+                        file_creation_date = file['file_creation_date']
+
+                        #select any file >= file_creation_date
+                        expected_selected_tree_files = tree_files[idx:]
+                        selected_tree_files = db.filesInTree(root_dir_id, file_creation_date, None)
+                        self.assertEqual(len(expected_selected_tree_files), len(selected_tree_files))
+                        selected_tree_files_ids = set([f['file_id'] for f in selected_tree_files])
+                        for expected_file in expected_selected_tree_files:
+                            self.assertTrue(expected_file['file_id'] in selected_tree_files_ids)
+
+                        # and check the totals as well
+                        tree_totals = db.totalFileSizeAndNumFilesInTree(root_dir_id, file_creation_date, None)
+                        self.assertEqual(len(expected_selected_tree_files), tree_totals['tree_num_files'])
+                        self.assertEqual(sum(f['file_size'] for f in expected_selected_tree_files), tree_totals['tree_total_file_size'])
+
+                        #select any file <= file_creation_date
+                        expected_selected_tree_files = tree_files[:idx+1]
+                        selected_tree_files = db.filesInTree(root_dir_id, None, file_creation_date)
+                        self.assertEqual(len(expected_selected_tree_files), len(selected_tree_files))
+                        selected_tree_files_ids = set([f['file_id'] for f in selected_tree_files])
+                        for expected_file in expected_selected_tree_files:
+                            self.assertTrue(expected_file['file_id'] in selected_tree_files_ids)
+
+                        # and check the totals as well
+                        tree_totals = db.totalFileSizeAndNumFilesInTree(root_dir_id, None, file_creation_date)
+                        self.assertEqual(len(expected_selected_tree_files), tree_totals['tree_num_files'])
+                        self.assertEqual(sum(f['file_size'] for f in expected_selected_tree_files), tree_totals['tree_total_file_size'])
+
+# run tests if main
+if __name__ == '__main__':
+    logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
+                        level=logging.INFO)
+
+    unittest.main()

LTA/ltastorageoverview/test/integration_test_store.run

+#!/bin/bash
+
+source python-coverage.sh
+python_coverage_test "ltas*" integration_test_store.py

LTA/ltastorageoverview/test/integration_test_store.sh

+#!/bin/sh
+
+./runctest.sh integration_test_store

LTA/ltastorageoverview/test/test_store.py

@@ -19,65 +19,18 @@
 
 # $Id$
 
-import unittest
 import logging
 from datetime import datetime, timedelta
 import time
-import os, os.path
-import psycopg2
-from pprint import *
-import lofar.common.dbcredentials as dbc
+from pprint import pformat
+
+from common_test_ltastoragedb import *
 from lofar.lta.ltastorageoverview import store
 from lofar.common.postgres import FETCH_NONE,FETCH_ONE,FETCH_ALL
 
-try:
-    import testing.postgresql
-except ImportError as e:
-    print str(e)
-    print 'Please install python package testing.test_psql: sudo pip install testing.test_psql'
-    exit(3)  # special lofar test exit code: skipped test
-
 logger = logging.getLogger(__name__)
 
-class TestLTAStorageDb(unittest.TestCase):
-    def setUp(self):
-        logger.info('setting up test LTASO database server...')
-
-        # create a test db
-        logger.info('  creating test postgres server')
-        self.test_psql = testing.postgresql.Postgresql()
-        dsn = self.test_psql.dsn()
-        logger.info('  created test postgres server, dsn=%s', dsn)
-
-        self.dbcreds = dbc.Credentials()
-        self.dbcreds.user = 'test_user'
-        self.dbcreds.password = 'test_password'
-
-        with psycopg2.connect(**dsn) as conn:
-            cursor = conn.cursor()
-            #use same user/pass as stored in local dbcreds
-            query = "CREATE USER %s WITH SUPERUSER PASSWORD '%s'" % (self.dbcreds.user, self.dbcreds.password)
-            cursor.execute(query)
-
-            create_script_path = os.path.normpath(os.path.join(os.environ['LOFARROOT'], 'share', 'ltaso', 'create_db_ltastorageoverview.sql'))
-            logger.info('  running ltaso create script create_script=%s', create_script_path)
-            with open(create_script_path, 'r') as script:
-                cursor.execute(script.read())
-            logger.info('  completed ltaso create script')
-
-        # copy the test postgres server settings into dbcreds
-        # we can use these dbcreds in each test method to connect to the testing ltaso database
-        self.dbcreds.host = dsn['host']
-        self.dbcreds.database = dsn['database']
-        self.dbcreds.port = dsn['port']
-
-        logger.info('finished setting up test LTASO database')
-
-    def tearDown(self):
-        logger.info('removing test LTASO database server...')
-        self.test_psql.stop()
-        logger.info('removed test LTASO database server')
-
+class TestLTAStorageDb(CommonLTAStorageDbTest):
     def testSites(self):
         with store.LTAStorageDb(self.dbcreds, True) as db:
             siteA_id = db.insertSite('siteA', 'srm://siteA.org')
@@ -132,141 +85,6 @@ class TestLTAStorageDb(unittest.TestCase):
             self.assertEqual(siteB_id, rootDirsDict[dirB2_id]['site_id'])
             self.assertEqual('siteB', rootDirsDict[dirB2_id]['site_name'])
 
-    def testDirectoryTreesAndStats(self):
-        with store.LTAStorageDb(self.dbcreds, True) as db:
-            base_time = datetime.utcnow()
-            base_time -= timedelta(seconds=base_time.second, microseconds=base_time.microsecond)
-
-            for site_nr in range(2):
-                site_name = 'site%d' % site_nr
-                site_url = 'srm://%s.org' % site_name
-                site_id = db.insertSite(site_name, site_url)
-
-                dir2subdir = {}
-
-                for project_nr in range(3):
-                    rootDir_id = db.insertRootDirectory(site_name, 'rootDir_%d' % project_nr)
-                    dir2subdir[rootDir_id] = []
-
-                    for subdir_nr in range(4):
-                        subDir_id = db.insertSubDirectory(rootDir_id, 'subDir_%d' % subdir_nr)
-                        dir2subdir[subDir_id] = []
-                        dir2subdir[rootDir_id].append(subDir_id)
-                        for file_nr in range(project_nr*subdir_nr):
-                            db.insertFileInfo('file_%d' % file_nr, 271*(file_nr+1), base_time + timedelta(days=10*site_nr+project_nr, hours=subdir_nr, seconds=file_nr), subDir_id)
-
-                            dir_files = db.filesInDirectory(subDir_id)
-                            dir_stats = db.directoryTreeStats(subDir_id)
-
-                            self.assertEqual(sum(f['size'] for f in dir_files), dir_stats['dir_total_file_size'])
-                            self.assertEqual(len(dir_files), dir_stats['dir_num_files'])
-                            if dir_files:
-                                self.assertEqual(min(f['size'] for f in dir_files), dir_stats['dir_min_file_size'])
-                                self.assertEqual(max(f['size'] for f in dir_files), dir_stats['dir_max_file_size'])
-                                self.assertEqual(min(f['creation_date'] for f in dir_files), dir_stats['dir_min_file_creation_date'])
-                                self.assertEqual(max(f['creation_date'] for f in dir_files), dir_stats['dir_max_file_creation_date'])
-
-                        for subsubdir_nr in range(5):
-                            subsubDir_id = db.insertSubDirectory(subDir_id, 'subsubDir_%d' % subsubdir_nr)
-                            dir2subdir[subsubDir_id] = []
-                            dir2subdir[subDir_id].append(subsubDir_id)
-                            for kk in range(project_nr*subdir_nr*subsubdir_nr):
-                                db.insertFileInfo('file_%d_%d' % (subdir_nr,kk), 314*(kk+1), base_time + timedelta(days=10*site_nr+project_nr, hours=10*subdir_nr+subsubdir_nr+2, seconds=kk), subsubDir_id)
-
-                                dir_files = db.filesInDirectory(subsubDir_id)
-                                dir_stats = db.directoryTreeStats(subsubDir_id)
-
-                                self.assertEqual(sum(f['size'] for f in dir_files), dir_stats['dir_total_file_size'])
-                                self.assertEqual(len(dir_files), dir_stats['dir_num_files'])
-                                if dir_files:
-                                    self.assertEqual(min(f['size'] for f in dir_files), dir_stats['dir_min_file_size'])
-                                    self.assertEqual(max(f['size'] for f in dir_files), dir_stats['dir_max_file_size'])
-                                    self.assertEqual(min(f['creation_date'] for f in dir_files), dir_stats['dir_min_file_creation_date'])
-                                    self.assertEqual(max(f['creation_date'] for f in dir_files), dir_stats['dir_max_file_creation_date'])
-
-                                    tree_totals = db.totalFileSizeAndNumFilesInTree(subDir_id, dir_stats['dir_min_file_creation_date'], dir_stats['dir_max_file_creation_date'])
-                                    self.assertEqual(tree_totals['tree_num_files'], dir_stats['dir_num_files'])
-                                    self.assertEqual(tree_totals['tree_total_file_size'], dir_stats['dir_total_file_size'])
-
-                        # test 1st level subdir again, and also check inclusion of 2nd level subdirs in tree stats
-                        dir_files = db.filesInDirectory(subDir_id)
-                        dir_stats = db.directoryTreeStats(subDir_id)
-                        # this dir only...
-                        self.assertEqual(sum(f['size'] for f in dir_files), dir_stats['dir_total_file_size'])
-                        self.assertEqual(len(dir_files), dir_stats['dir_num_files'])
-                        if dir_files:
-                            self.assertEqual(min(f['size'] for f in dir_files), dir_stats['dir_min_file_size'])
-                            self.assertEqual(max(f['size'] for f in dir_files), dir_stats['dir_max_file_size'])
-                            self.assertEqual(min(f['creation_date'] for f in dir_files), dir_stats['dir_min_file_creation_date'])
-                            self.assertEqual(max(f['creation_date'] for f in dir_files), dir_stats['dir_max_file_creation_date'])
-
-                        # including subdirs in tree...
-                        self.assertEqual(sum(f['file_size'] for f in db.filesInTree(subDir_id)), dir_stats['tree_total_file_size'])
-                        self.assertEqual(len(db.filesInTree(subDir_id)), dir_stats['tree_num_files'])
-
-                rootDirs = db.rootDirectoriesForSite(site_id)
-                self.assertEquals(3, len(rootDirs))
-
-                for root_dir_id in [x['root_dir_id'] for x in rootDirs]:
-                    subDirs = db.subDirectories(root_dir_id, 1, False)
-                    for subDir in subDirs:
-                        subDir_parent_id = subDir['parent_dir_id']
-                        self.assertEquals(root_dir_id, subDir_parent_id)
-                        self.assertTrue(subDir['id'] in dir2subdir[root_dir_id])
-
-                        subsubDirs = db.subDirectories(subDir['id'], 1, False)
-                        for subsubDir in subsubDirs:
-                            subsubDir_parent_id = subsubDir['parent_dir_id']
-                            self.assertEquals(subDir['id'], subsubDir_parent_id)
-                            self.assertTrue(subsubDir['id'] in dir2subdir[subDir['id']])
-
-                    # check various selects of files in the tree, for each file
-                    tree_files = sorted(db.filesInTree(root_dir_id), key=lambda f: f['file_creation_date'])
-                    for file in tree_files:
-                        # check if filesInTree return this one file when time delimited for this specific file_creation_date
-                        file_creation_date = file['file_creation_date']
-                        selected_tree_files = db.filesInTree(root_dir_id, file_creation_date, file_creation_date)
-                        self.assertEqual(1, len(selected_tree_files))
-                        self.assertEqual(file['file_creation_date'], selected_tree_files[0]['file_creation_date'])
-                        self.assertEqual(file['file_size'], selected_tree_files[0]['file_size'])
-
-                        # get the 'totals' for this root_dir, but select only this file by date.
-                        # should return 1 file.
-                        tree_totals = db.totalFileSizeAndNumFilesInTree(root_dir_id, file_creation_date, file_creation_date)
-                        self.assertEqual(1, tree_totals['tree_num_files'])
-                        self.assertEqual(file['file_size'], tree_totals['tree_total_file_size'])
-
-                    # check some ranges files/times
-                    for idx, file in enumerate(tree_files):
-                        file_creation_date = file['file_creation_date']
-
-                        #select any file >= file_creation_date
-                        expected_selected_tree_files = tree_files[idx:]
-                        selected_tree_files = db.filesInTree(root_dir_id, file_creation_date, None)
-                        self.assertEqual(len(expected_selected_tree_files), len(selected_tree_files))
-                        selected_tree_files_ids = set([f['file_id'] for f in selected_tree_files])
-                        for expected_file in expected_selected_tree_files:
-                            self.assertTrue(expected_file['file_id'] in selected_tree_files_ids)
-
-                        # and check the totals as well
-                        tree_totals = db.totalFileSizeAndNumFilesInTree(root_dir_id, file_creation_date, None)
-                        self.assertEqual(len(expected_selected_tree_files), tree_totals['tree_num_files'])
-                        self.assertEqual(sum(f['file_size'] for f in expected_selected_tree_files), tree_totals['tree_total_file_size'])
-
-                        #select any file <= file_creation_date
-                        expected_selected_tree_files = tree_files[:idx+1]
-                        selected_tree_files = db.filesInTree(root_dir_id, None, file_creation_date)
-                        self.assertEqual(len(expected_selected_tree_files), len(selected_tree_files))
-                        selected_tree_files_ids = set([f['file_id'] for f in selected_tree_files])
-                        for expected_file in expected_selected_tree_files:
-                            self.assertTrue(expected_file['file_id'] in selected_tree_files_ids)
-
-                        # and check the totals as well
-                        tree_totals = db.totalFileSizeAndNumFilesInTree(root_dir_id, None, file_creation_date)
-                        self.assertEqual(len(expected_selected_tree_files), tree_totals['tree_num_files'])
-                        self.assertEqual(sum(f['file_size'] for f in expected_selected_tree_files), tree_totals['tree_total_file_size'])
-
-
     def testLeastRecentlyVisitedDirectory(self):
         with store.LTAStorageDb(self.dbcreds, True) as db:
             db.insertSite('siteA', 'srm://siteA.org')