diff --git a/LTA/ltastorageoverview/lib/scraper.py b/LTA/ltastorageoverview/lib/scraper.py
index 8552d3cf818d639652239eb476c2014992fbc6e0..53aac285d4d088f6a0d6279798632da705659c9c 100755
--- a/LTA/ltastorageoverview/lib/scraper.py
+++ b/LTA/ltastorageoverview/lib/scraper.py
@@ -388,9 +388,9 @@ class ResultGetterThread(threading.Thread):
def populateDbWithLTASitesAndRootDirs(db):
if not db.sites():
#db.insertSite('nikhef', 'srm://tbn18.nikhef.nl:8446')
- sara_id = db.insertSite('sara', 'srm://srm.grid.sara.nl:8443')
- juelich_id = db.insertSite('juelich', 'srm://lofar-srm.fz-juelich.de:8443')
- poznan_id = db.insertSite('poznan', 'srm://lta-head.lofar.psnc.pl:8443')
+ sara_id = db.insertSiteIfNotExists('sara', 'srm://srm.grid.sara.nl:8443')
+ juelich_id = db.insertSiteIfNotExists('juelich', 'srm://lofar-srm.fz-juelich.de:8443')
+ poznan_id = db.insertSiteIfNotExists('poznan', 'srm://lta-head.lofar.psnc.pl:8443')
for site_id in [sara_id, juelich_id, poznan_id]:
for i in range(1, 8):
diff --git a/LTA/ltastorageoverview/lib/store.py b/LTA/ltastorageoverview/lib/store.py
index 34884cc01f6f7341d1135d28a002f644e5434bb6..36dd76c6ab201113bc935640e074db2866bf9e2c 100644
--- a/LTA/ltastorageoverview/lib/store.py
+++ b/LTA/ltastorageoverview/lib/store.py
@@ -51,19 +51,29 @@ class LTAStorageDb(PostgresDatabaseConnection):
:param string srmurls: the srm url to that site
:return int: the new id of the inserted site
"""
+ site_id = self.executeQuery('insert into lta.site (name, url) values (%s, %s) returning id;', (siteName, srmurl), fetch=FETCH_ONE)['id']
+ self.commit()
+ return site_id
+
+ def insertSiteIfNotExists(self, siteName, srmurl):
+ """insert a site into the database and return the id.
+ If the site already exists, then the id of that site is just returned.
+ :param string siteName: the name of the site
+ :param string srmurls: the srm url to that site
+ :return int: the new id of the inserted site
+ """
site = self.siteByName(siteName)
if site:
return site['id']
- site_id = self.executeQuery('insert into lta.site (name, url) values (%s, %s) returning id;', (siteName, srmurl), fetch=FETCH_ONE)['id']
- self.commit()
- return site_id
+ return self.insertSite(siteName, srmurl)
def insertRootDirectory(self, siteName, rootDirectory):
"""
Insert a root directory for a site. Each site has at least one root directory (with no parent).
- For all non-root directories, use insertSubDirectory
+ For all non-root directories, use insertSubDirectory.
+ Beware: Uniqueness of the root dir for a site is not enforced.
:param string siteName: the name of the site (should already be in the database)
:param string rootDirectory: the full path of the directory
:return integer: the new id of the inserted root directory
@@ -81,7 +91,7 @@ class LTAStorageDb(PostgresDatabaseConnection):
self.commit()
return dir_id
- def insertSubDirectory(self, parent_dir_id, sub_directory_path):
+ def insertSubDirectory(self, sub_directory_path, parent_dir_id):
"""
Insert a sub directory which is a child of the directory with parent_dir_id
:param int parent_dir_id: the id of this subdirectories parent
diff --git a/LTA/ltastorageoverview/test/db_performance_test.py b/LTA/ltastorageoverview/test/db_performance_test.py
index 04b5f7f92831129c9bacd7da231dd1897350acf1..f224b3ee9c90b7b80dc6a753fc5857cd0d0b1ad2 100755
--- a/LTA/ltastorageoverview/test/db_performance_test.py
+++ b/LTA/ltastorageoverview/test/db_performance_test.py
@@ -75,9 +75,9 @@ def main():
db = store.LTAStorageDb(dbcreds, options.verbose)
- db.insertSite('sara', 'srm://srm.siteA.nl:8444')
+ db.insertSiteIfNotExists('sara', 'srm://srm.siteA.nl:8444')
rootdir_id = db.insertRootDirectory('sara', '/pnfs/grid.siteA.nl/data/lofar/ops')
- projects_dir_id = db.insertSubDirectory(rootdir_id, '/pnfs/grid.siteA.nl/data/lofar/ops/projects')
+ projects_dir_id = db.insertSubDirectory('/pnfs/grid.siteA.nl/data/lofar/ops/projects', rootdir_id)
total_num_files_inserted = 0
@@ -86,13 +86,13 @@ def main():
for project_nr in range(1, 10):
# project_name = 'lc%d_%03d/%d' % (cycle_nr, project_nr, os.getpid())
project_name = 'lc%d_%03d' % (cycle_nr, project_nr)
- projectdir_id = db.insertSubDirectory(projects_dir_id, '/pnfs/grid.siteA.nl/data/lofar/ops/projects/%s' % (project_name,))
+ projectdir_id = db.insertSubDirectory('/pnfs/grid.siteA.nl/data/lofar/ops/projects/%s' % (project_name,), projects_dir_id)
obs_base_id = cycle_nr*100000+project_nr*1000
for obs_nr, obsId in enumerate(range(obs_base_id, obs_base_id+20)):
obsName = 'L%s' % obsId
- obsdir_id = db.insertSubDirectory(projectdir_id, '/pnfs/grid.siteA.nl/data/lofar/ops/projects/%s/%s' % (project_name, obsName))
+ obsdir_id = db.insertSubDirectory('/pnfs/grid.siteA.nl/data/lofar/ops/projects/%s/%s' % (project_name, obsName), projectdir_id)
fileinfos = [('%s_SB%3d' % (obsName, sbNr), 1000+sbNr+project_nr*cycle_nr, base_date + timedelta(days=10*cycle_nr+project_nr, minutes=obs_nr, seconds=sbNr), obsdir_id) for sbNr in range(0, 2)]
now = datetime.utcnow()
diff --git a/LTA/ltastorageoverview/test/integration_test_store.py b/LTA/ltastorageoverview/test/integration_test_store.py
index 8f38c40e76abef0d619fc9648582db187c418750..b141011a2d23bd0377f55a8e9307f2d6f6f77007 100755
--- a/LTA/ltastorageoverview/test/integration_test_store.py
+++ b/LTA/ltastorageoverview/test/integration_test_store.py
@@ -60,14 +60,14 @@ class IntegrationTestLTAStorageDb(CommonLTAStorageDbTest):
for site_nr in range(NUM_SITES):
site_name = 'site%d' % site_nr
site_url = 'srm://%s.org' % site_name
- db.insertSite(site_name, site_url)
+ db.insertSiteIfNotExists(site_name, site_url)
for project_nr in range(NUM_PROJECTS):
rootDir_id = db.insertRootDirectory(site_name, 'rootDir_%d' % project_nr)
dir2subdir[rootDir_id] = []
for subdir_nr in range(NUM_PROJECT_SUBDIRS):
- subDir_id = db.insertSubDirectory(rootDir_id, 'subDir_%d' % subdir_nr)
+ subDir_id = db.insertSubDirectory('subDir_%d' % subdir_nr, rootDir_id)
dir2subdir[subDir_id] = []
dir2subdir[rootDir_id].append(subDir_id)
for file_nr in range(project_nr*subdir_nr):
@@ -85,7 +85,7 @@ class IntegrationTestLTAStorageDb(CommonLTAStorageDbTest):
self.assertEqual(max(f['creation_date'] for f in dir_files), dir_stats['dir_max_file_creation_date'])
for subsubdir_nr in range(NUM_SUB_SUBDIRS):
- subsubDir_id = db.insertSubDirectory(subDir_id, 'subsubDir_%d' % subsubdir_nr)
+ subsubDir_id = db.insertSubDirectory('subsubDir_%d' % subsubdir_nr, subDir_id)
dir2subdir[subsubDir_id] = []
dir2subdir[subDir_id].append(subsubDir_id)
for kk in range(project_nr*subdir_nr*subsubdir_nr):
diff --git a/LTA/ltastorageoverview/test/test_lso_webservice.py b/LTA/ltastorageoverview/test/test_lso_webservice.py
index b074daca0b40eff0c6c2cbb729cb043ca2f244e9..c81e140777b5da48c96e59bdbd3459ff0311f8d1 100755
--- a/LTA/ltastorageoverview/test/test_lso_webservice.py
+++ b/LTA/ltastorageoverview/test/test_lso_webservice.py
@@ -90,8 +90,8 @@ def setUpModule():
webservice.db = store.LTAStorageDb(dbcreds, True)
logger.info('filling test LTASO database with test data')
- webservice.db.insertSite('siteA', 'srm://siteA.org')
- webservice.db.insertSite('siteB', 'srm://siteB.org')
+ webservice.db.insertSiteIfNotExists('siteA', 'srm://siteA.org')
+ webservice.db.insertSiteIfNotExists('siteB', 'srm://siteB.org')
rootDir_ids = []
rootDir_ids.append(webservice.db.insertRootDirectory('siteA', 'rootDir1'))
@@ -100,13 +100,13 @@ def setUpModule():
for rootDir_id in rootDir_ids:
for j in range(2):
- subDir_id = webservice.db.insertSubDirectory(rootDir_id, 'subDir_%d' % j)
+ subDir_id = webservice.db.insertSubDirectory('subDir_%d' % j, rootDir_id)
if j == 0:
webservice.db.insertFileInfo('file_%d' % j, 271 * (j + 1), datetime.datetime.utcnow(), subDir_id)
for k in range(2):
- subsubDir_id = webservice.db.insertSubDirectory(subDir_id, 'subsubDir_%d' % k)
+ subsubDir_id = webservice.db.insertSubDirectory('subsubDir_%d' % k, subDir_id)
for l in range((j + 1) * (k + 1)):
webservice.db.insertFileInfo('file_%d' % l, 314 * (l + 1), datetime.datetime.utcnow(), subsubDir_id)
diff --git a/LTA/ltastorageoverview/test/test_store.py b/LTA/ltastorageoverview/test/test_store.py
index 7e39ddbed4570aae23177856f4dc7e76e72b4f15..c3abd73231ddf1d62c8bbfec21a36da2fb5c9e3c 100755
--- a/LTA/ltastorageoverview/test/test_store.py
+++ b/LTA/ltastorageoverview/test/test_store.py
@@ -33,8 +33,8 @@ logger = logging.getLogger(__name__)
class TestLTAStorageDb(CommonLTAStorageDbTest):
def testSites(self):
with store.LTAStorageDb(self.dbcreds, True) as db:
- siteA_id = db.insertSite('siteA', 'srm://siteA.org')
- siteB_id = db.insertSite('siteB', 'srm://siteB.org')
+ siteA_id = db.insertSiteIfNotExists('siteA', 'srm://siteA.org')
+ siteB_id = db.insertSiteIfNotExists('siteB', 'srm://siteB.org')
sites = db.sites()
siteNames = [x['name'] for x in sites]
@@ -50,8 +50,8 @@ class TestLTAStorageDb(CommonLTAStorageDbTest):
def testRootDirs(self):
with store.LTAStorageDb(self.dbcreds, True) as db:
- siteA_id = db.insertSite('siteA', 'srm://siteA.org')
- siteB_id = db.insertSite('siteB', 'srm://siteB.org')
+ siteA_id = db.insertSiteIfNotExists('siteA', 'srm://siteA.org')
+ siteB_id = db.insertSiteIfNotExists('siteB', 'srm://siteB.org')
dirA1_id = db.insertRootDirectory('siteA', 'rootDir1')
dirA2_id = db.insertRootDirectory('siteA', 'rootDir2')
@@ -85,9 +85,23 @@ class TestLTAStorageDb(CommonLTAStorageDbTest):
self.assertEqual(siteB_id, rootDirsDict[dirB2_id]['site_id'])
self.assertEqual('siteB', rootDirsDict[dirB2_id]['site_name'])
+ root_dir_ids_siteA = set(d['root_dir_id'] for d in db.rootDirectoriesForSite(siteA_id))
+ self.assertEqual(set([dirA1_id, dirA2_id, dirA3_id]), root_dir_ids_siteA)
+
+ root_dir_ids_siteB = set(d['root_dir_id'] for d in db.rootDirectoriesForSite(siteB_id))
+ self.assertEqual(set([dirB1_id, dirB2_id]), root_dir_ids_siteB)
+
+ root_dirs_non_existing_site = db.rootDirectoriesForSite(999)
+ self.assertEqual([], root_dirs_non_existing_site)
+
+ def testNonExistingDir(self):
+ with store.LTAStorageDb(self.dbcreds, True) as db:
+ dir = db.directoryByName('fjsdka;58432aek5843rfsjd8-sa')
+ self.assertEqual(None, dir)
+
def testLeastRecentlyVisitedDirectory(self):
with store.LTAStorageDb(self.dbcreds, True) as db:
- db.insertSite('siteA', 'srm://siteA.org')
+ db.insertSiteIfNotExists('siteA', 'srm://siteA.org')
dir_ids = []
for i in range(3):
@@ -113,44 +127,44 @@ class TestLTAStorageDb(CommonLTAStorageDbTest):
def testDuplicateSubDirs(self):
with store.LTAStorageDb(self.dbcreds, True) as db:
- db.insertSite('siteA', 'srm://siteA.org')
- db.insertSite('siteB', 'srm://siteB.org')
+ db.insertSiteIfNotExists('siteA', 'srm://siteA.org')
+ db.insertSiteIfNotExists('siteB', 'srm://siteB.org')
dirA_id = db.insertRootDirectory('siteA', 'rootDir1')
dirB_id = db.insertRootDirectory('siteB', 'rootDir1')
- subDirA1_id = db.insertSubDirectory(dirA_id, 'foo')
- subDirA2_id = db.insertSubDirectory(dirA_id, 'bar')
- subDirB1_id = db.insertSubDirectory(dirB_id, 'foo')
+ subDirA1_id = db.insertSubDirectory('foo', dirA_id)
+ subDirA2_id = db.insertSubDirectory('bar', dirA_id)
+ subDirB1_id = db.insertSubDirectory('foo', dirB_id)
self.assertNotEquals(None, subDirA1_id)
self.assertNotEquals(None, subDirA2_id)
self.assertNotEquals(None, subDirB1_id)
- subDirA1a_id = db.insertSubDirectory(dirA_id, 'foo')
+ subDirA1a_id = db.insertSubDirectory('foo', dirA_id)
self.assertEquals(None, subDirA1a_id)
def testProjectsAndObservations(self):
with store.LTAStorageDb(self.dbcreds, True) as db:
#first insert a lot of data...
- db.insertSite('juelich', 'srm://lofar-srm.fz-juelich.de:8443')
- db.insertSite('sara', 'srm://srm.grid.sara.nl:8443')
+ db.insertSiteIfNotExists('juelich', 'srm://lofar-srm.fz-juelich.de:8443')
+ db.insertSiteIfNotExists('sara', 'srm://srm.grid.sara.nl:8443')
juelich_root_dir_id = db.insertRootDirectory('juelich', '/pnfs/fz-juelich.de/data/lofar/ops/')
sara_root_dir_id = db.insertRootDirectory('sara', '/pnfs/grid.sara.nl/data/lofar/ops')
- juelich_projects_dir_id = db.insertSubDirectory(juelich_root_dir_id, '/pnfs/fz-juelich.de/data/lofar/ops/projects')
- sara_projects_dir_id = db.insertSubDirectory(sara_root_dir_id, '/pnfs/grid.sara.nl/data/lofar/ops/projects')
+ juelich_projects_dir_id = db.insertSubDirectory('/pnfs/fz-juelich.de/data/lofar/ops/projects', juelich_root_dir_id)
+ sara_projects_dir_id = db.insertSubDirectory('/pnfs/grid.sara.nl/data/lofar/ops/projects', sara_root_dir_id)
for project_nr, project_name in enumerate(['lc8_001', '2017lofarobs', 'ddt5_001']):
# projects are sometimes stored at multiple sites
for projects_dir_id in [juelich_projects_dir_id, sara_projects_dir_id]:
- project_dir_id = db.insertSubDirectory(projects_dir_id,
- '/pnfs/fz-juelich.de/data/lofar/ops/projects/' + project_name)
+ project_dir_id = db.insertSubDirectory('/pnfs/fz-juelich.de/data/lofar/ops/projects/' + project_name,
+ projects_dir_id)
for obs_nr in range(3):
obs_name = 'L%06d' % ((project_nr+1)*1000 + obs_nr)
- obs_dir_id = db.insertSubDirectory(project_dir_id,
- '/pnfs/fz-juelich.de/data/lofar/ops/projects/' + project_name + '/' + obs_name)
+ obs_dir_id = db.insertSubDirectory('/pnfs/fz-juelich.de/data/lofar/ops/projects/' + project_name + '/' + obs_name,
+ project_dir_id)
for sb_nr in range(244):
file_name = '%s_SB%03d.MS.tar' % (obs_name, sb_nr)