diff --git a/.gitattributes b/.gitattributes
index d774ca5b7db852942edb127c41229c7c9c4abbe1..5442aaa7d7785bf767d2554e3391c767eaec2508 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -2829,8 +2829,11 @@ LTA/ltastorageoverview/CMakeLists.txt -text
LTA/ltastorageoverview/bin/CMakeLists.txt -text
LTA/ltastorageoverview/bin/ltastorageoverviewscraper -text
LTA/ltastorageoverview/lib/CMakeLists.txt -text
+LTA/ltastorageoverview/lib/__init__.py -text
LTA/ltastorageoverview/lib/scraper.py -text
LTA/ltastorageoverview/lib/store.py -text
+LTA/ltastorageoverview/lib/webservice/__init__.py -text
+LTA/ltastorageoverview/lib/webservice/templates/index.html -text
LTA/ltastorageoverview/lib/webservice/webservice.py -text
LTA/ltastorageoverview/test/CMakeLists.txt -text
LTA/ltastorageoverview/test/test_store.py -text
diff --git a/LTA/ltastorageoverview/lib/CMakeLists.txt b/LTA/ltastorageoverview/lib/CMakeLists.txt
index 9c40bde523c2bc76fb896cbf1162caf669c4e28c..081fd866c626128939175880cccf6ec5b758d4aa 100644
--- a/LTA/ltastorageoverview/lib/CMakeLists.txt
+++ b/LTA/ltastorageoverview/lib/CMakeLists.txt
@@ -1,8 +1,23 @@
# $Id: CMakeLists.txt 32341 2015-08-28 11:59:26Z schaap $
python_install(
+ __init__.py
scraper.py
store.py
webservice/webservice.py
+ webservice/__init__.py
DESTINATION ltastorageoverview)
+set(web_files webservice/templates/index.html)
+
+install(FILES ${web_files}
+ DESTINATION ltastorageoverview/webservice/templates/)
+
+foreach(web_file ${web_files})
+ get_filename_component(web_file_path ${web_file} PATH)
+ get_filename_component(web_file_abs ${web_file} ABSOLUTE)
+
+ file(MAKE_DIRECTORY ${_build_dir}/${web_file_path})
+ execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${web_file_abs} ${_build_dir}/${web_file})
+endforeach(web_file ${web_files})
+
diff --git a/LTA/ltastorageoverview/lib/__init__.py b/LTA/ltastorageoverview/lib/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..4222a93ee5cc228f9155c69f7a81a6285f59c0da
--- /dev/null
+++ b/LTA/ltastorageoverview/lib/__init__.py
@@ -0,0 +1,19 @@
+#!/usr/bin/python
+
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
diff --git a/LTA/ltastorageoverview/lib/store.py b/LTA/ltastorageoverview/lib/store.py
index 35c93037c8a8d4d5f2603472c7c0341098380e10..d7d22e50fdff33ebee9b2f07d143c22f300eb668 100644
--- a/LTA/ltastorageoverview/lib/store.py
+++ b/LTA/ltastorageoverview/lib/store.py
@@ -227,3 +227,13 @@ class LTAStorageDb:
where directory_id = ?
''', [directory_id]).fetchall()
+ def filesInTree(self, base_directory_id):
+ with sqlite3.connect(self.db_filename) as conn:
+ return conn.execute('''
+ SELECT dir.id, dir.name, fileinfo.id, fileinfo.name, fileinfo.size, fileinfo.creation_date FROM directory_closure
+ join directory dir on dir.id = directory_closure.descendant_id
+ join fileinfo on fileinfo.directory_id = directory_closure.descendant_id
+ where ancestor_id = ? and depth > 0
+ order by depth asc
+ ''', [base_directory_id]).fetchall()
+
diff --git a/LTA/ltastorageoverview/lib/webservice/__init__.py b/LTA/ltastorageoverview/lib/webservice/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..4222a93ee5cc228f9155c69f7a81a6285f59c0da
--- /dev/null
+++ b/LTA/ltastorageoverview/lib/webservice/__init__.py
@@ -0,0 +1,19 @@
+#!/usr/bin/python
+
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
diff --git a/LTA/ltastorageoverview/lib/webservice/templates/index.html b/LTA/ltastorageoverview/lib/webservice/templates/index.html
new file mode 100644
index 0000000000000000000000000000000000000000..a664bb6bf6096294ce3d2e9b67f40369ec0f0d2d
--- /dev/null
+++ b/LTA/ltastorageoverview/lib/webservice/templates/index.html
@@ -0,0 +1,50 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+ <title>{{title}}</title>
+
+ <script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/2.1.3/jquery.min.js"></script>
+ <script type="text/javascript" src="http://code.highcharts.com/highcharts.js"></script>
+ <script type="text/javascript" src="http://code.highcharts.com/modules/exporting.js"></script>
+</head>
+<body>
+ <script type="text/javascript">
+ $(function () {
+ $('#container').highcharts({
+ chart: {
+ plotBackgroundColor: null,
+ plotBorderWidth: null,
+ plotShadow: false,
+ type: 'pie'
+ },
+ title: {
+ text: 'Browser market shares January, 2015 to May, 2015'
+ },
+ tooltip: {
+ pointFormat: '{series.name}: <b>{point.percentage:.1f}%</b>'
+ },
+ plotOptions: {
+ pie: {
+ allowPointSelect: true,
+ cursor: 'pointer',
+ dataLabels: {
+ enabled: true,
+ format: '<b>{point.name}</b>: {point.percentage:.1f} %',
+ style: {
+ color: (Highcharts.theme && Highcharts.theme.contrastTextColor) || 'black'
+ }
+ }
+ }
+ },
+ series: [{
+ name: "StorageSiteUsage",
+ colorByPoint: true,
+ data: {{storagesitedata|safe}}
+ }]
+ });
+ });
+ </script>
+ <div id="container" style="min-width: 310px; min-height: 400px; margin: 0 auto"></div>
+</body>
+</html>
diff --git a/LTA/ltastorageoverview/lib/webservice/webservice.py b/LTA/ltastorageoverview/lib/webservice/webservice.py
index 3ff614feae401f41dbd8db44589ec60ae6384dd6..c70d32e570b2a4f11a6db71428f60fe57f81b187 100755
--- a/LTA/ltastorageoverview/lib/webservice/webservice.py
+++ b/LTA/ltastorageoverview/lib/webservice/webservice.py
@@ -20,47 +20,70 @@
import sys
import os
import os.path
-import flask
+from flask import Flask
+from flask import Config
+from flask import render_template
+from flask import json
import threading
from ltastorageoverview import store
-app = flask.Flask('LTA storage overview')
-db = None
+app = Flask('LTA storage overview')
+print str(app.config)
+print __file__
+print os.path.dirname(__file__)
+app.config.root_path = os.path.dirname(__file__)
+print
+print str(app.config)
+db = store.LTAStorageDb('/tmp/test.sqlite')
@app.route('/')
-def hello_world():
- return 'Hello World!'
+@app.route('/index.html')
+def index():
+ return render_template('index.html',
+ title='LTA storage overview',
+ storagesitedata='''[{name: "Microsoft Internet Explorer",y: 56.33}, {name: "Chrome",y: 24.03,sliced: true,selected: true}, {name: "Firefox",y: 10.38}, {name: "Safari",y: 4.77}, {name: "Opera",y: 0.91}, {name: "Proprietary or Undetectable",y: 0.2}]''')
@app.route('/rest/sites/')
def get_sites():
sites = {'sites': [{'id': x[0], 'name': x[1], 'url': x[2]} for x in db.sites()]}
- return flask.json.jsonify(sites)
+ return json.jsonify(sites)
@app.route('/rest/sites/<int:site_id>')
def get_site(site_id):
site = db.site(site_id)
site_dict = {'id': site[0], 'name': site[1], 'url': site[2]}
- return flask.json.jsonify(site_dict)
+ return json.jsonify(site_dict)
+
+@app.route('/rest/sites/usages')
+def get_sites_usages():
+ rootDirs = {'rootDirectories': [{'id': x[0], 'name': x[1], 'site_id': x[2], 'site_name': x[3]} for x in db.rootDirectories()]}
+
+ for rootDir in rootDirs['rootDirectories']:
+ print '\n'.join([str(x) for x in db.filesInTree(rootDir['id'])])
+ rootDir['usage'] = sum([x[4] for x in db.filesInTree(rootDir['id'])])
+ print
+
+ return json.jsonify(rootDirs)
@app.route('/rest/rootdirectories/',)
def get_rootDirectories():
rootDirs = {'rootDirectories': [{'id': x[0], 'name': x[1], 'site_id': x[2], 'site_name': x[3]} for x in db.rootDirectories()]}
- return flask.json.jsonify(rootDirs)
+ return json.jsonify(rootDirs)
@app.route('/rest/directory/<int:dir_id>/subdirectories/',)
def get_directoryTree(dir_id):
subDirsList = {'subdirectories': [{'id': x[0], 'name': x[1], 'parent_dir_id': x[2]} for x in db.subDirectories(dir_id, 1, False)]}
- return flask.json.jsonify(subDirsList)
+ return json.jsonify(subDirsList)
@app.route('/rest/directory/<int:dir_id>/files')
def get_filesInDirectory(dir_id):
files = {'files': [{'id': x[0], 'name': x[1], 'size': x[2], 'creation_date': x[3]} for x in db.filesInDirectory(dir_id)]}
- return flask.json.jsonify(files)
+ return json.jsonify(files)
def main(argv):
- db = store.LTAStorageDb(argv[0] if argv else 'ltastoragedb.sqlite')
- app.run()
+ #db = store.LTAStorageDb(argv[0] if argv else 'ltastoragedb.sqlite')
+ app.run(debug=True)
if __name__ == '__main__':
main(sys.argv[1:])
diff --git a/LTA/ltastorageoverview/test/test_store.py b/LTA/ltastorageoverview/test/test_store.py
index ff47f047a626c634df0f739bd4eb3fa902903bcd..9a66355d6ad86ffb5085e555312fd32ff68d15ba 100755
--- a/LTA/ltastorageoverview/test/test_store.py
+++ b/LTA/ltastorageoverview/test/test_store.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python
import unittest
+import datetime
import os
import os.path
import tempfile
@@ -14,9 +15,9 @@ class TestLTAStorageDb(unittest.TestCase):
self.assertTrue(os.path.exists(self.db.db_filename))
- def tearDown(self):
- if os.path.exists(self.db.db_filename):
- os.remove(self.db.db_filename)
+ #def tearDown(self):
+ #if os.path.exists(self.db.db_filename):
+ #os.remove(self.db.db_filename)
def testSites(self):
self.db.insertSite('siteA', 'srm://siteA.org')
@@ -62,9 +63,11 @@ class TestLTAStorageDb(unittest.TestCase):
for j in range(2):
subDir_id = self.db.insertSubDirectory(rootDir_id, 'subDir_%d' % j)
+ self.db.insertFileInfo('file_%d' % j, 271*(j+1), datetime.datetime.utcnow(), subDir_id)
for k in range(2):
subsubDir_id = self.db.insertSubDirectory(subDir_id, 'subsubDir_%d' % k)
+ self.db.insertFileInfo('file_%d_%d' % (j,k), 314*(k+1), datetime.datetime.utcnow(), subsubDir_id)
rootDirs = self.db.rootDirectories()
self.assertEquals(2, len(rootDirs))
@@ -75,6 +78,8 @@ class TestLTAStorageDb(unittest.TestCase):
subDir_parent_id = subDir[2]
self.assertEquals(id, subDir_parent_id)
+ print '\n'.join([str(x) for x in self.db.filesInTree(rootDir_id)])
+
# run tests if main
if __name__ == '__main__':
unittest.main(verbosity=2)
diff --git a/LTA/ltastorageoverview/test/test_webservice.py b/LTA/ltastorageoverview/test/test_webservice.py
index de0dc44dc3d0a57731442b0761c2d7741cd5ed96..536c11da0f851a2263193d12dccef099c40f2bc8 100755
--- a/LTA/ltastorageoverview/test/test_webservice.py
+++ b/LTA/ltastorageoverview/test/test_webservice.py
@@ -12,7 +12,7 @@ import datetime
from StringIO import StringIO
from flask.ext.testing import LiveServerTestCase as FlaskLiveTestCase
from ltastorageoverview import store
-import ltastorageoverview.webservice.webservice as webservice
+from ltastorageoverview.webservice import webservice as webservice
def setUpModule():
tmpfile = os.path.join(tempfile.gettempdir(), 'test.sqlite')
@@ -31,13 +31,13 @@ def setUpModule():
rootDir_ids.append(webservice.db.insertRootDirectory('siteB', 'path/to/rootDir3'))
for rootDir_id in rootDir_ids:
- for j in range(3):
+ for j in range(2):
subDir_id = webservice.db.insertSubDirectory(rootDir_id, 'subDir_%d' % j)
if j == 0:
webservice.db.insertFileInfo('file_%d' % j, 271*(j+1), datetime.datetime.utcnow(), subDir_id)
- for k in range(5):
+ for k in range(2):
subsubDir_id = webservice.db.insertSubDirectory(subDir_id, 'subsubDir_%d' % k)
for l in range((j+1)*(k+1)):
@@ -49,6 +49,7 @@ def tearDownModule():
class TestLTAStorageWebService(FlaskLiveTestCase):
def create_app(self):
+ webservice.app.debug = True
return webservice.app
def testSites(self):