Task #8291: intermediate commit

.gitattributes

@@ -2829,8 +2829,11 @@ LTA/ltastorageoverview/CMakeLists.txt -text
 LTA/ltastorageoverview/bin/CMakeLists.txt -text
 LTA/ltastorageoverview/bin/ltastorageoverviewscraper -text
 LTA/ltastorageoverview/lib/CMakeLists.txt -text
+LTA/ltastorageoverview/lib/__init__.py -text
 LTA/ltastorageoverview/lib/scraper.py -text
 LTA/ltastorageoverview/lib/store.py -text
+LTA/ltastorageoverview/lib/webservice/__init__.py -text
+LTA/ltastorageoverview/lib/webservice/templates/index.html -text
 LTA/ltastorageoverview/lib/webservice/webservice.py -text
 LTA/ltastorageoverview/test/CMakeLists.txt -text
 LTA/ltastorageoverview/test/test_store.py -text

LTA/ltastorageoverview/lib/CMakeLists.txt

 # $Id: CMakeLists.txt 32341 2015-08-28 11:59:26Z schaap $
 
 python_install(
+    __init__.py
     scraper.py
     store.py
     webservice/webservice.py
+    webservice/__init__.py
     DESTINATION ltastorageoverview)
 
+set(web_files webservice/templates/index.html)
+
+install(FILES ${web_files}
+  DESTINATION ltastorageoverview/webservice/templates/)
+
+foreach(web_file ${web_files})
+    get_filename_component(web_file_path ${web_file} PATH)
+    get_filename_component(web_file_abs ${web_file} ABSOLUTE)
+
+    file(MAKE_DIRECTORY ${_build_dir}/${web_file_path})
+    execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${web_file_abs} ${_build_dir}/${web_file})
+endforeach(web_file ${web_files})
+

LTA/ltastorageoverview/lib/__init__.py

+#!/usr/bin/python
+
+# Copyright (C) 2012-2015  ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+

LTA/ltastorageoverview/lib/store.py

@@ -227,3 +227,13 @@ class LTAStorageDb:
                 where directory_id = ?
                 ''', [directory_id]).fetchall()
 
+    def filesInTree(self, base_directory_id):
+        with sqlite3.connect(self.db_filename) as conn:
+            return conn.execute('''
+                SELECT dir.id, dir.name, fileinfo.id, fileinfo.name, fileinfo.size, fileinfo.creation_date FROM directory_closure
+                join directory dir on dir.id = directory_closure.descendant_id
+                join fileinfo on fileinfo.directory_id = directory_closure.descendant_id
+                where ancestor_id = ? and depth > 0
+                order by depth asc
+                ''', [base_directory_id]).fetchall()
+

LTA/ltastorageoverview/lib/webservice/__init__.py

+#!/usr/bin/python
+
+# Copyright (C) 2012-2015  ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+

LTA/ltastorageoverview/lib/webservice/templates/index.html

+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+    <title>{{title}}</title>
+
+    <script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/2.1.3/jquery.min.js"></script>
+    <script type="text/javascript" src="http://code.highcharts.com/highcharts.js"></script>
+    <script type="text/javascript" src="http://code.highcharts.com/modules/exporting.js"></script>
+</head>
+<body>
+    <script type="text/javascript">
+        $(function () {
+        $('#container').highcharts({
+        chart: {
+        plotBackgroundColor: null,
+        plotBorderWidth: null,
+        plotShadow: false,
+        type: 'pie'
+        },
+        title: {
+        text: 'Browser market shares January, 2015 to May, 2015'
+        },
+        tooltip: {
+        pointFormat: '{series.name}: <b>{point.percentage:.1f}%</b>'
+        },
+        plotOptions: {
+        pie: {
+        allowPointSelect: true,
+        cursor: 'pointer',
+        dataLabels: {
+        enabled: true,
+        format: '<b>{point.name}</b>: {point.percentage:.1f} %',
+        style: {
+        color: (Highcharts.theme && Highcharts.theme.contrastTextColor) || 'black'
+        }
+        }
+        }
+        },
+        series: [{
+        name: "StorageSiteUsage",
+        colorByPoint: true,
+        data: {{storagesitedata|safe}}
+        }]
+        });
+        });
+        </script>
+    <div id="container" style="min-width: 310px; min-height: 400px; margin: 0 auto"></div>
+</body>
+</html>

LTA/ltastorageoverview/lib/webservice/webservice.py

@@ -20,47 +20,70 @@
 import sys
 import os
 import os.path
-import flask
+from flask import Flask
+from flask import Config
+from flask import render_template
+from flask import json
 import threading
 from ltastorageoverview import store
 
-app = flask.Flask('LTA storage overview')
-db = None
+app = Flask('LTA storage overview')
+print str(app.config)
+print __file__
+print os.path.dirname(__file__)
+app.config.root_path = os.path.dirname(__file__)
+print
+print str(app.config)
+db = store.LTAStorageDb('/tmp/test.sqlite')
 
 @app.route('/')
-def hello_world():
-    return 'Hello World!'
+@app.route('/index.html')
+def index():
+    return render_template('index.html',
+                           title='LTA storage overview',
+                           storagesitedata='''[{name: "Microsoft Internet Explorer",y: 56.33}, {name: "Chrome",y: 24.03,sliced: true,selected: true}, {name: "Firefox",y: 10.38}, {name: "Safari",y: 4.77}, {name: "Opera",y: 0.91}, {name: "Proprietary or Undetectable",y: 0.2}]''')
 
 @app.route('/rest/sites/')
 def get_sites():
     sites = {'sites': [{'id': x[0], 'name': x[1], 'url': x[2]} for x in db.sites()]}
-    return flask.json.jsonify(sites)
+    return json.jsonify(sites)
 
 @app.route('/rest/sites/<int:site_id>')
 def get_site(site_id):
     site = db.site(site_id)
     site_dict = {'id': site[0], 'name': site[1], 'url': site[2]}
-    return flask.json.jsonify(site_dict)
+    return json.jsonify(site_dict)
+
+@app.route('/rest/sites/usages')
+def get_sites_usages():
+    rootDirs = {'rootDirectories': [{'id': x[0], 'name': x[1], 'site_id': x[2], 'site_name': x[3]} for x in db.rootDirectories()]}
+
+    for rootDir in rootDirs['rootDirectories']:
+        print '\n'.join([str(x) for x in db.filesInTree(rootDir['id'])])
+        rootDir['usage'] = sum([x[4] for x in db.filesInTree(rootDir['id'])])
+        print
+
+    return json.jsonify(rootDirs)
 
 @app.route('/rest/rootdirectories/',)
 def get_rootDirectories():
     rootDirs = {'rootDirectories': [{'id': x[0], 'name': x[1], 'site_id': x[2], 'site_name': x[3]} for x in db.rootDirectories()]}
-    return flask.json.jsonify(rootDirs)
+    return json.jsonify(rootDirs)
 
 @app.route('/rest/directory/<int:dir_id>/subdirectories/',)
 def get_directoryTree(dir_id):
     subDirsList = {'subdirectories': [{'id': x[0], 'name': x[1], 'parent_dir_id': x[2]} for x in db.subDirectories(dir_id, 1, False)]}
-    return flask.json.jsonify(subDirsList)
+    return json.jsonify(subDirsList)
 
 @app.route('/rest/directory/<int:dir_id>/files')
 def get_filesInDirectory(dir_id):
     files = {'files': [{'id': x[0], 'name': x[1], 'size': x[2], 'creation_date': x[3]} for x in db.filesInDirectory(dir_id)]}
-    return flask.json.jsonify(files)
+    return json.jsonify(files)
 
 
 def main(argv):
-    db = store.LTAStorageDb(argv[0] if argv else 'ltastoragedb.sqlite')
-    app.run()
+    #db = store.LTAStorageDb(argv[0] if argv else 'ltastoragedb.sqlite')
+    app.run(debug=True)
 
 if __name__ == '__main__':
     main(sys.argv[1:])

LTA/ltastorageoverview/test/test_store.py

 #!/usr/bin/env python
 
 import unittest
+import datetime
 import os
 import os.path
 import tempfile
@@ -14,9 +15,9 @@ class TestLTAStorageDb(unittest.TestCase):
 
         self.assertTrue(os.path.exists(self.db.db_filename))
 
-    def tearDown(self):
-        if os.path.exists(self.db.db_filename):
-            os.remove(self.db.db_filename)
+    #def tearDown(self):
+        #if os.path.exists(self.db.db_filename):
+            #os.remove(self.db.db_filename)
 
     def testSites(self):
         self.db.insertSite('siteA', 'srm://siteA.org')
@@ -62,9 +63,11 @@ class TestLTAStorageDb(unittest.TestCase):
 
             for j in range(2):
                 subDir_id = self.db.insertSubDirectory(rootDir_id, 'subDir_%d' % j)
+                self.db.insertFileInfo('file_%d' % j, 271*(j+1), datetime.datetime.utcnow(), subDir_id)
 
                 for k in range(2):
                     subsubDir_id = self.db.insertSubDirectory(subDir_id, 'subsubDir_%d' % k)
+                    self.db.insertFileInfo('file_%d_%d' % (j,k), 314*(k+1), datetime.datetime.utcnow(), subsubDir_id)
 
         rootDirs = self.db.rootDirectories()
         self.assertEquals(2, len(rootDirs))
@@ -75,6 +78,8 @@ class TestLTAStorageDb(unittest.TestCase):
                 subDir_parent_id = subDir[2]
                 self.assertEquals(id, subDir_parent_id)
 
+        print '\n'.join([str(x) for x in self.db.filesInTree(rootDir_id)])
+
 # run tests if main
 if __name__ == '__main__':
     unittest.main(verbosity=2)

LTA/ltastorageoverview/test/test_webservice.py

@@ -12,7 +12,7 @@ import datetime
 from StringIO import StringIO
 from flask.ext.testing import LiveServerTestCase as FlaskLiveTestCase
 from ltastorageoverview import store
-import ltastorageoverview.webservice.webservice as webservice
+from ltastorageoverview.webservice import webservice as webservice
 
 def setUpModule():
     tmpfile = os.path.join(tempfile.gettempdir(), 'test.sqlite')
@@ -31,13 +31,13 @@ def setUpModule():
     rootDir_ids.append(webservice.db.insertRootDirectory('siteB', 'path/to/rootDir3'))
 
     for rootDir_id in rootDir_ids:
-        for j in range(3):
+        for j in range(2):
             subDir_id = webservice.db.insertSubDirectory(rootDir_id, 'subDir_%d' % j)
 
             if j == 0:
                 webservice.db.insertFileInfo('file_%d' % j, 271*(j+1), datetime.datetime.utcnow(), subDir_id)
 
-            for k in range(5):
+            for k in range(2):
                 subsubDir_id = webservice.db.insertSubDirectory(subDir_id, 'subsubDir_%d' % k)
 
                 for l in range((j+1)*(k+1)):
@@ -49,6 +49,7 @@ def tearDownModule():
 
 class TestLTAStorageWebService(FlaskLiveTestCase):
     def create_app(self):
+        webservice.app.debug = True
         return webservice.app
 
     def testSites(self):