Auto organisation of the navbar -> Web page are auto-inserted. Created skeleton webpage

var/www/Flask_server.py

@@ -27,20 +27,49 @@ cfg = Flask_config.cfg
 Flask_config.app = Flask(__name__, static_url_path='/static/')
 app = Flask_config.app
 
-# dynamically import routes and functions from modules
+# Dynamically import routes and functions from modules
+# Also, prepare header.html
+to_add_to_header_dico = {}
 for root, dirs, files in os.walk('modules/'):
-    for directory in dirs:
+    sys.path.append(join(root))
-        #if directory == 'templates':
-        #    continue
-        sys.path.append(join(root, directory))
     for name in files:
+        module_name = root.split('/')[-2]
         if name.startswith('Flask_') and name.endswith('.py'):
             if name == 'Flask_config.py':
                 continue
             name = name.strip('.py')
-                print('importing {}'.format(name))
+            #print('importing {}'.format(name))
             importlib.import_module(name)
+        elif name == 'header_{}.html'.format(module_name):
+            with open(join(root, name), 'r') as f:
+                to_add_to_header_dico[module_name] = f.read()
 
+#create header.html
+complete_header = ""
+with open('templates/header_base.html', 'r') as f:
+    complete_header = f.read()
+modified_header = complete_header
+
+#Add the header in the supplied order
+for module_name, txt in to_add_to_header_dico.items():
+    to_replace = '<!--{}-->'.format(module_name)
+    if to_replace in complete_header:
+        modified_header = modified_header.replace(to_replace, txt)
+        del to_add_to_header_dico[module_name]
+
+#Add the header for no-supplied order
+to_add_to_header = []
+for module_name, txt in to_add_to_header_dico.items():
+    to_add_to_header.append(txt)
+
+modified_header = modified_header.replace('<!--insert here-->', '\n'.join(to_add_to_header))
+
+#Write the header.html file
+with open('templates/header.html', 'w') as f:
+    f.write(modified_header)
+
+
+# ========= JINJA2 FUNCTIONS ========
 def list_len(s):
     return len(s)
 app.jinja_env.filters['list_len'] = list_len
@@ -57,6 +86,12 @@ def add_header(response):
     response.headers['Cache-Control'] = 'public, max-age=0'
     return response
 
+# ========== ROUTES ============
+@app.route('/searchbox/')
+def searchbox():
+    return render_template("searchbox.html")
+
+
 # ============ MAIN ============
 
 if __name__ == "__main__":

var/www/modules/browsepastes/Flask_browsepastes.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+
+'''
+    Flask functions and routes for the trending modules page
+'''
+import redis
+import json
+import flask
+from flask import Flask, render_template, jsonify, request, Blueprint
+
+import Paste
+
+# ============ VARIABLES ============
+import Flask_config
+
+app = Flask_config.app
+cfg = Flask_config.cfg
+max_preview_char = Flask_config.max_preview_char
+max_preview_modal = Flask_config.max_preview_modal
+r_serv_db = Flask_config.r_serv_db
+
+browsepastes = Blueprint('browsepastes', __name__, template_folder='templates')
+
+# ============ FUNCTIONS ============
+
+def getPastebyType(server, module_name):
+    all_path = []
+    for path in server.smembers('WARNING_'+module_name):
+        all_path.append(path)
+    return all_path
+
+
+def event_stream_getImportantPasteByModule(module_name):
+    index = 0
+    all_pastes_list = getPastebyType(r_serv_db, module_name)
+    for path in all_pastes_list:
+        index += 1
+        paste = Paste.Paste(path)
+        content = paste.get_p_content().decode('utf8', 'ignore')
+        content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
+        curr_date = str(paste._get_p_date())
+        curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
+        data = {}
+        data["module"] = module_name
+        data["index"] = index
+        data["path"] = path
+        data["content"] = content[0:content_range]
+        data["linenum"] = paste.get_lines_info()[0]
+        data["date"] = curr_date
+        data["char_to_display"] = max_preview_modal
+        data["finished"] = True if index == len(all_pastes_list) else False
+        yield 'retry: 100000\ndata: %s\n\n' % json.dumps(data) #retry to avoid reconnection of the browser
+
+# ============ ROUTES ============
+
+@browsepastes.route("/browseImportantPaste/", methods=['GET'])
+def browseImportantPaste():
+    module_name = request.args.get('moduleName')
+    return render_template("browse_important_paste.html")
+
+
+@browsepastes.route("/importantPasteByModule/", methods=['GET'])
+def importantPasteByModule():
+    module_name = request.args.get('moduleName')
+
+    all_content = []
+    paste_date = []
+    paste_linenum = []
+    all_path = []
+    allPastes = getPastebyType(r_serv_db, module_name)
+
+    for path in allPastes[0:10]:
+        all_path.append(path)
+        paste = Paste.Paste(path)
+        content = paste.get_p_content().decode('utf8', 'ignore')
+        content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
+        all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " "))
+        curr_date = str(paste._get_p_date())
+        curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
+        paste_date.append(curr_date)
+        paste_linenum.append(paste.get_lines_info()[0])
+
+    if len(allPastes) > 10:
+        finished = False
+    else:
+        finished = True
+
+    return render_template("important_paste_by_module.html",
+            moduleName=module_name, 
+            all_path=all_path, 
+            content=all_content, 
+            paste_date=paste_date, 
+            paste_linenum=paste_linenum, 
+            char_to_display=max_preview_modal, 
+            finished=finished)
+
+@browsepastes.route("/_getImportantPasteByModule")
+def getImportantPasteByModule():
+    module_name = request.args.get('moduleName')
+    return flask.Response(event_stream_getImportantPasteByModule(module_name), mimetype="text/event-stream")
+
+
+# ========= REGISTRATION =========
+app.register_blueprint(browsepastes)

var/www/modules/browsepastes/templates/browse_important_paste.html

@@ -35,24 +35,7 @@
 </head>
 <body>
 
-    <div id="wrapper">
+   {% include 'navbar.html' %}
-      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
-      {% include 'header.html' %}
-      <!-- /.navbar-top-links -->
-          <div class="navbar-default sidebar" role="navigation">
-              <div class="sidebar-collapse">
-                  <ul class="nav" id="side-menu">
-                      <li class="sidebar-search">
-                          {% include 'searchbox.html' %}
-                      </li>
-                 </ul>
-                  <!-- /#side-menu -->
-              </div>
-              <!-- /.sidebar-collapse -->
-           <a  href="{{ url_for('dashboard.index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
-          </div>
-          <!-- /.navbar-static-side -->
-      </nav>
 
 <!-- Modal -->
 <div id="mymodal" class="modal fade" role="dialog">

var/www/modules/browsepastes/templates/header_browsepastes.html

@@ -0,0 +1 @@
+<li id='page-browse'><a href="{{ url_for('browsepastes.browseImportantPaste') }}"><i class="fa fa-search-plus "></i> Browse important pastes</a></li>

var/www/modules/dashboard/templates/header_dashboard.html

@@ -0,0 +1 @@
+<li id='page-index'><a href="{{ url_for('dashboard.index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li>

var/www/modules/search/templates/search.html

@@ -37,23 +37,7 @@
 </head>
 <body>
 
-    <div id="wrapper">
+   {% include 'navbar.html' %}
-      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
-          {% include 'header.html' %}
-          <!-- /.navbar-top-links -->
-          <div class="navbar-default sidebar" role="navigation">
-              <div class="sidebar-collapse">
-                  <ul class="nav" id="side-menu">
-                 {% include 'searchbox.html' %}
-                 </ul>
-                  <!-- /#side-menu -->
-              </div>
-              <!-- /.sidebar-collapse -->
-           <a  href="{{ url_for('dashboard.index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
-          </div>
-          <!-- /.navbar-static-side -->
-      </nav>
-
 
       <!-- Modal -->
       <div id="mymodal" class="modal fade" role="dialog">

var/www/modules/sentiment/templates/header_sentiment.html

@@ -0,0 +1,7 @@
+<li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-heart"></i> Sentiment Analysis
+    <span class="caret"></span></a>
+    <ul class="dropdown-menu">
+      <li><a href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-bar-chart-o"> </i> Sentiment trending</a></li>
+      <li><a href="{{ url_for('sentiments.sentiment_analysis_plot_tool') }}"><i class="fa fa-wrench"> </i> Sentiment plot Tool</a></li>
+    </ul>
+</li>

var/www/modules/sentiment/templates/sentiment_analysis_plot_tool.html

@@ -31,25 +31,7 @@
 </head>
 <body>
 
-   <div id="wrapper">
+   {% include 'navbar.html' %}
-      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
-      {% include 'header.html' %}
-      <!-- /.navbar-top-links -->
-          <div class="navbar-default sidebar" role="navigation">
-              <div class="sidebar-collapse">
-                  <ul class="nav" id="side-menu">
-                      <li class="sidebar-search">
-                          {% include 'searchbox.html' %}
-                      </li>
-                 </ul>
-                  <!-- /#side-menu -->
-              </div>
-              <!-- /.sidebar-collapse -->
-           <a  href="{{ url_for('dashboard.index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
-          </div>
-          <!-- /.navbar-static-side -->
-      </nav>
-   </div>
 
 <div id="page-wrapper">
   <div class="row">

var/www/modules/sentiment/templates/sentiment_analysis_trending.html

@@ -84,25 +84,7 @@
 </head>
 <body>
 
-   <div id="wrapper">
+   {% include 'navbar.html' %}
-      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
-      {% include 'header.html' %}
-      <!-- /.navbar-top-links -->
-          <div class="navbar-default sidebar" role="navigation">
-              <div class="sidebar-collapse">
-                  <ul class="nav" id="side-menu">
-                      <li class="sidebar-search">
-                          {% include 'searchbox.html' %}
-                      </li>
-                 </ul>
-                  <!-- /#side-menu -->
-              </div>
-              <!-- /.sidebar-collapse -->
-           <a  href="{{ url_for('dashboard.index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
-          </div>
-          <!-- /.navbar-static-side -->
-      </nav>
-   </div>
 
 <div id="page-wrapper">
   <div class="row">

var/www/modules/skeleton/Flask_skeleton.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+
+'''
+    Flask functions and routes for the trending modules page
+'''
+import redis
+from flask import Flask, render_template, jsonify, request, Blueprint
+
+# ============ VARIABLES ============
+import Flask_config
+
+app = Flask_config.app
+cfg = Flask_config.cfg
+
+skeleton = Blueprint('skeleton', __name__, template_folder='templates')
+
+# ============ FUNCTIONS ============
+def one():
+    return 1
+
+# ============= ROUTES ==============
+
+@skeleton.route("/skeleton/", methods=['GET'])
+def skeleton_page():
+    return render_template("skeleton.html")
+
+
+# ========= REGISTRATION =========
+app.register_blueprint(skeleton)

var/www/modules/skeleton/templates/header_skeleton.html

@@ -0,0 +1 @@
+<li id='page-skeleton'><a href="{{ url_for('skeleton.skeleton_page') }}"><i class="glyphicon glyphicon-new-window"></i> Skeleton page</a></li>

var/www/modules/skeleton/templates/skeleton.html

@@ -0,0 +1,41 @@
+<!DOCTYPE html>
+<html>
+
+	<head>
+	  <meta charset="utf-8">
+	  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+	  <title>Analysis Information Leak framework Dashboard</title>
+
+	  <!-- Core CSS -->
+	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+	  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
+	  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
+	  <link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
+	  <!-- JS -->
+	  <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
+	  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+          <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
+          <script src="{{ url_for('static', filename='js/jquery.flot.pie.js') }}"></script>
+          <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
+
+	</head>
+	<body>
+
+            {% include 'navbar.html' %}
+
+            <div id="page-wrapper">
+                <p> Text </p>
+            </div>
+            <!-- /#page-wrapper -->
+
+            <script>
+                $(document).ready(function(){
+                    activePage = "page-skeleton"
+                    $("#"+activePage).addClass("active");
+                });
+            </script>
+            <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
+        </body>
+
+</html>

var/www/modules/terms/templates/header_terms.html

@@ -0,0 +1,8 @@
+<li id='page-termsfrequency'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-eye"></i> Terms frequency
+    <span class="caret"></span></a>
+    <ul class="dropdown-menu">
+      <li><a href="{{ url_for('terms.terms_management') }}"><i class="fa fa-gear "> </i> Terms managements</a></li>
+      <li><a href="{{ url_for('terms.terms_plot_top') }}"><i class="glyphicon glyphicon-fire"> </i> Terms plot top</a></li>
+      <li><a href="{{ url_for('terms.terms_plot_tool') }}"><i class="fa fa-wrench"> </i> Terms plot tool</a></li>
+    </ul>
+</li>

var/www/modules/terms/templates/terms_management.html

@@ -57,25 +57,7 @@
       </div>
     </div>
 
-   <div id="wrapper">
+   {% include 'navbar.html' %}
-      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
-      {% include 'header.html' %}
-      <!-- /.navbar-top-links -->
-          <div class="navbar-default sidebar" role="navigation">
-              <div class="sidebar-collapse">
-                  <ul class="nav" id="side-menu">
-                      <li class="sidebar-search">
-                          {% include 'searchbox.html' %}
-                      </li>
-                 </ul>
-                  <!-- /#side-menu -->
-              </div>
-              <!-- /.sidebar-collapse -->
-           <a  href="{{ url_for('dashboard.index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
-          </div>
-          <!-- /.navbar-static-side -->
-      </nav>
-   </div>
 
 <div id="page-wrapper">
   <div class="row">

var/www/modules/terms/templates/terms_plot_tool.html

@@ -32,25 +32,7 @@
 </head>
 <body>
 
-   <div id="wrapper">
+   {% include 'navbar.html' %}
-      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
-      {% include 'header.html' %}
-      <!-- /.navbar-top-links -->
-          <div class="navbar-default sidebar" role="navigation">
-              <div class="sidebar-collapse">
-                  <ul class="nav" id="side-menu">
-                      <li class="sidebar-search">
-                          {% include 'searchbox.html' %}
-                      </li>
-                 </ul>
-                  <!-- /#side-menu -->
-              </div>
-              <!-- /.sidebar-collapse -->
-           <a  href="{{ url_for('dashboard.index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
-          </div>
-          <!-- /.navbar-static-side -->
-      </nav>
-   </div>
 
 <div id="page-wrapper">
   <div class="row">

var/www/modules/terms/templates/terms_plot_top.html

@@ -33,25 +33,7 @@
 </head>
 <body>
 
-   <div id="wrapper">
+   {% include 'navbar.html' %}
-      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
-      {% include 'header.html' %}
-      <!-- /.navbar-top-links -->
-          <div class="navbar-default sidebar" role="navigation">
-              <div class="sidebar-collapse">
-                  <ul class="nav" id="side-menu">
-                      <li class="sidebar-search">
-                          {% include 'searchbox.html' %}
-                      </li>
-                 </ul>
-                  <!-- /#side-menu -->
-              </div>
-              <!-- /.sidebar-collapse -->
-           <a  href="{{ url_for('dashboard.index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
-          </div>
-          <!-- /.navbar-static-side -->
-      </nav>
-   </div>
 
 <div id="page-wrapper">
   <div class="row">

var/www/modules/trendingcharts/templates/Trending.html

@@ -30,24 +30,8 @@
 	</head>
 	<body>
 
-	    <div id="wrapper">
+            {% include 'navbar.html' %}
-	      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
+
-                  {% include 'header.html' %}
-          <!-- /.navbar-top-links -->
-          <div class="navbar-default sidebar" role="navigation">
-              <div class="sidebar-collapse">
-                  <ul class="nav" id="side-menu">
-                      <li class="sidebar-search">
-                          {% include 'searchbox.html' %}
-                      </li>
-                 </ul>
-                  <!-- /#side-menu -->
-              </div>
-              <!-- /.sidebar-collapse -->
-           <a  href="{{ url_for('dashboard.index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
-          </div>
-          <!-- /.navbar-static-side -->
-      </nav>
         <div id="page-wrapper">
           <div class="row">
               <div class="col-lg-12">

var/www/modules/trendingcharts/templates/header_trendingcharts.html

@@ -0,0 +1 @@
+<li id='page-trendingchart'><a href="{{ url_for('trendings.trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>

var/www/modules/trendingmodules/templates/Moduletrending.html

@@ -22,24 +22,8 @@
 	</head>
 	<body>
 
-	    <div id="wrapper">
+            {% include 'navbar.html' %}
-	      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
+
-              {% include 'header.html' %}
-          <!-- /.navbar-top-links -->
-          <div class="navbar-default sidebar" role="navigation">
-              <div class="sidebar-collapse">
-                  <ul class="nav" id="side-menu">
-                      <li class="sidebar-search">
-                          {% include 'searchbox.html' %}
-                      </li>
-                 </ul>
-                  <!-- /#side-menu -->
-              </div>
-              <!-- /.sidebar-collapse -->
-           <a  href="{{ url_for('dashboard.index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
-          </div>
-          <!-- /.navbar-static-side -->
-      </nav>
         <div id="page-wrapper">
            </br>
            {% include 'trending_graphs/Moduletrending.html' %}

var/www/modules/trendingmodules/templates/header_trendingmodules.html

@@ -0,0 +1 @@
+<li id='page-modulestats'><a href="{{ url_for('trendingmodules.moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>

var/www/templates/header.html

@@ -1,9 +1,13 @@
 <div class="navbar-header">
     <ul class="nav navbar-nav">
 	<li id='page-index'><a href="{{ url_for('dashboard.index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li>
+
 	<li id='page-trendingchart'><a href="{{ url_for('trendings.trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>
+
 	<li id='page-modulestats'><a href="{{ url_for('trendingmodules.moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>
+
 	<li id='page-browse'><a href="{{ url_for('browsepastes.browseImportantPaste') }}"><i class="fa fa-search-plus "></i> Browse important pastes</a></li>
+
 	<li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-heart"></i> Sentiment Analysis
     <span class="caret"></span></a>
     <ul class="dropdown-menu">
@@ -11,6 +15,7 @@
       <li><a href="{{ url_for('sentiments.sentiment_analysis_plot_tool') }}"><i class="fa fa-wrench"> </i> Sentiment plot Tool</a></li>
     </ul>
 </li>
+
         <li id='page-termsfrequency'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-eye"></i> Terms frequency
     <span class="caret"></span></a>
     <ul class="dropdown-menu">
@@ -19,5 +24,8 @@
       <li><a href="{{ url_for('terms.terms_plot_tool') }}"><i class="fa fa-wrench"> </i> Terms plot tool</a></li>
     </ul>
 </li>
+
+	<li id='page-skeleton'><a href="{{ url_for('skeleton.skeleton_page') }}"><i class="glyphicon glyphicon-new-window"></i> Skeleton page</a></li>
+
     </ul>
 </div>