Merge pull request #59 from mokaddem/processed-pastes-graph

Processed pastes graph

bin/Attribute.py

@@ -51,12 +51,13 @@ if __name__ == "__main__":
                 PST = Paste.Paste(message)
             else:
                 publisher.debug("Script Attribute is idling 1s")
+                print 'sleeping'
                 time.sleep(1)
                 continue
 
             # FIXME do it directly in the class
             PST.save_attribute_redis("p_encoding", PST._get_p_encoding())
-            PST.save_attribute_redis("p_language", PST._get_p_language())
+            #PST.save_attribute_redis("p_language", PST._get_p_language())
             # FIXME why not all saving everything there.
             PST.save_all_attributes_redis()
             # FIXME Not used.

bin/Global.py

@@ -31,6 +31,8 @@ from Helper import Process
 if __name__ == '__main__':
     publisher.port = 6380
     publisher.channel = 'Script'
+    processed_paste = 0
+    time_1 = time.time()
 
     config_section = 'Global'
 
@@ -54,6 +56,12 @@ if __name__ == '__main__':
                 continue
         else:
             print "Empty Queues: Waiting..."
+            if int(time.time() - time_1) > 30:
+                to_print = 'Global; ; ; ;glob Processed {0} paste(s)'.format(processed_paste)
+                print to_print
+                publisher.info(to_print)
+                time_1 = time.time()
+                processed_paste = 0
             time.sleep(1)
             continue
         # Creating the full filepath
@@ -66,3 +74,4 @@ if __name__ == '__main__':
         with open(filename, 'wb') as f:
             f.write(base64.standard_b64decode(gzip64encoded))
         p.populate_set_out(filename)
+        processed_paste+=1

bin/packages/config.cfg.sample

@@ -4,6 +4,11 @@ pastes = PASTES
 wordtrending_csv = var/www/static/csv/wordstrendingdata
 wordsfile = files/wordfile
 
+##### Flask #####
+[Flask]
+#Number of minutes displayed for the number of processed pastes.
+minute_processed_paste = 10
+
 ##### Redis #####
 [Redis_Cache]
 host = localhost

var/www/Flask_server.py

@@ -122,7 +122,8 @@ def search():
 
 @app.route("/")
 def index():
-    return render_template("index.html")
+    default_minute = cfg.get("Flask", "minute_processed_paste")
+    return render_template("index.html", default_minute = default_minute)
 
 
 @app.route("/monitoring/")

var/www/static/js/indexjavascript.js

@@ -1,3 +1,53 @@
+// Plot and update the number of processed pastes
+$(function() {
+    var data = [];
+    var default_minute = (typeof window.default_minute !== "undefined") ? parseInt(window.default_minute) : 10;
+    var totalPoints = 60*parseInt(default_minute); //60s*minute
+    var curr_max = 0;
+    
+    function getData() {
+        if (data.length > 0){
+             var data_old = data[0];
+             data = data.slice(1);
+             curr_max = curr_max == data_old ? Math.max.apply(null, data) : curr_max;
+        }
+        
+        while (data.length < totalPoints) {
+            var y = (typeof window.paste_num_tabvar !== "undefined") ? parseInt(window.paste_num_tabvar) : 0;
+            curr_max = y > curr_max ? y : curr_max;
+            data.push(y);
+        }
+        // Zip the generated y values with the x values
+        var res = [];
+        for (var i = 0; i < data.length; ++i) {
+            res.push([i, data[i]])
+        }
+        return res;
+    }
+
+    var updateInterval = 1000; //1s
+    var options = {
+        series: { shadowSize: 1 },
+        lines: { fill: true, fillColor: { colors: [ { opacity: 1 }, { opacity: 0.1 } ] }},
+        yaxis: { min: 0, max: 40 },
+        colors: ["#a971ff"],
+        grid: {
+            tickColor: "#dddddd",
+            borderWidth: 0 
+        },
+    };
+    var plot = $.plot("#realtimechart", [ getData() ], options);
+    
+    function update() {
+        plot.setData([getData()]);
+        plot.getOptions().yaxes[0].max = curr_max;
+        plot.setupGrid();
+        plot.draw();
+        setTimeout(update, updateInterval);
+    }
+    update();
+});
+
 function initfunc( csvay, scroot) {
   window.csv = csvay;
   window.scroot = scroot;
@@ -38,6 +88,12 @@ function create_log_table(obj_json) {
     var chansplit = obj_json.channel.split('.');
     var parsedmess = obj_json.data.split(';');
 
+    if (parsedmess[0] == "Global"){
+        var paste_processed = parsedmess[4].split(" ")[2];
+        window.paste_num_tabvar = paste_processed;
+        return;
+    }
+
     if( chansplit[1] == "INFO" ){
         tr.className = "info";
     }
@@ -270,3 +326,5 @@ $(document).ready(function () {
     }
 
 });
+
+

var/www/templates/index.html

@@ -14,10 +14,12 @@
   <link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
   <!-- JS -->
   <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
   <script src="{{ url_for('static', filename='js/jquery.tagcanvas.js') }}"></script>
   <script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
   <script>
+      window.default_minute = {{ default_minute }};
       function update_values() {
           $SCRIPT_ROOT = {{ request.script_root|tojson|safe }};
           $.getJSON($SCRIPT_ROOT+"/_stuff",
@@ -50,7 +52,16 @@
                   <!-- /#side-menu -->
               </div>
               <!-- /.sidebar-collapse -->
-              <div class="table-responsive", id="queueing" style="margin-top:40px;"></div>
+              <div class="panel panel-default">
+                  <div class="panel-heading">
+                      <i class="fa fa-dashboard  fa-fw"></i> Pastes since {{ default_minute }} min
+                  </div>
+                  <div class="panel-body">
+                       <div id="realtimechart" style="height: 90px; padding: 0px; position: relative;"></div>
+                  </div>
+              </div>
+                  <!-- <div id="Graph_paste_num" style="height:90px; width:100%;"></div> -->
+              <div class="table-responsive", id="queueing" style="margin-top:10px;"></div>
              <a  href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
           </div>
           <!-- /.navbar-static-side -->
@@ -132,7 +143,6 @@
     initfunc( "{{ url_for('static', filename='csv/wordstrendingdata.csv') }}", {{ request.script_root|tojson|safe }} );
     </script>
 
-    <script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
     <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
     <script src="{{ url_for('static', filename='js/plugins/metisMenu/metisMenu.js') }}"></script>