Added Draft of ModuleStats + Paste size average

bin/Credential.py

@@ -17,6 +17,7 @@ if __name__ == "__main__":
 
     regex_web = "((?:https?:\/\/)[-_0-9a-zA-Z]+\.[0-9a-zA-Z]+)"
     regex_cred = "[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}:[a-zA-Z0-9\_\-]+"
+    regex_site_for_stats = "@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}:"
     while True:
         message = p.get_from_set()
         if message is None:
@@ -37,7 +38,12 @@ if __name__ == "__main__":
         if len(creds) == 0:
             continue
 
+        sites_for_stats = []
+        for elem in re.findall(regex_site_for_stats, content):
+            sites.append(elem[1:-1])
+
         sites = set(re.findall(regex_web, content))
+        sites_for_stats = set(sites_for_stats)
 
         message = 'Checked {} credentials found.'.format(len(creds))
         if sites:
@@ -51,7 +57,22 @@ if __name__ == "__main__":
             print("========> Found more than 10 credentials in this file : {}".format(filepath))
             publisher.warning(to_print)
             #Send to duplicate
-            p.populate_set_out(filepath)
+            p.populate_set_out(filepath, 'Duplicate')
+            
+            #Put in form, then send to moduleStats
+            creds_sites = {}
+            for cred in creds:
+                user_and_site, password = cred.split(':')
+                site = user_web.split('@')[1]
+                if site in sites: # if the parsing went fine
+                    if site in creds_sites.keys(): # check if the key already exists
+                        creds_sites[site] = creds_sites[web]+1
+                    else:
+                        creds_sites[site] = 1
+            for site, num in creds_sites.iteritems(): # Send for each different site to moduleStats
+               print 'Credential;{};{};{}'.format(num, site, paste.p_date)
+               #p.populate_set_out('Credential;{};{};{}'.format(num, site, paste.p_date), 'ModuleStats')
+
             if sites:
                 print("=======> Probably on : {}".format(', '.join(sites)))
         else:

bin/CreditCard.py

@@ -66,7 +66,7 @@ if __name__ == "__main__":
                     publisher.warning('{}Checked {} valid number(s)'.format(
                         to_print, len(creditcard_set)))
                     #Send to duplicate
-                    p.populate_set_out(filename)
+                    p.populate_set_out(filepath, 'Redis_Duplicate')
                 else:
                     publisher.info('{}CreditCard related'.format(to_print))
         else:

bin/Global.py

@@ -52,7 +52,7 @@ if __name__ == '__main__':
             else:
                 # TODO Store the name of the empty paste inside a Redis-list.
                 print "Empty Paste: not processed"
-                publisher.debug("Empty Paste: {0} not processed".format(paste))
+                publisher.debug("Empty Paste: {0} not processed".format(message))
                 continue
         else:
             print "Empty Queues: Waiting..."

bin/LAUNCH.sh

@@ -140,6 +140,8 @@ function launching_scripts {
     screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x'
     sleep 0.1
     screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
+    sleep 0.1
+    screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x'
 }
 
 #If no params, display the help

bin/Mail.py

@@ -61,9 +61,15 @@ if __name__ == "__main__":
                     if MX_values[0] > is_critical:
                         publisher.warning(to_print)
                         #Send to duplicate
-                        p.populate_set_out(filename)
+                        p.populate_set_out(filename, 'Duplicate')
+                        
                     else:
                         publisher.info(to_print)
+                #Send to ModuleStats 
+                for mail in MX_values[1]:
+                    print 'mail;{};{};{}'.format(1, mail, PST.p_date)
+                    p.populate_set_out('mail;{};{};{}'.format(1, mail, PST.p_date), 'ModuleStats')
+
             prec_filename = filename
 
         else:

bin/ModuleStats.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+"""
+    Template for new modules
+"""
+
+import time
+import datetime
+import redis
+import os
+from packages import lib_words
+from packages.Date import Date
+from pubsublogger import publisher
+from Helper import Process
+from packages import Paste
+
+# Config Var
+max_set_cardinality = 7
+num_day_to_look = 5
+
+def get_date_range(num_day):
+    curr_date = datetime.date.today()
+    date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
+    date_list = []
+
+    for i in range(0, num_day+1):
+        date_list.append(date.substract_day(i))
+    return date_list
+
+
+def compute_most_posted(server, message, num_day):
+    module, num, keyword, paste_date = message.split(';')
+
+    redis_progression_name_set = 'top_'+ module +'_set'
+
+    # Add/Update in Redis
+    prev_score = server.hget(paste_date, module+'-'+keyword)
+    if prev_score is not None:
+        server.hset(paste_date, module+'-'+keyword, int(prev_score) + int(num))
+    else:
+        server.hset(paste_date, module+'-'+keyword, int(num))
+
+    # Compute Most Posted
+    date_range = get_date_range(num_day) 
+    # check if this keyword is eligible for progression
+    keyword_total_sum = 0 
+    for date in date_range:
+        curr_value = server.hget(date, module+'-'+keyword)
+        keyword_total_sum += int(curr_value) if curr_value is not None else 0
+        
+    if (server.scard(redis_progression_name_set) < max_set_cardinality):
+        server.sadd(redis_progression_name_set, keyword)
+
+    else: #not in the set
+        #Check value for all members
+        member_set = []
+        for keyw in server.smembers(redis_progression_name_set):
+            member_set.append((keyw, int(server.hget(paste_date, module+'-'+keyw))))
+        member_set.sort(key=lambda tup: tup[1])
+        if member_set[0][1] < keyword_total_sum:
+            #remove min from set and add the new one
+            print module + ': adding ' +keyword+ '(' +str(keyword_total_sum)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
+            server.srem(redis_progression_name_set, member_set[0][0])
+            server.sadd(redis_progression_name_set, keyword)
+
+
+def compute_provider_size(server, path, num_day_to_look):
+    
+    redis_progression_name_set = 'top_size_set'
+    paste = Paste.Paste(path)
+    
+    paste_size = paste._get_p_size()
+    paste_provider = paste.p_source
+    paste_date = paste._get_p_date()
+    new_avg = paste_size
+
+    # Add/Update in Redis
+    prev_num_paste = server.hget(paste_provider+'_num', paste_date)
+    if prev_num_paste is not None:
+        server.hset(paste_provider+'_num', paste_date, int(prev_num_paste)+1)
+        prev_sum_size = server.hget(paste_provider+'_size', paste_date)
+        
+        if prev_sum_size is not None:
+            server.hset(paste_provider+'_size', paste_date, paste_size)
+            new_avg = (float(prev_sum_size)+paste_size) / (int(prev_num_paste)+1)
+        else:
+            server.hset(paste_provider+'_size', paste_date, paste_size)
+
+    else:
+        server.hset(paste_provider+'_num', paste_date, 1)
+
+    # Compute Most Posted
+    #date_range = get_date_range(num_day_to_look) 
+    # check if this keyword is eligible for progression
+    provider_total_sum = 0 
+    #for date in date_range:
+    #    curr_value = server.hget(paste_provider+'_size', date)
+    #    provider_total_sum += int(curr_value) if curr_value is not None else 0
+        
+    #if paste_provider in server.smembers(redis_progression_name_set): # if it is already in the set
+    #    return
+
+    if (server.scard(redis_progression_name_set) < max_set_cardinality):
+        server.sadd(redis_progression_name_set, paste_provider)
+
+    else: #not in the set
+        #Check value for all members
+        member_set = []
+        for provider in server.smembers(redis_progression_name_set):
+            curr_avg = 0.0
+        #    for date in date_range:
+            curr_size = server.hget(provider+'_size', paste_date)
+            curr_num = server.hget(provider+'_num', paste_date)
+            print curr_size
+            if (curr_size is not None) and (curr_num is not None):
+                curr_avg += float(curr_size) / float(curr_num)
+                print str(curr_avg)
+            member_set.append((provider, curr_avg))
+        member_set.sort(key=lambda tup: tup[1])
+        if member_set[0][1] < new_avg:
+            #remove min from set and add the new one
+            print 'Adding ' +paste_provider+ '(' +str(new_avg)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
+            server.srem(redis_progression_name_set, member_set[0][0])
+            server.srem(redis_progression_name_set, member_set[0][0])
+            server.sadd(redis_progression_name_set, paste_provider)
+
+
+
+if __name__ == '__main__':
+    # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
+    # Port of the redis instance used by pubsublogger
+    publisher.port = 6380
+    # Script is the default channel used for the modules.
+    publisher.channel = 'Script'
+
+    # Section name in bin/packages/modules.cfg
+    config_section = 'ModuleStats'
+
+    # Setup the I/O queues
+    p = Process(config_section)
+
+    # Sent to the logging a description of the module
+    publisher.info("Makes statistics about valid URL")
+
+    # REDIS #
+    r_serv_trend = redis.StrictRedis(
+        host=p.config.get("Redis_Level_DB_Trending", "host"),
+        port=p.config.get("Redis_Level_DB_Trending", "port"),
+        db=p.config.get("Redis_Level_DB_Trending", "db"))
+
+    # Endless loop getting messages from the input queue
+    while True:
+        # Get one message from the input queue
+        message = p.get_from_set()
+
+        if message is None:
+            publisher.debug("{} queue is empty, waiting".format(config_section))
+            print 'sleeping'
+            time.sleep(2)
+            continue
+
+        else:
+            # Do something with the message from the queue
+            print message.split(';')
+            if len(message.split(';')) > 1:
+                compute_most_posted(r_serv_trend, message, num_day_to_look)
+            else:
+                compute_provider_size(r_serv_trend, message, num_day_to_look)

bin/packages/modules.cfg

@@ -1,6 +1,6 @@
 [Global]
 subscribe = ZMQ_Global
-publish = Redis_Global
+publish = Redis_Global,Redis_ModuleStats
 
 [Duplicates]
 subscribe = Redis_Duplicate
@@ -30,12 +30,12 @@ subscribe = Redis_Global
 publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode,Redis_Cve
 
 [CreditCards]
-subscribe = Redis_CreditCards
+subscribe = Redis_CreditCard
-publish = Redis_Duplicate 
+publish = Redis_Duplicate,Redis_ModuleStats
 
 [Mail]
 subscribe = Redis_Mail
-publish = Redis_Duplicate 
+publish = Redis_Duplicate,Redis_ModuleStats  
 
 [Onion]
 subscribe = Redis_Onion
@@ -52,12 +52,15 @@ publish = Redis_Url,ZMQ_Url
 [WebStats]
 subscribe = Redis_Url
 
+[ModuleStats]
+subscribe = Redis_ModuleStats
+
 [Release]
 subscribe = Redis_Global
 
 [Credential]
 subscribe = Redis_Credential
-publish = Redis_Duplicate 
+publish = Redis_Duplicate,Redis_ModuleStats
 
 [Cve]
 subscribe = Redis_Cve

var/www/Flask_server.py

@@ -163,6 +163,72 @@ def progressionCharts():
             member_set.append(("No relevant data", int(100)))
         return jsonify(member_set)
     
+@app.route("/_moduleCharts", methods=['GET'])
+def modulesCharts():
+    #To be used later
+    keyword_name = request.args.get('keywordName')
+    module_name = request.args.get('moduleName')
+    bar_requested = True if request.args.get('bar') == "true" else False
+    
+    if (bar_requested):
+        num_day = int(request.args.get('days'))
+        bar_values = []
+
+	date_range = get_date_range(num_day) 
+        # Retreive all data from the last num_day
+        for date in date_range:
+            curr_value = r_serv_charts.hget(date, module_name+'-'+keyword_name)
+            bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
+        return jsonify(bar_values)
+ 
+    else:
+        redis_progression_name_set = 'top_'+ module_name +'_set'
+
+        member_set = []
+        for keyw in r_serv_charts.smembers(redis_progression_name_set):
+            redis_progression_name = module_name+'-'+keyw
+            keyw_value = r_serv_charts.hget(get_date_range(0)[0] ,redis_progression_name)
+            keyw_value = keyw_value if keyw_value is not None else 0
+            member_set.append((keyw, int(keyw_value)))
+        member_set.sort(key=lambda tup: tup[1], reverse=True)
+        if len(member_set) == 0:
+            member_set.append(("No relevant data", int(100)))
+        return jsonify(member_set)
+
+
+@app.route("/_sizeCharts", methods=['GET'])
+def sizeCharts():
+    #To be used later
+    keyword_name = request.args.get('keywordName')
+    module_name = request.args.get('moduleName')
+    bar_requested = True if request.args.get('bar') == "true" else False
+    
+    if (bar_requested):
+        num_day = int(request.args.get('days'))
+        bar_values = []
+
+	date_range = get_date_range(num_day) 
+        # Retreive all data from the last num_day
+        for date in date_range:
+            curr_value = r_serv_charts.hget(keyword_name+'_size', date)
+            bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], float(curr_value if curr_value is not None else 0)])
+        return jsonify(bar_values)
+ 
+    else:
+        redis_progression_name_set = 'top_size_set'
+
+        member_set = []
+        for keyw in r_serv_charts.smembers(redis_progression_name_set):
+            redis_progression_name = keyw+'_size'
+            keyw_value = r_serv_charts.hget(redis_progression_name, get_date_range(0)[0])
+            keyw_value = keyw_value if keyw_value is not None else 0
+            member_set.append((keyw, float(keyw_value)))
+        member_set.sort(key=lambda tup: tup[1], reverse=True)
+        if len(member_set) == 0:
+            member_set.append(("No relevant data", float(100)))
+        return jsonify(member_set)
+
+
 
 @app.route("/search", methods=['POST'])
 def search():
@@ -227,6 +293,11 @@ def trending():
     return render_template("Trending.html", default_display = default_display)
 
 
+@app.route("/moduletrending/")
+def moduletrending():
+    return render_template("Moduletrending.html")
+
+
 @app.route("/showsavedpaste/") #completely shows the paste in a new tab
 def showsavedpaste():
     return showpaste(0)

var/www/templates/Moduletrending.html

@@ -0,0 +1,145 @@
+	<!DOCTYPE html>
+	<html>
+
+	<head>
+	  <meta charset="utf-8">
+	  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+	  <title>Analysis Information Leak framework Dashboard</title>
+
+	  <!-- Core CSS -->
+	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+	  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
+	  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
+	  <link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
+	  <!-- JS -->
+	  <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
+	  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+          <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
+          <script src="{{ url_for('static', filename='js/jquery.flot.pie.js') }}"></script>
+          <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
+
+	</head>
+	<body>
+
+	    <div id="wrapper">
+	      <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
+		  <div class="navbar-header">
+		  <ul class="nav navbar-nav">
+		      <li><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li>
+                      <li class="active"><a href="{{ url_for('trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>
+                      <li class="active"><a href="{{ url_for('moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>
+                  </ul>
+          </div>
+          <!-- /.navbar-top-links -->
+          <div class="navbar-default sidebar" role="navigation">
+              <div class="sidebar-collapse">
+                  <ul class="nav" id="side-menu">
+                      <li class="sidebar-search">
+                          {% include 'searchbox.html' %}
+                      </li>
+                 </ul>
+                  <!-- /#side-menu -->
+              </div>
+              <!-- /.sidebar-collapse -->
+           <a  href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
+          </div>
+          <!-- /.navbar-static-side -->
+      </nav>
+        <div id="page-wrapper">
+           </br>
+           {% include 'trending_graphs/Moduletrending.html' %}
+        </div>
+        <!-- /#page-wrapper -->
+
+        <script>
+            $(document).ready(function(){
+                /* Already defined variable (Before the input)
+                * 
+                * var chart_1_num_day = 5;
+                * var chart_2_num_day = 15;
+                *
+                */
+                
+                plot_top_graph("credential");
+                plot_top_graph("mail");
+                plot_top_graph("size");
+            });
+        </script>
+        
+        <script>
+            var chart_1_num_day = 5;
+            var chart_2_num_day = 15;
+            $SCRIPT_ROOT = {{ request.script_root|tojson|safe }};
+
+            function plot_top_graph(module_name){
+                /**** Flot Pie Chart ****/
+                var options = {
+                                  series: {  pie: {  show: true  } },
+                                  grid: { hoverable: true, clickable: true },
+                                  legend: { show: false  }
+                              };
+                
+                
+                moduleCharts = "size" == module_name ? "sizeCharts" : "moduleCharts";
+                // Graph1
+                $.getJSON($SCRIPT_ROOT+"/_"+moduleCharts+"?moduleName="+module_name+"&num_day="+chart_1_num_day,
+                    function(data) {
+                              temp_data_pie = [];
+                              for(i=0; i<data.length; i++){
+                                  temp_data_pie.push({label: data[i][0], data: data[i][1]});
+                              }
+                              $.plot($("#flot-pie-chart-"+module_name), temp_data_pie, options);
+            
+                              setTimeout(function() { 
+                                 $("#flot-pie-chart-"+module_name).bind("plotclick", function (event, pos, item) {
+                                     if (item == null)
+                                         return; 
+                                     var clicked_label = item.series.label;
+                                     update_bar_chart("#flot-bar-chart-"+module_name, clicked_label, item.series.color, chart_1_num_day, "%m/%d");
+                                 });
+                              }, 500);
+                });
+                
+                    
+                // flot bar char
+                function update_bar_chart(chartID, involved_item, serie_color, num_day, timeformat){
+                    var barOptions = {
+                        series: {
+                            bars: { show: true, barWidth: 82800000 }
+                        },
+                        xaxis: {
+                            mode: "time",
+                            timeformat: timeformat,
+                            tickSize: [1, 'day'],
+                            minTickSize: [1, "day"]
+                        },
+                        grid: { hoverable: true },
+                        legend: { show: true },
+                        tooltip: true,
+                        tooltipOpts: { content: "x: %x, y: %y" }
+                    };
+                
+                    $.getJSON($SCRIPT_ROOT+"/_"+moduleCharts+"?keywordName="+involved_item+"&moduleName="+module_name+"&bar=true"+"&days="+num_day,
+                        function(data) {
+                            var temp_data_bar = []
+                            for(i=0; i<data.length; i++){
+                                var curr_date = data[i][0].split('/');
+                                temp_data_bar.push([new Date(curr_date[0], curr_date[1]-1, curr_date[2]), data[i][1]]);
+                            }
+                            var barData = {
+                                label: involved_item,
+                                data: temp_data_bar,
+                                color: serie_color
+                            };
+                            $.plot($(chartID), [barData], barOptions);
+                        });
+                };
+            };
+        </script>
+
+    </div>
+    <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
+</body>
+
+</html>

var/www/templates/Trending.html

@@ -32,6 +32,7 @@
 		  <ul class="nav navbar-nav">
 		      <li><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li>
                       <li class="active"><a href="{{ url_for('trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>
+                      <li class="active"><a href="{{ url_for('moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>
                   </ul>
           </div>
           <!-- /.navbar-top-links -->

var/www/templates/trending_graphs/Moduletrending.html

@@ -0,0 +1,71 @@
+<div class="row">
+    <div class="col-lg-12">
+        <div class="row">
+            <div class="col-lg-12">
+                <div class="panel panel-default">
+                    <div class="panel-heading">
+                        <i id="flash-tld" class="fa fa-unlock" flash-tld=""></i> <strong>  Credential</strong> - Most posted domain
+                    </div>
+                    <div class="panel-body">
+                        <div class="">
+                            <div class="flot-chart-content col-lg-6" id="flot-pie-chart-credential" style="height:250px; width:48%;"></div>
+                            <div class="flot-chart-content col-lg-6" id="flot-bar-chart-credential" style="height:250px; width:48%;"></div>
+                        </div>
+                        <!-- /.row -->
+                    </div>
+                    <!-- /.panel-body -->
+                </div>
+            </div>
+            <!-- /.panel -->
+        </div>
+        <!-- /.panel -->
+    </div>
+</div>
+
+<div class="row">
+    <div class="col-lg-12">
+        <div class="row">
+            <div class="col-lg-12">
+                <div class="panel panel-default">
+                    <div class="panel-heading">
+                        <i id="flash-tld" class="fa fa-envelope" flash-tld=""></i><strong>  Mail</strong> - Most posted domain (max 1 per paste)
+                    </div>
+                    <div class="panel-body">
+                        <div class="">
+                            <div class="flot-chart-content col-lg-6" id="flot-pie-chart-mail" style="height:250px; width:48%;"></div>
+                            <div class="flot-chart-content col-lg-6" id="flot-bar-chart-mail" style="height:250px; width:48%;"></div>
+                        </div>
+                        <!-- /.row -->
+                    </div>
+                    <!-- /.panel-body -->
+                </div>
+            </div>
+            <!-- /.panel -->
+        </div>
+        <!-- /.panel -->
+    </div>
+</div>
+
+<div class="row">
+    <div class="col-lg-12">
+        <div class="row">
+            <div class="col-lg-12">
+                <div class="panel panel-default">
+                    <div class="panel-heading">
+                        <i id="flash-size" class="fa fa-database" flash-tld=""></i><strong>  Paste size</strong> - by paste provider
+                    </div>
+                    <div class="panel-body">
+                        <div class="">
+                            <div class="flot-chart-content col-lg-6" id="flot-pie-chart-size" style="height:250px; width:48%;"></div>
+                            <div class="flot-chart-content col-lg-6" id="flot-bar-chart-size" style="height:250px; width:48%;"></div>
+                        </div>
+                        <!-- /.row -->
+                    </div>
+                    <!-- /.panel-body -->
+                </div>
+            </div>
+            <!-- /.panel -->
+        </div>
+        <!-- /.panel -->
+    </div>
+</div>