Added new modules and started WebTrending web interface

This commit is contained in:
Mokaddem 2016-06-30 14:38:28 +02:00
parent 3dc356dc5e
commit beeeb76de9
6 changed files with 387 additions and 0 deletions

57
bin/Cve.py Executable file
View file

@ -0,0 +1,57 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
"""
Template for new modules
"""
import time
import re
from pubsublogger import publisher
from packages import Paste
from Helper import Process
def search_cve(message):
filepath, count = message.split()
paste = Paste.Paste(filepath)
content = paste.get_p_content()
# regex to find CVE
reg_cve = re.compile(r'(CVE-)[1-2]\d{1,4}-\d{1,5}')
# list of the regex results in the Paste, may be null
results = set(reg_cve.findall(content))
# if the list is greater than 2, we consider the Paste may contain a list of cve
if len(results) > 0:
print('{} contains CVEs'.format(paste.p_name))
publisher.warning('{} contains CVEs'.format(paste.p_name))
if __name__ == '__main__':
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
# Port of the redis instance used by pubsublogger
publisher.port = 6380
# Script is the default channel used for the modules.
publisher.channel = 'Script'
# Section name in bin/packages/modules.cfg
config_section = 'Cve'
# Setup the I/O queues
p = Process(config_section)
# Sent to the logging a description of the module
publisher.info("Run CVE module")
# Endless loop getting messages from the input queue
while True:
# Get one message from the input queue
message = p.get_from_set()
if message is None:
publisher.debug("{} queue is empty, waiting".format(config_section))
time.sleep(1)
continue
# Do something with the message from the queue
search_cve(message)
# (Optional) Send that thing to the next queue
#p.populate_set_out(something_has_been_done)

95
bin/WebStats.py Executable file
View file

@ -0,0 +1,95 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
"""
Template for new modules
"""
import time
import re
import redis
import os
from pubsublogger import publisher
from packages import Paste
from Helper import Process
if __name__ == '__main__':
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
# Port of the redis instance used by pubsublogger
publisher.port = 6380
# Script is the default channel used for the modules.
publisher.channel = 'Script'
# Section name in bin/packages/modules.cfg
config_section = 'WebStats'
# Setup the I/O queues
p = Process(config_section)
# Sent to the logging a description of the module
publisher.info("Makes statistics about valid URL")
# REDIS #
r_serv1 = redis.StrictRedis(
host=p.config.get("Redis_Level_DB", "host"),
port=p.config.get("Redis_Level_DB", "port"),
db=p.config.get("Redis_Level_DB", "db"))
# FILE CURVE SECTION #
csv_path = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "protocolstrending_csv"))
protocolsfile_path = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "protocolsfile"))
# Endless loop getting messages from the input queue
while True:
# Get one message from the input queue
message = p.get_from_set()
generate_new_graph = False
if message is None:
if generate_new_graph:
generate_new_graph = False
print 'Building graph'
today = datetime.date.today()
year = today.year
month = today.month
lib_words.create_curve_with_word_file(r_serv1, csv_path,
protocolsfile_path, year,
month)
publisher.debug("{} queue is empty, waiting".format(config_section))
time.sleep(1)
continue
else:
generate_new_graph = True
# Do something with the message from the queue
scheme, credential, subdomain, domain, host, tld, \
port, resource_path, query_string, f1, f2, f3, \
f4 , date= message.split()
prev_score = r_serv1.hget(scheme, date)
if prev_score is not None:
r_serv1.hset(scheme, date, int(prev_score) + int(score))
else:
r_serv1.hset(scheme, date, score)

35
bin/empty_queue.py Executable file
View file

@ -0,0 +1,35 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
"""
The Duplicate module
====================
This huge module is, in short term, checking duplicates.
Requirements:
-------------
"""
import redis
import os
import time
from packages import Paste
from pubsublogger import publisher
from Helper import Process
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
config_section = ['Global', 'Duplicates', 'Indexer', 'Attributes', 'Lines', 'DomClassifier', 'Tokenize', 'Curve', 'Categ', 'CreditCards', 'Mail', 'Onion', 'DumpValidOnion', 'Web', 'WebStats', 'Release', 'Credential', 'Cve', 'Phone', 'SourceCode', 'Keys']
for queue in config_section:
print 'dropping: ' + queue
p = Process(queue)
while True:
message = p.get_from_set()
if message is None:
break

1
files/Cve Normal file
View file

@ -0,0 +1 @@
CVE

3
files/protocolsfile Normal file
View file

@ -0,0 +1,3 @@
FTP
HTTP
HTTPS

View file

@ -0,0 +1,196 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
<div class="navbar-header">
<ul class="nav navbar-nav">
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a><li><a href="{{ url_for('protocolsstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a><li></ul>
</div>
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
<div class="input-group custom-search-form">
<input type="text" class="form-control" placeholder="Search Paste">
<span class="input-group-btn">
<button class="btn btn-default" type="button">
<i class="fa fa-search"></i>
</button>
</span>
</div>
<!-- /input-group -->
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header">ProtocolsTrendings</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="row">
<div class="col-lg-12">
<div class="panel panel-default">
<div class="panel-heading">
<i class="fa fa-bar-chart-o fa-fw"></i> Protocols Trend
<div class="pull-right">
<div class="btn-group">
<button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown">
Actions
<span class="caret"></span>
</button>
<ul class="dropdown-menu pull-right" role="menu">
<li><a href="#" id="linear">Linear Scale</a>
</li>
<li><a href="#" id="log">Log Scale</a>
</li>
<li><a href="#" id="unzoom" onclick="unzoomGraph()">Unzoom</a>
</li>
<li class="divider"></li>
<li><a href="#" id="edit_graph">Edit graph words</a>
</li>
</ul>
</div>
</div>
</div>
<!-- /.panel-heading -->
<div class="panel-body">
<!-- <div id="WordTrending" style="width:100%;"></div> -->
<div id="WordTrending" style="width:100%; height:800px;"></div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
<script type="text/javascript">
g2 = new Dygraph(
document.getElementById("WordTrending"),
// path to CSV file
"{{ url_for('static', filename='csv/wordstrendingdata.csv') }}",
//"../csv/wordstrendingdata.csv",
//window.csv,
{
rollPeriod: 1,
showRoller: true,
//drawPoints: true,
//fillGraph: true,
logscale: true,
animatedZooms: true,
labelsKMB: true,
highlightCircleSize: 3,
highlightSeriesOpts: {
strokeWidth: 3,
strokeBorderWidth: 1,
highlightCircleSize: 5,
},
underlayCallback: function(canvas, area, g) {
canvas.fillStyle = "rgba(255, 193, 37, 0.5)";
function highlight_period(x_start, x_end) {
var canvas_left_x = g.toDomXCoord(x_start);
var canvas_right_x = g.toDomXCoord(x_end);
var canvas_width = canvas_right_x - canvas_left_x;
canvas.fillRect(canvas_left_x, area.y, canvas_width, area.h);
}
var min_data_x = g.getValue(0,0);
var max_data_x = g.getValue(g.numRows()-1,0);
// get day of week
var d = new Date(min_data_x);
var dow = d.getUTCDay();
var ds = d.toUTCString();
var w = min_data_x;
// starting on Sunday is a special case
if (dow == 0) {
highlight_period(w,w+12*3600*1000);
}
// find first saturday
while (dow != 5) {
w += 24*3600*1000;
d = new Date(w);
dow = d.getUTCDay();
}
// shift back 1/2 day to center highlight around the point for the day
w -= 12*3600*1000;
while (w < max_data_x) {
var start_x_highlight = w;
var end_x_highlight = w + 2*24*3600*1000;
// make sure we don't try to plot outside the graph
if (start_x_highlight < min_data_x) {
start_x_highlight = min_data_x;
}
if (end_x_highlight > max_data_x) {
end_x_highlight = max_data_x;
}
highlight_period(start_x_highlight,end_x_highlight);
// calculate start of highlight for next Saturday
w += 7*24*3600*1000;
}
}
});
onclick = function(ev) {
if (g2.isSeriesLocked()) {
g2.clearSelection();
}
else {
g2.setSelection(g2.getSelection(), g2.getHighlightSeries(), true);
}
};
g2.updateOptions({clickCallback: onclick}, true);
var linear = document.getElementById("linear");
var log = document.getElementById("log");
linear.onclick = function() { setLog(false); }
log.onclick = function() { setLog(true); }
var setLog = function(val) {
g2.updateOptions({ logscale: val });
linear.disabled = !val;
log.disabled = val;
}
function unzoomGraph() {
g2.updateOptions({
dateWindow:null,
valueRange:null
});
}
</script>
</div>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
</body>
</html>