mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-29 09:17:16 +00:00
Added new modules and started WebTrending web interface
This commit is contained in:
Mokaddem
parent
3dc356dc5e
commit
beeeb76de9
6 changed files with 387 additions and 0 deletions
57
bin/Cve.py
Executable file
57
bin/Cve.py
Executable file
|
|
@ -0,0 +1,57 @@
|
|||
#!/usr/bin/env python2
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
Template for new modules
|
||||
"""
|
||||
|
||||
import time
|
||||
import re
|
||||
from pubsublogger import publisher
|
||||
from packages import Paste
|
||||
from Helper import Process
|
||||
|
||||
|
||||
def search_cve(message):
|
||||
filepath, count = message.split()
|
||||
paste = Paste.Paste(filepath)
|
||||
content = paste.get_p_content()
|
||||
# regex to find CVE
|
||||
reg_cve = re.compile(r'(CVE-)[1-2]\d{1,4}-\d{1,5}')
|
||||
# list of the regex results in the Paste, may be null
|
||||
results = set(reg_cve.findall(content))
|
||||
|
||||
# if the list is greater than 2, we consider the Paste may contain a list of cve
|
||||
if len(results) > 0:
|
||||
print('{} contains CVEs'.format(paste.p_name))
|
||||
publisher.warning('{} contains CVEs'.format(paste.p_name))
|
||||
|
||||
if __name__ == '__main__':
|
||||
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
|
||||
# Port of the redis instance used by pubsublogger
|
||||
publisher.port = 6380
|
||||
# Script is the default channel used for the modules.
|
||||
publisher.channel = 'Script'
|
||||
|
||||
# Section name in bin/packages/modules.cfg
|
||||
config_section = 'Cve'
|
||||
|
||||
# Setup the I/O queues
|
||||
p = Process(config_section)
|
||||
|
||||
# Sent to the logging a description of the module
|
||||
publisher.info("Run CVE module")
|
||||
|
||||
# Endless loop getting messages from the input queue
|
||||
while True:
|
||||
# Get one message from the input queue
|
||||
message = p.get_from_set()
|
||||
if message is None:
|
||||
publisher.debug("{} queue is empty, waiting".format(config_section))
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
# Do something with the message from the queue
|
||||
search_cve(message)
|
||||
|
||||
# (Optional) Send that thing to the next queue
|
||||
#p.populate_set_out(something_has_been_done)
|
||||
95
bin/WebStats.py
Executable file
95
bin/WebStats.py
Executable file
|
|
@ -0,0 +1,95 @@
|
|||
#!/usr/bin/env python2
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
Template for new modules
|
||||
"""
|
||||
|
||||
import time
|
||||
import re
|
||||
import redis
|
||||
import os
|
||||
from pubsublogger import publisher
|
||||
from packages import Paste
|
||||
from Helper import Process
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
|
||||
# Port of the redis instance used by pubsublogger
|
||||
publisher.port = 6380
|
||||
# Script is the default channel used for the modules.
|
||||
publisher.channel = 'Script'
|
||||
|
||||
# Section name in bin/packages/modules.cfg
|
||||
config_section = 'WebStats'
|
||||
|
||||
# Setup the I/O queues
|
||||
p = Process(config_section)
|
||||
|
||||
# Sent to the logging a description of the module
|
||||
publisher.info("Makes statistics about valid URL")
|
||||
|
||||
# REDIS #
|
||||
r_serv1 = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Level_DB", "host"),
|
||||
port=p.config.get("Redis_Level_DB", "port"),
|
||||
db=p.config.get("Redis_Level_DB", "db"))
|
||||
|
||||
# FILE CURVE SECTION #
|
||||
csv_path = os.path.join(os.environ['AIL_HOME'],
|
||||
p.config.get("Directories", "protocolstrending_csv"))
|
||||
protocolsfile_path = os.path.join(os.environ['AIL_HOME'],
|
||||
p.config.get("Directories", "protocolsfile"))
|
||||
|
||||
# Endless loop getting messages from the input queue
|
||||
while True:
|
||||
# Get one message from the input queue
|
||||
message = p.get_from_set()
|
||||
generate_new_graph = False
|
||||
|
||||
if message is None:
|
||||
if generate_new_graph:
|
||||
generate_new_graph = False
|
||||
print 'Building graph'
|
||||
today = datetime.date.today()
|
||||
year = today.year
|
||||
month = today.month
|
||||
lib_words.create_curve_with_word_file(r_serv1, csv_path,
|
||||
protocolsfile_path, year,
|
||||
month)
|
||||
|
||||
publisher.debug("{} queue is empty, waiting".format(config_section))
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
else:
|
||||
generate_new_graph = True
|
||||
# Do something with the message from the queue
|
||||
scheme, credential, subdomain, domain, host, tld, \
|
||||
port, resource_path, query_string, f1, f2, f3, \
|
||||
f4 , date= message.split()
|
||||
|
||||
prev_score = r_serv1.hget(scheme, date)
|
||||
if prev_score is not None:
|
||||
r_serv1.hset(scheme, date, int(prev_score) + int(score))
|
||||
else:
|
||||
r_serv1.hset(scheme, date, score)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
35
bin/empty_queue.py
Executable file
35
bin/empty_queue.py
Executable file
|
|
@ -0,0 +1,35 @@
|
|||
#!/usr/bin/env python2
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
The Duplicate module
|
||||
====================
|
||||
|
||||
This huge module is, in short term, checking duplicates.
|
||||
|
||||
Requirements:
|
||||
-------------
|
||||
|
||||
|
||||
"""
|
||||
import redis
|
||||
import os
|
||||
import time
|
||||
from packages import Paste
|
||||
from pubsublogger import publisher
|
||||
from Helper import Process
|
||||
|
||||
if __name__ == "__main__":
|
||||
publisher.port = 6380
|
||||
publisher.channel = "Script"
|
||||
|
||||
config_section = ['Global', 'Duplicates', 'Indexer', 'Attributes', 'Lines', 'DomClassifier', 'Tokenize', 'Curve', 'Categ', 'CreditCards', 'Mail', 'Onion', 'DumpValidOnion', 'Web', 'WebStats', 'Release', 'Credential', 'Cve', 'Phone', 'SourceCode', 'Keys']
|
||||
|
||||
for queue in config_section:
|
||||
print 'dropping: ' + queue
|
||||
p = Process(queue)
|
||||
while True:
|
||||
message = p.get_from_set()
|
||||
if message is None:
|
||||
break
|
||||
|
||||
1
files/Cve
Normal file
1
files/Cve
Normal file
|
|
@ -0,0 +1 @@
|
|||
CVE
|
||||
3
files/protocolsfile
Normal file
3
files/protocolsfile
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
FTP
|
||||
HTTP
|
||||
HTTPS
|
||||
196
var/www/templates/Protocolstrending.html
Normal file
196
var/www/templates/Protocolstrending.html
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>Analysis Information Leak framework Dashboard</title>
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
|
||||
<!-- JS -->
|
||||
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div id="wrapper">
|
||||
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
|
||||
<div class="navbar-header">
|
||||
<ul class="nav navbar-nav">
|
||||
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a><li><a href="{{ url_for('protocolsstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a><li></ul>
|
||||
</div>
|
||||
<!-- /.navbar-top-links -->
|
||||
<div class="navbar-default sidebar" role="navigation">
|
||||
<div class="sidebar-collapse">
|
||||
<ul class="nav" id="side-menu">
|
||||
<li class="sidebar-search">
|
||||
<div class="input-group custom-search-form">
|
||||
<input type="text" class="form-control" placeholder="Search Paste">
|
||||
<span class="input-group-btn">
|
||||
<button class="btn btn-default" type="button">
|
||||
<i class="fa fa-search"></i>
|
||||
</button>
|
||||
</span>
|
||||
</div>
|
||||
<!-- /input-group -->
|
||||
</li>
|
||||
</ul>
|
||||
<!-- /#side-menu -->
|
||||
</div>
|
||||
<!-- /.sidebar-collapse -->
|
||||
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
|
||||
</div>
|
||||
<!-- /.navbar-static-side -->
|
||||
</nav>
|
||||
<div id="page-wrapper">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<h1 class="page-header">ProtocolsTrendings</h1>
|
||||
</div>
|
||||
<!-- /.col-lg-12 -->
|
||||
</div>
|
||||
<!-- /.row -->
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<i class="fa fa-bar-chart-o fa-fw"></i> Protocols Trend
|
||||
<div class="pull-right">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown">
|
||||
Actions
|
||||
<span class="caret"></span>
|
||||
</button>
|
||||
<ul class="dropdown-menu pull-right" role="menu">
|
||||
<li><a href="#" id="linear">Linear Scale</a>
|
||||
</li>
|
||||
<li><a href="#" id="log">Log Scale</a>
|
||||
</li>
|
||||
<li><a href="#" id="unzoom" onclick="unzoomGraph()">Unzoom</a>
|
||||
</li>
|
||||
<li class="divider"></li>
|
||||
<li><a href="#" id="edit_graph">Edit graph words</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.panel-heading -->
|
||||
<div class="panel-body">
|
||||
<!-- <div id="WordTrending" style="width:100%;"></div> -->
|
||||
<div id="WordTrending" style="width:100%; height:800px;"></div>
|
||||
</div>
|
||||
<!-- /.panel-body -->
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.row -->
|
||||
</div>
|
||||
<!-- /#page-wrapper -->
|
||||
<script type="text/javascript">
|
||||
g2 = new Dygraph(
|
||||
document.getElementById("WordTrending"),
|
||||
// path to CSV file
|
||||
"{{ url_for('static', filename='csv/wordstrendingdata.csv') }}",
|
||||
//"../csv/wordstrendingdata.csv",
|
||||
//window.csv,
|
||||
{
|
||||
rollPeriod: 1,
|
||||
showRoller: true,
|
||||
//drawPoints: true,
|
||||
//fillGraph: true,
|
||||
logscale: true,
|
||||
animatedZooms: true,
|
||||
labelsKMB: true,
|
||||
highlightCircleSize: 3,
|
||||
highlightSeriesOpts: {
|
||||
strokeWidth: 3,
|
||||
strokeBorderWidth: 1,
|
||||
highlightCircleSize: 5,
|
||||
},
|
||||
underlayCallback: function(canvas, area, g) {
|
||||
canvas.fillStyle = "rgba(255, 193, 37, 0.5)";
|
||||
|
||||
function highlight_period(x_start, x_end) {
|
||||
var canvas_left_x = g.toDomXCoord(x_start);
|
||||
var canvas_right_x = g.toDomXCoord(x_end);
|
||||
var canvas_width = canvas_right_x - canvas_left_x;
|
||||
canvas.fillRect(canvas_left_x, area.y, canvas_width, area.h);
|
||||
}
|
||||
|
||||
var min_data_x = g.getValue(0,0);
|
||||
var max_data_x = g.getValue(g.numRows()-1,0);
|
||||
|
||||
// get day of week
|
||||
var d = new Date(min_data_x);
|
||||
var dow = d.getUTCDay();
|
||||
var ds = d.toUTCString();
|
||||
|
||||
var w = min_data_x;
|
||||
// starting on Sunday is a special case
|
||||
if (dow == 0) {
|
||||
highlight_period(w,w+12*3600*1000);
|
||||
}
|
||||
// find first saturday
|
||||
while (dow != 5) {
|
||||
w += 24*3600*1000;
|
||||
d = new Date(w);
|
||||
dow = d.getUTCDay();
|
||||
}
|
||||
|
||||
// shift back 1/2 day to center highlight around the point for the day
|
||||
w -= 12*3600*1000;
|
||||
while (w < max_data_x) {
|
||||
var start_x_highlight = w;
|
||||
var end_x_highlight = w + 2*24*3600*1000;
|
||||
// make sure we don't try to plot outside the graph
|
||||
if (start_x_highlight < min_data_x) {
|
||||
start_x_highlight = min_data_x;
|
||||
}
|
||||
if (end_x_highlight > max_data_x) {
|
||||
end_x_highlight = max_data_x;
|
||||
}
|
||||
highlight_period(start_x_highlight,end_x_highlight);
|
||||
// calculate start of highlight for next Saturday
|
||||
w += 7*24*3600*1000;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
onclick = function(ev) {
|
||||
if (g2.isSeriesLocked()) {
|
||||
g2.clearSelection();
|
||||
}
|
||||
else {
|
||||
g2.setSelection(g2.getSelection(), g2.getHighlightSeries(), true);
|
||||
}
|
||||
};
|
||||
g2.updateOptions({clickCallback: onclick}, true);
|
||||
|
||||
var linear = document.getElementById("linear");
|
||||
var log = document.getElementById("log");
|
||||
linear.onclick = function() { setLog(false); }
|
||||
log.onclick = function() { setLog(true); }
|
||||
var setLog = function(val) {
|
||||
g2.updateOptions({ logscale: val });
|
||||
linear.disabled = !val;
|
||||
log.disabled = val;
|
||||
}
|
||||
function unzoomGraph() {
|
||||
g2.updateOptions({
|
||||
dateWindow:null,
|
||||
valueRange:null
|
||||
});
|
||||
}
|
||||
</script>
|
||||
</div>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
Loading…
Reference in a new issue