mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 00:28:22 +00:00
Merge branch 'master' of https://github.com/CIRCL/AIL-framework
This commit is contained in:
commit
192074e569
4 changed files with 83 additions and 38 deletions
33
README.md
33
README.md
|
@ -23,11 +23,16 @@ sudo apt-get install screen
|
|||
|
||||
You need to create a variable AILENV that will be the installation path:
|
||||
|
||||
``export AILENV="/home/user/ail"``
|
||||
``export AILENV="/home/user/AIL-framework"``
|
||||
|
||||
Usually the installation path is where the project is cloned.
|
||||
|
||||
Then create a Python virtual environment:
|
||||
|
||||
``virtualenv AILENV``
|
||||
```
|
||||
cd $AILENV
|
||||
virtualenv AILENV
|
||||
```
|
||||
|
||||
And install these few more packets:
|
||||
```
|
||||
|
@ -35,6 +40,9 @@ sudo apt-get install g++
|
|||
sudo apt-get install python-dev
|
||||
sudo apt-get install python-tk
|
||||
sudo apt-get install screen
|
||||
sudo apt-get install libssl-dev
|
||||
sudo apt-get install libfreetype6-dev
|
||||
sudo apt-get install python-numpy
|
||||
```
|
||||
|
||||
Then these modules need to be install with pip inside the virtual environment:
|
||||
|
@ -54,17 +62,18 @@ That's all the packages you can install with pip:
|
|||
|
||||
```
|
||||
pip install redis
|
||||
pip install logbook
|
||||
pip install networkx
|
||||
pip install crcmod
|
||||
pip install mmh3
|
||||
pip install dnspython
|
||||
pip install pyzmq
|
||||
pip install texttable ----- Queues Monitoring (Optional)
|
||||
pip install texttable
|
||||
pip install -U textblob
|
||||
python -m textblob.download_corpora
|
||||
pip install python-magic
|
||||
pip install numpy
|
||||
pip install flask ----- (Optional)
|
||||
pip install flask
|
||||
pip install nltk
|
||||
pip install matplotlib ----- (sudo ln -s freetype2/ft2build.h in /usr/include/)
|
||||
pip install pybloomfiltermmap ----- (you may need to sudo apt-get install libssl-dev)
|
||||
|
@ -115,6 +124,22 @@ cd bin
|
|||
./LAUNCH.sh
|
||||
```
|
||||
|
||||
To start with the web interface, you need to fetch the required Javascript/CSS files:
|
||||
|
||||
```
|
||||
cd $AILENV
|
||||
cd var/www/
|
||||
bash update_thirdparty.sh
|
||||
```
|
||||
|
||||
and then you can start the web interface:
|
||||
|
||||
```
|
||||
cd $AILENV
|
||||
cd var/www/
|
||||
Flask_server.py
|
||||
```
|
||||
|
||||
Then you can browse the status of the AIL framework at the following URL:
|
||||
|
||||
``http://localhost:7000/``
|
||||
|
|
|
@ -59,44 +59,47 @@ def main():
|
|||
url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
|
||||
|
||||
while True:
|
||||
if message != None:
|
||||
channel, filename, word, score = message.split()
|
||||
try:
|
||||
if message != None:
|
||||
channel, filename, word, score = message.split()
|
||||
|
||||
if prec_filename == None or filename != prec_filename:
|
||||
domains_list = []
|
||||
PST = P.Paste(filename)
|
||||
if prec_filename == None or filename != prec_filename:
|
||||
domains_list = []
|
||||
PST = P.Paste(filename)
|
||||
|
||||
for x in PST.get_regex(url_regex):
|
||||
scheme, credential, subdomain, domain, host, tld, port, resource_path, query_string, f1, f2, f3, f4 = x
|
||||
domains_list.append(domain)
|
||||
msg = pubchannel + " " + str(x)
|
||||
Pub.send_message(msg)
|
||||
publisher.debug('{0} Published'.format(x))
|
||||
for x in PST.get_regex(url_regex):
|
||||
scheme, credential, subdomain, domain, host, tld, port, resource_path, query_string, f1, f2, f3, f4 = x
|
||||
domains_list.append(domain)
|
||||
msg = pubchannel + " " + str(x)
|
||||
Pub.send_message(msg)
|
||||
publisher.debug('{0} Published'.format(x))
|
||||
|
||||
if f1 == "onion":
|
||||
print domain
|
||||
if f1 == "onion":
|
||||
print domain
|
||||
|
||||
A_values = lib_refine.checking_A_record(r_serv2, domains_list)
|
||||
A_values = lib_refine.checking_A_record(r_serv2, domains_list)
|
||||
|
||||
if A_values[0] >= 1:
|
||||
PST.__setattr__(channel, A_values)
|
||||
PST.save_attribute_redis(r_serv1, channel, (A_values[0],list(A_values[1])))
|
||||
if A_values[0] >= 1:
|
||||
PST.__setattr__(channel, A_values)
|
||||
PST.save_attribute_redis(r_serv1, channel, (A_values[0],list(A_values[1])))
|
||||
|
||||
pprint.pprint(A_values)
|
||||
publisher.info('{0};{1};{2};{3};{4}'.format("Url", PST.p_source, PST.p_date, PST.p_name, str(A_values[0])+" Valid url detected" ))
|
||||
prec_filename = filename
|
||||
pprint.pprint(A_values)
|
||||
publisher.info('{0};{1};{2};{3};{4}'.format("Url", PST.p_source, PST.p_date, PST.p_name, str(A_values[0])+" Valid url detected" ))
|
||||
prec_filename = filename
|
||||
|
||||
else:
|
||||
if r_serv.sismember("SHUTDOWN_FLAGS", "Urls"):
|
||||
r_serv.srem("SHUTDOWN_FLAGS", "Urls")
|
||||
print "Shutdown Flag Up: Terminating"
|
||||
publisher.warning("Shutdown Flag Up: Terminating.")
|
||||
break
|
||||
publisher.debug("Script url is Idling 10s")
|
||||
time.sleep(10)
|
||||
|
||||
message = Sub.get_msg_from_queue(r_serv)
|
||||
else:
|
||||
if r_serv.sismember("SHUTDOWN_FLAGS", "Urls"):
|
||||
r_serv.srem("SHUTDOWN_FLAGS", "Urls")
|
||||
print "Shutdown Flag Up: Terminating"
|
||||
publisher.warning("Shutdown Flag Up: Terminating.")
|
||||
break
|
||||
publisher.debug("Script url is Idling 10s")
|
||||
time.sleep(10)
|
||||
|
||||
message = Sub.get_msg_from_queue(r_serv)
|
||||
except dns.exception.Timeout:
|
||||
print "dns.exception.Timeout"
|
||||
pass
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
@ -16,7 +16,7 @@ Conditions to fulfill to be able to use this class correctly:
|
|||
|
||||
"""
|
||||
|
||||
import os, magic, gzip, langid, pprint, redis, operator, string, re, json
|
||||
import os, magic, gzip, langid, pprint, redis, operator, string, re, json, ConfigParser
|
||||
from Date import Date
|
||||
from Hash import Hash
|
||||
|
||||
|
@ -46,12 +46,22 @@ class Paste(object):
|
|||
"""
|
||||
|
||||
def __init__(self, p_path):
|
||||
|
||||
configfile = './packages/config.cfg'
|
||||
cfg = ConfigParser.ConfigParser()
|
||||
cfg.read(configfile)
|
||||
|
||||
self.p_path = p_path
|
||||
|
||||
self.p_name = self.p_path.split('/')[-1]
|
||||
|
||||
self.p_size = round(os.path.getsize(self.p_path)/1024.0,2)
|
||||
|
||||
self.cache = redis.StrictRedis(
|
||||
host = cfg.get("Redis_Queues", "host"),
|
||||
port = cfg.getint("Redis_Queues", "port"),
|
||||
db = cfg.getint("Redis_Queues", "db"))
|
||||
|
||||
self.p_mime = magic.from_buffer(self.get_p_content(), mime = True)
|
||||
|
||||
self.p_encoding = None
|
||||
|
@ -82,8 +92,15 @@ class Paste(object):
|
|||
PST.get_p_content()
|
||||
|
||||
"""
|
||||
with gzip.open(self.p_path, 'rb') as F:
|
||||
return F.read()
|
||||
r_serv = self.cache
|
||||
|
||||
paste = r_serv.get(self.p_path)
|
||||
if paste is None:
|
||||
with gzip.open(self.p_path, 'rb') as F:
|
||||
paste = F.read()
|
||||
r_serv.set(self.p_path, paste)
|
||||
r_serv.expire(self.p_path, 300)
|
||||
return paste
|
||||
|
||||
def get_lines_info(self):
|
||||
"""
|
||||
|
|
|
@ -63,4 +63,4 @@ def wordstrending():
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(host='172.16.100.123' ,port=7000, threaded=True)
|
||||
app.run(host='0.0.0.0' ,port=7000, threaded=True)
|
||||
|
|
Loading…
Reference in a new issue