mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-27 00:07:16 +00:00
Merge branch 'master' of https://github.com/CIRCL/AIL-framework
This commit is contained in:
Terrtia
commit
1bf752fbfc
7 changed files with 89 additions and 68 deletions
|
|
@ -1,7 +1,7 @@
|
|||
language: python
|
||||
|
||||
python:
|
||||
- "3.5"
|
||||
- "3.6"
|
||||
|
||||
sudo: required
|
||||
|
||||
|
|
@ -12,7 +12,7 @@ env:
|
|||
AIL_FLASK=$TRAVIS_BUILD_DIR/var/www/ AIL_REDIS=$TRAVIS_BUILD_DIR/redis/src/ \
|
||||
AIL_LEVELDB=$TRAVIS_BUILD_DIR/redis-leveldb/ PATH=$AIL_HOME:$AIL_REDIS:$AIL_LEVELDB:$PATH
|
||||
|
||||
|
||||
dist: xenial
|
||||
|
||||
install:
|
||||
- ./installing_deps.sh
|
||||
|
|
@ -20,10 +20,7 @@ install:
|
|||
|
||||
script:
|
||||
- pushd bin
|
||||
- ./launch_redis.sh
|
||||
- ./launch_lvldb.sh
|
||||
- ./launch_logs.sh
|
||||
- ./launch_queues.sh
|
||||
- ./LAUNCH.sh -l
|
||||
- popd
|
||||
- cd tests
|
||||
- nosetests --with-coverage --cover-package=../bin -d
|
||||
|
|
|
|||
23
README.md
23
README.md
|
|
@ -1,18 +1,17 @@
|
|||
AIL
|
||||
===
|
||||
|
||||
|
||||
|
||||
AIL framework - Framework for Analysis of Information Leaks
|
||||
|
||||
AIL is a modular framework to analyse potential information leaks from unstructured data sources like pastes from Pastebin or similar services or unstructured data streams. AIL framework is flexible and can be extended to support other functionalities to mine or process sensitive information (e.g. data leak prevention).
|
||||
|
||||
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td>Latest Release</td>
|
||||
<td><a href="https://github.com/CIRCL/AIL-framework/releases/latest"><img src="https://img.shields.io/github/release/CIRCL/AIL-framework/all.svg"></a></td>
|
||||
</tr>
|
||||
<td>Travis</td>
|
||||
<td><a href="https://travis-ci.org/CIRCL/AIL-framework"><img src="https://img.shields.io/travis/CIRCL/AIL-framework.svg" /></a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Gitter</td>
|
||||
<td><a href="https://gitter.im/SteveClement/AIL-framework?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge"><img src="https://badges.gitter.im/SteveClement/AIL-framework.svg" /></a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Contributors</td>
|
||||
|
|
@ -24,6 +23,14 @@ AIL is a modular framework to analyse potential information leaks from unstructu
|
|||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
|
||||
AIL framework - Framework for Analysis of Information Leaks
|
||||
|
||||
AIL is a modular framework to analyse potential information leaks from unstructured data sources like pastes from Pastebin or similar services or unstructured data streams. AIL framework is flexible and can be extended to support other functionalities to mine or process sensitive information (e.g. data leak prevention).
|
||||
|
||||
|
||||
|
||||
|
||||
Features
|
||||
--------
|
||||
|
|
|
|||
|
|
@ -17,6 +17,11 @@ cd ${AIL_HOME}
|
|||
|
||||
if [ -e "${DIR}/AILENV/bin/python" ]; then
|
||||
ENV_PY="${DIR}/AILENV/bin/python"
|
||||
export AIL_VENV=${AIL_HOME}/AILENV/
|
||||
elif [ ! -z "$TRAVIS" ]; then
|
||||
echo "Travis detected"
|
||||
ENV_PY="~/virtualenv/python3.6/bin/python"
|
||||
export AIL_VENV="~/virtualenv/python3.6/"
|
||||
else
|
||||
echo "Please make sure you have a AIL-framework environment, au revoir"
|
||||
exit 1
|
||||
|
|
@ -29,7 +34,6 @@ export AIL_BIN=${AIL_HOME}/bin/
|
|||
export AIL_FLASK=${AIL_HOME}/var/www/
|
||||
export AIL_REDIS=${AIL_HOME}/redis/src/
|
||||
export AIL_ARDB=${AIL_HOME}/ardb/src/
|
||||
export AIL_VENV=${AIL_HOME}/AILENV/
|
||||
|
||||
export PATH=$AIL_VENV/bin:$PATH
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
|
|
|
|||
|
|
@ -1,40 +1,43 @@
|
|||
#!/bin/bash
|
||||
|
||||
# halt on errors
|
||||
set -e
|
||||
set -x
|
||||
|
||||
## bash debug mode togle below
|
||||
#set -x
|
||||
|
||||
sudo apt-get update
|
||||
|
||||
sudo apt-get install python3-pip virtualenv python3-dev python3-tk libfreetype6-dev \
|
||||
screen g++ python-tk unzip libsnappy-dev cmake -y
|
||||
screen g++ python-tk unzip libsnappy-dev cmake -qq
|
||||
|
||||
#optional tor install
|
||||
sudo apt-get install tor -y
|
||||
sudo apt-get install tor -qq
|
||||
|
||||
#Needed for bloom filters
|
||||
sudo apt-get install libssl-dev libfreetype6-dev python-numpy -y
|
||||
sudo apt-get install libssl-dev libfreetype6-dev python-numpy -qq
|
||||
|
||||
#pyMISP
|
||||
#sudo apt-get -y install python3-pip
|
||||
|
||||
# DNS deps
|
||||
sudo apt-get install libadns1 libadns1-dev -y
|
||||
sudo apt-get install libadns1 libadns1-dev -qq
|
||||
|
||||
#Needed for redis-lvlDB
|
||||
sudo apt-get install libev-dev libgmp-dev -y
|
||||
sudo apt-get install libev-dev libgmp-dev -qq
|
||||
|
||||
#Need for generate-data-flow graph
|
||||
sudo apt-get install graphviz -y
|
||||
sudo apt-get install graphviz -qq
|
||||
|
||||
# install nosetests
|
||||
sudo pip3 install nose
|
||||
sudo apt-get install python3-nose -qq
|
||||
|
||||
# ssdeep
|
||||
sudo apt-get install libfuzzy-dev -y
|
||||
sudo apt-get install build-essential libffi-dev automake autoconf libtool -y
|
||||
sudo apt-get install libfuzzy-dev -qq
|
||||
sudo apt-get install build-essential libffi-dev automake autoconf libtool -qq
|
||||
|
||||
# sflock, gz requirement
|
||||
sudo apt-get install p7zip-full -y
|
||||
sudo apt-get install p7zip-full -qq
|
||||
|
||||
# REDIS #
|
||||
test ! -d redis/ && git clone https://github.com/antirez/redis.git
|
||||
|
|
|
|||
0
installing_deps_archlinux.sh
Normal file → Executable file
0
installing_deps_archlinux.sh
Normal file → Executable file
|
|
@ -105,6 +105,7 @@ def all_hash_search():
|
|||
show_decoded_files = request.form.get('show_decoded_files')
|
||||
return redirect(url_for('hashDecoded.hashDecoded_page', date_from=date_from, date_to=date_to, type=type, encoding=encoding, show_decoded_files=show_decoded_files))
|
||||
|
||||
|
||||
@hashDecoded.route("/hashDecoded/", methods=['GET'])
|
||||
def hashDecoded_page():
|
||||
date_from = request.args.get('date_from')
|
||||
|
|
@ -221,17 +222,20 @@ def hashDecoded_page():
|
|||
return render_template("hashDecoded.html", l_64=b64_metadata, vt_enabled=vt_enabled, l_type=l_type, type=type, daily_type_chart=daily_type_chart, daily_date=daily_date,
|
||||
encoding=encoding, all_encoding=all_encoding, date_from=date_from, date_to=date_to, show_decoded_files=show_decoded_files)
|
||||
|
||||
|
||||
@hashDecoded.route('/hashDecoded/hash_by_type')
|
||||
def hash_by_type():
|
||||
type = request.args.get('type')
|
||||
type = 'text/plain'
|
||||
return render_template('hash_type.html',type = type)
|
||||
|
||||
|
||||
@hashDecoded.route('/hashDecoded/hash_hash')
|
||||
def hash_hash():
|
||||
hash = request.args.get('hash')
|
||||
return render_template('hash_hash.html')
|
||||
|
||||
|
||||
@hashDecoded.route('/hashDecoded/showHash')
|
||||
def showHash():
|
||||
hash = request.args.get('hash')
|
||||
|
|
@ -284,6 +288,7 @@ def showHash():
|
|||
first_seen=first_seen, list_hash_decoder=list_hash_decoder,
|
||||
last_seen=last_seen, nb_seen_in_all_pastes=nb_seen_in_all_pastes, sparkline_values=sparkline_values)
|
||||
|
||||
|
||||
@hashDecoded.route('/hashDecoded/downloadHash')
|
||||
def downloadHash():
|
||||
hash = request.args.get('hash')
|
||||
|
|
@ -319,6 +324,7 @@ def downloadHash():
|
|||
else:
|
||||
return 'hash: ' + hash + " don't exist"
|
||||
|
||||
|
||||
@hashDecoded.route('/hashDecoded/hash_by_type_json')
|
||||
def hash_by_type_json():
|
||||
type = request.args.get('type')
|
||||
|
|
@ -351,6 +357,7 @@ def hash_by_type_json():
|
|||
else:
|
||||
return jsonify()
|
||||
|
||||
|
||||
@hashDecoded.route('/hashDecoded/decoder_type_json')
|
||||
def decoder_type_json():
|
||||
date_from = request.args.get('date_from')
|
||||
|
|
@ -405,6 +412,7 @@ def decoder_type_json():
|
|||
to_json.append({'name': decoder, 'value': nb_decoded[decoder]})
|
||||
return jsonify(to_json)
|
||||
|
||||
|
||||
@hashDecoded.route('/hashDecoded/top5_type_json')
|
||||
def top5_type_json():
|
||||
date_from = request.args.get('date_from')
|
||||
|
|
@ -481,6 +489,7 @@ def daily_type_json():
|
|||
|
||||
return jsonify(type_value)
|
||||
|
||||
|
||||
@hashDecoded.route('/hashDecoded/range_type_json')
|
||||
def range_type_json():
|
||||
date_from = request.args.get('date_from')
|
||||
|
|
@ -536,14 +545,13 @@ def range_type_json():
|
|||
|
||||
return jsonify(range_type)
|
||||
|
||||
|
||||
@hashDecoded.route('/hashDecoded/hash_graph_line_json')
|
||||
def hash_graph_line_json():
|
||||
hash = request.args.get('hash')
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
|
||||
#hash = '9c748d28d78a64aef99e7ba866a433eb635c6d7a'
|
||||
|
||||
if date_from is None or date_to is None:
|
||||
nb_days_seen_in_pastes = 30
|
||||
else:
|
||||
|
|
@ -633,12 +641,14 @@ def hash_graph_node_json():
|
|||
else:
|
||||
return jsonify({})
|
||||
|
||||
|
||||
@hashDecoded.route('/hashDecoded/hash_types')
|
||||
def hash_types():
|
||||
date_from = 20180701
|
||||
date_to = 20180706
|
||||
return render_template('hash_types.html', date_from=date_from, date_to=date_to)
|
||||
|
||||
|
||||
@hashDecoded.route('/hashDecoded/send_file_to_vt_js')
|
||||
def send_file_to_vt_js():
|
||||
hash = request.args.get('hash')
|
||||
|
|
@ -690,7 +700,7 @@ def update_vt_result():
|
|||
return jsonify(hash=hash, report_vt=b64_vt_report)
|
||||
elif response.status_code == 403:
|
||||
Flask_config.vt_enabled = False
|
||||
print('VT is disabled')
|
||||
print('Virustotal key is incorrect (e.g. for public API not for virustotal intelligence), authentication failed or reaching limits.')
|
||||
return jsonify()
|
||||
else:
|
||||
# TODO FIXME make json response
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
set -e
|
||||
|
||||
wget http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js
|
||||
wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js
|
||||
|
||||
SBADMIN_VERSION='3.3.7'
|
||||
BOOTSTRAP_VERSION='4.2.1'
|
||||
|
|
@ -12,28 +12,28 @@ D3_JS_VERSION='5.5.0'
|
|||
rm -rf temp
|
||||
mkdir temp
|
||||
|
||||
wget https://github.com/twbs/bootstrap/releases/download/v${BOOTSTRAP_VERSION}/bootstrap-${BOOTSTRAP_VERSION}-dist.zip -O temp/bootstrap${BOOTSTRAP_VERSION}.zip
|
||||
wget https://github.com/FezVrasta/popper.js/archive/v1.14.3.zip -O temp/popper.zip
|
||||
wget https://github.com/BlackrockDigital/startbootstrap-sb-admin/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}.zip
|
||||
wget https://github.com/BlackrockDigital/startbootstrap-sb-admin-2/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}-2.zip
|
||||
wget https://github.com/FortAwesome/Font-Awesome/archive/v4.7.0.zip -O temp/FONT_AWESOME_4.7.0.zip
|
||||
wget https://github.com/FortAwesome/Font-Awesome/archive/5.7.1.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip
|
||||
wget https://github.com/d3/d3/releases/download/v${D3_JS_VERSION}/d3.zip -O temp/d3_${D3_JS_VERSION}.zip
|
||||
wget -q https://github.com/twbs/bootstrap/releases/download/v${BOOTSTRAP_VERSION}/bootstrap-${BOOTSTRAP_VERSION}-dist.zip -O temp/bootstrap${BOOTSTRAP_VERSION}.zip
|
||||
wget -q https://github.com/FezVrasta/popper.js/archive/v1.14.3.zip -O temp/popper.zip
|
||||
wget -q https://github.com/BlackrockDigital/startbootstrap-sb-admin/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}.zip
|
||||
wget -q https://github.com/BlackrockDigital/startbootstrap-sb-admin-2/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}-2.zip
|
||||
wget -q https://github.com/FortAwesome/Font-Awesome/archive/v4.7.0.zip -O temp/FONT_AWESOME_4.7.0.zip
|
||||
wget -q https://github.com/FortAwesome/Font-Awesome/archive/5.7.1.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip
|
||||
wget -q https://github.com/d3/d3/releases/download/v${D3_JS_VERSION}/d3.zip -O temp/d3_${D3_JS_VERSION}.zip
|
||||
|
||||
# dateRangePicker
|
||||
wget https://github.com/moment/moment/archive/2.22.2.zip -O temp/moment_2.22.2.zip
|
||||
wget https://github.com/longbill/jquery-date-range-picker/archive/v0.18.0.zip -O temp/daterangepicker_v0.18.0.zip
|
||||
wget -q https://github.com/moment/moment/archive/2.22.2.zip -O temp/moment_2.22.2.zip
|
||||
wget -q https://github.com/longbill/jquery-date-range-picker/archive/v0.18.0.zip -O temp/daterangepicker_v0.18.0.zip
|
||||
|
||||
unzip temp/bootstrap${BOOTSTRAP_VERSION}.zip -d temp/
|
||||
unzip temp/popper.zip -d temp/
|
||||
unzip temp/${SBADMIN_VERSION}.zip -d temp/
|
||||
unzip temp/${SBADMIN_VERSION}-2.zip -d temp/
|
||||
unzip temp/FONT_AWESOME_4.7.0.zip -d temp/
|
||||
unzip temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/
|
||||
unzip temp/d3_${D3_JS_VERSION}.zip -d temp/
|
||||
unzip -qq temp/bootstrap${BOOTSTRAP_VERSION}.zip -d temp/
|
||||
unzip -qq temp/popper.zip -d temp/
|
||||
unzip -qq temp/${SBADMIN_VERSION}.zip -d temp/
|
||||
unzip -qq temp/${SBADMIN_VERSION}-2.zip -d temp/
|
||||
unzip -qq temp/FONT_AWESOME_4.7.0.zip -d temp/
|
||||
unzip -qq temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/
|
||||
unzip -qq temp/d3_${D3_JS_VERSION}.zip -d temp/
|
||||
|
||||
unzip temp/moment_2.22.2.zip -d temp/
|
||||
unzip temp/daterangepicker_v0.18.0.zip -d temp/
|
||||
unzip -qq temp/moment_2.22.2.zip -d temp/
|
||||
unzip -qq temp/daterangepicker_v0.18.0.zip -d temp/
|
||||
|
||||
mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/js/bootstrap.min.js ./static/js/bootstrap4.min.js
|
||||
mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/js/bootstrap.min.js.map ./static/js/bootstrap.min.js.map
|
||||
|
|
@ -70,38 +70,38 @@ mv temp/jquery-date-range-picker-0.18.0/dist/jquery.daterangepicker.min.js ./sta
|
|||
rm -rf temp
|
||||
|
||||
JQVERSION="1.12.4"
|
||||
wget http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js
|
||||
wget -q http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js
|
||||
|
||||
#Ressources for dataTable
|
||||
wget https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min.js -O ./static/js/jquery.dataTables.min.js
|
||||
wget https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTables.bootstrap.css -O ./static/css/dataTables.bootstrap.css
|
||||
wget https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTables.bootstrap.js -O ./static/js/dataTables.bootstrap.js
|
||||
wget -q https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min.js -O ./static/js/jquery.dataTables.min.js
|
||||
wget -q https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTables.bootstrap.css -O ./static/css/dataTables.bootstrap.css
|
||||
wget -q https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTables.bootstrap.js -O ./static/js/dataTables.bootstrap.js
|
||||
|
||||
wget https://cdn.datatables.net/1.10.18/css/dataTables.bootstrap4.min.css -O ./static/css/dataTables.bootstrap.min.css
|
||||
wget https://cdn.datatables.net/1.10.18/js/dataTables.bootstrap4.min.js -O ./static/js/dataTables.bootstrap.min.js
|
||||
wget -q https://cdn.datatables.net/1.10.18/css/dataTables.bootstrap4.min.css -O ./static/css/dataTables.bootstrap.min.css
|
||||
wget -q https://cdn.datatables.net/1.10.18/js/dataTables.bootstrap4.min.js -O ./static/js/dataTables.bootstrap.min.js
|
||||
|
||||
#Ressource for graph
|
||||
wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.js -O ./static/js/jquery.flot.js
|
||||
wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.pie.js -O ./static/js/jquery.flot.pie.js
|
||||
wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.time.js -O ./static/js/jquery.flot.time.js
|
||||
wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.stack.js -O ./static/js/jquery.flot.stack.js
|
||||
wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.js -O ./static/js/jquery.flot.js
|
||||
wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.pie.js -O ./static/js/jquery.flot.pie.js
|
||||
wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.time.js -O ./static/js/jquery.flot.time.js
|
||||
wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.stack.js -O ./static/js/jquery.flot.stack.js
|
||||
|
||||
#Ressources for sparkline and canvasJS and slider
|
||||
wget http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js
|
||||
wget -q http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js
|
||||
mkdir temp
|
||||
wget http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip
|
||||
unzip temp/canvasjs.zip -d temp/
|
||||
wget -q http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip
|
||||
unzip -qq temp/canvasjs.zip -d temp/
|
||||
mv temp/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js
|
||||
|
||||
wget https://jqueryui.com/resources/download/jquery-ui-1.12.0.zip -O temp/jquery-ui.zip
|
||||
unzip temp/jquery-ui.zip -d temp/
|
||||
wget -q https://jqueryui.com/resources/download/jquery-ui-1.12.0.zip -O temp/jquery-ui.zip
|
||||
unzip -qq temp/jquery-ui.zip -d temp/
|
||||
mv temp/jquery-ui-1.12.0/jquery-ui.min.js ./static/js/jquery-ui.min.js
|
||||
mv temp/jquery-ui-1.12.0/jquery-ui.min.css ./static/css/jquery-ui.min.css
|
||||
rm -rf temp
|
||||
|
||||
mkdir -p ./static/image
|
||||
pushd static/image
|
||||
wget https://www.circl.lu/assets/images/logos/AIL.png -O AIL.png
|
||||
wget -q https://www.circl.lu/assets/images/logos/AIL.png -O AIL.png
|
||||
popd
|
||||
|
||||
if ! [[ -n "$AIL_HOME" ]]
|
||||
|
|
|
|||
Loading…
Reference in a new issue