diff --git a/.travis.yml b/.travis.yml index 41117210..0cae47c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: - - "3.5" + - "3.6" sudo: required @@ -12,7 +12,7 @@ env: AIL_FLASK=$TRAVIS_BUILD_DIR/var/www/ AIL_REDIS=$TRAVIS_BUILD_DIR/redis/src/ \ AIL_LEVELDB=$TRAVIS_BUILD_DIR/redis-leveldb/ PATH=$AIL_HOME:$AIL_REDIS:$AIL_LEVELDB:$PATH - +dist: xenial install: - ./installing_deps.sh @@ -20,10 +20,7 @@ install: script: - pushd bin - - ./launch_redis.sh - - ./launch_lvldb.sh - - ./launch_logs.sh - - ./launch_queues.sh + - ./LAUNCH.sh -l - popd - cd tests - nosetests --with-coverage --cover-package=../bin -d diff --git a/README.md b/README.md index 9ca65d2a..1b6eab35 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,17 @@ AIL === -![Logo](./doc/logo/logo-small.png?raw=true "AIL logo") - -AIL framework - Framework for Analysis of Information Leaks - -AIL is a modular framework to analyse potential information leaks from unstructured data sources like pastes from Pastebin or similar services or unstructured data streams. AIL framework is flexible and can be extended to support other functionalities to mine or process sensitive information (e.g. data leak prevention). - -![Dashboard](./doc/screenshots/dashboard.png?raw=true "AIL framework dashboard") - + + + + + + + @@ -24,6 +23,14 @@ AIL is a modular framework to analyse potential information leaks from unstructu
Latest Release
Travis
Gitter
Contributors
+![Logo](./doc/logo/logo-small.png?raw=true "AIL logo") + +AIL framework - Framework for Analysis of Information Leaks + +AIL is a modular framework to analyse potential information leaks from unstructured data sources like pastes from Pastebin or similar services or unstructured data streams. AIL framework is flexible and can be extended to support other functionalities to mine or process sensitive information (e.g. data leak prevention). + +![Dashboard](./doc/screenshots/dashboard.png?raw=true "AIL framework dashboard") + Features -------- diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 86e64dfb..6d7c6bf7 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -17,6 +17,11 @@ cd ${AIL_HOME} if [ -e "${DIR}/AILENV/bin/python" ]; then ENV_PY="${DIR}/AILENV/bin/python" + export AIL_VENV=${AIL_HOME}/AILENV/ +elif [ ! -z "$TRAVIS" ]; then + echo "Travis detected" + ENV_PY="~/virtualenv/python3.6/bin/python" + export AIL_VENV="~/virtualenv/python3.6/" else echo "Please make sure you have a AIL-framework environment, au revoir" exit 1 @@ -29,7 +34,6 @@ export AIL_BIN=${AIL_HOME}/bin/ export AIL_FLASK=${AIL_HOME}/var/www/ export AIL_REDIS=${AIL_HOME}/redis/src/ export AIL_ARDB=${AIL_HOME}/ardb/src/ -export AIL_VENV=${AIL_HOME}/AILENV/ export PATH=$AIL_VENV/bin:$PATH export PATH=$AIL_HOME:$PATH diff --git a/installing_deps.sh b/installing_deps.sh index fdadd9e7..f8f2846f 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -1,40 +1,43 @@ #!/bin/bash +# halt on errors set -e -set -x + +## bash debug mode togle below +#set -x sudo apt-get update sudo apt-get install python3-pip virtualenv python3-dev python3-tk libfreetype6-dev \ - screen g++ python-tk unzip libsnappy-dev cmake -y + screen g++ python-tk unzip libsnappy-dev cmake -qq #optional tor install -sudo apt-get install tor -y +sudo apt-get install tor -qq #Needed for bloom filters -sudo apt-get install libssl-dev libfreetype6-dev python-numpy -y +sudo apt-get install libssl-dev libfreetype6-dev python-numpy -qq #pyMISP #sudo apt-get -y install python3-pip # DNS deps -sudo apt-get install libadns1 libadns1-dev -y +sudo apt-get install libadns1 libadns1-dev -qq #Needed for redis-lvlDB -sudo apt-get install libev-dev libgmp-dev -y +sudo apt-get install libev-dev libgmp-dev -qq #Need for generate-data-flow graph -sudo apt-get install graphviz -y +sudo apt-get install graphviz -qq # install nosetests -sudo pip3 install nose +sudo apt-get install python3-nose -qq # ssdeep -sudo apt-get install libfuzzy-dev -y -sudo apt-get install build-essential libffi-dev automake autoconf libtool -y +sudo apt-get install libfuzzy-dev -qq +sudo apt-get install build-essential libffi-dev automake autoconf libtool -qq # sflock, gz requirement -sudo apt-get install p7zip-full -y +sudo apt-get install p7zip-full -qq # REDIS # test ! -d redis/ && git clone https://github.com/antirez/redis.git diff --git a/installing_deps_archlinux.sh b/installing_deps_archlinux.sh old mode 100644 new mode 100755 diff --git a/var/www/modules/hashDecoded/Flask_hashDecoded.py b/var/www/modules/hashDecoded/Flask_hashDecoded.py index 705fc328..8a7945d2 100644 --- a/var/www/modules/hashDecoded/Flask_hashDecoded.py +++ b/var/www/modules/hashDecoded/Flask_hashDecoded.py @@ -105,6 +105,7 @@ def all_hash_search(): show_decoded_files = request.form.get('show_decoded_files') return redirect(url_for('hashDecoded.hashDecoded_page', date_from=date_from, date_to=date_to, type=type, encoding=encoding, show_decoded_files=show_decoded_files)) + @hashDecoded.route("/hashDecoded/", methods=['GET']) def hashDecoded_page(): date_from = request.args.get('date_from') @@ -221,17 +222,20 @@ def hashDecoded_page(): return render_template("hashDecoded.html", l_64=b64_metadata, vt_enabled=vt_enabled, l_type=l_type, type=type, daily_type_chart=daily_type_chart, daily_date=daily_date, encoding=encoding, all_encoding=all_encoding, date_from=date_from, date_to=date_to, show_decoded_files=show_decoded_files) + @hashDecoded.route('/hashDecoded/hash_by_type') def hash_by_type(): type = request.args.get('type') type = 'text/plain' return render_template('hash_type.html',type = type) + @hashDecoded.route('/hashDecoded/hash_hash') def hash_hash(): hash = request.args.get('hash') return render_template('hash_hash.html') + @hashDecoded.route('/hashDecoded/showHash') def showHash(): hash = request.args.get('hash') @@ -284,6 +288,7 @@ def showHash(): first_seen=first_seen, list_hash_decoder=list_hash_decoder, last_seen=last_seen, nb_seen_in_all_pastes=nb_seen_in_all_pastes, sparkline_values=sparkline_values) + @hashDecoded.route('/hashDecoded/downloadHash') def downloadHash(): hash = request.args.get('hash') @@ -319,6 +324,7 @@ def downloadHash(): else: return 'hash: ' + hash + " don't exist" + @hashDecoded.route('/hashDecoded/hash_by_type_json') def hash_by_type_json(): type = request.args.get('type') @@ -351,6 +357,7 @@ def hash_by_type_json(): else: return jsonify() + @hashDecoded.route('/hashDecoded/decoder_type_json') def decoder_type_json(): date_from = request.args.get('date_from') @@ -405,6 +412,7 @@ def decoder_type_json(): to_json.append({'name': decoder, 'value': nb_decoded[decoder]}) return jsonify(to_json) + @hashDecoded.route('/hashDecoded/top5_type_json') def top5_type_json(): date_from = request.args.get('date_from') @@ -481,6 +489,7 @@ def daily_type_json(): return jsonify(type_value) + @hashDecoded.route('/hashDecoded/range_type_json') def range_type_json(): date_from = request.args.get('date_from') @@ -536,14 +545,13 @@ def range_type_json(): return jsonify(range_type) + @hashDecoded.route('/hashDecoded/hash_graph_line_json') def hash_graph_line_json(): hash = request.args.get('hash') date_from = request.args.get('date_from') date_to = request.args.get('date_to') - #hash = '9c748d28d78a64aef99e7ba866a433eb635c6d7a' - if date_from is None or date_to is None: nb_days_seen_in_pastes = 30 else: @@ -552,7 +560,7 @@ def hash_graph_line_json(): date_range_seen_in_pastes = get_date_range(nb_days_seen_in_pastes) - #verify input + # verify input if r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type') is not None: json_seen_in_paste = [] for date in date_range_seen_in_pastes: @@ -560,7 +568,7 @@ def hash_graph_line_json(): if nb_seen_this_day is None: nb_seen_this_day = 0 date = date[0:4] + '-' + date[4:6] + '-' + date[6:8] - json_seen_in_paste.append({ 'date' : date, 'value' : int( nb_seen_this_day )}) + json_seen_in_paste.append({'date': date, 'value': int(nb_seen_this_day)}) return jsonify(json_seen_in_paste) else: @@ -633,12 +641,14 @@ def hash_graph_node_json(): else: return jsonify({}) + @hashDecoded.route('/hashDecoded/hash_types') def hash_types(): date_from = 20180701 date_to = 20180706 return render_template('hash_types.html', date_from=date_from, date_to=date_to) + @hashDecoded.route('/hashDecoded/send_file_to_vt_js') def send_file_to_vt_js(): hash = request.args.get('hash') @@ -667,7 +677,7 @@ def update_vt_result(): hash = request.args.get('hash') params = {'apikey': vt_auth, 'resource': hash} - response = requests.get('https://www.virustotal.com/vtapi/v2/file/report',params=params) + response = requests.get('https://www.virustotal.com/vtapi/v2/file/report', params=params) if response.status_code == 200: json_response = response.json() response_code = json_response['response_code'] @@ -676,7 +686,7 @@ def update_vt_result(): total = json_response['total'] positive = json_response['positives'] - b64_vt_report = 'Detection {}/{}'.format(positive,total) + b64_vt_report = 'Detection {}/{}'.format(positive, total) # no report found elif response_code == 0: b64_vt_report = 'No report found' @@ -690,7 +700,7 @@ def update_vt_result(): return jsonify(hash=hash, report_vt=b64_vt_report) elif response.status_code == 403: Flask_config.vt_enabled = False - print('VT is disabled') + print('Virustotal key is incorrect (e.g. for public API not for virustotal intelligence), authentication failed or reaching limits.') return jsonify() else: # TODO FIXME make json response diff --git a/var/www/update_thirdparty.sh b/var/www/update_thirdparty.sh index c06302dd..50adf2a0 100755 --- a/var/www/update_thirdparty.sh +++ b/var/www/update_thirdparty.sh @@ -2,7 +2,7 @@ set -e -wget http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js +wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js SBADMIN_VERSION='3.3.7' BOOTSTRAP_VERSION='4.2.1' @@ -12,28 +12,28 @@ D3_JS_VERSION='5.5.0' rm -rf temp mkdir temp -wget https://github.com/twbs/bootstrap/releases/download/v${BOOTSTRAP_VERSION}/bootstrap-${BOOTSTRAP_VERSION}-dist.zip -O temp/bootstrap${BOOTSTRAP_VERSION}.zip -wget https://github.com/FezVrasta/popper.js/archive/v1.14.3.zip -O temp/popper.zip -wget https://github.com/BlackrockDigital/startbootstrap-sb-admin/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}.zip -wget https://github.com/BlackrockDigital/startbootstrap-sb-admin-2/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}-2.zip -wget https://github.com/FortAwesome/Font-Awesome/archive/v4.7.0.zip -O temp/FONT_AWESOME_4.7.0.zip -wget https://github.com/FortAwesome/Font-Awesome/archive/5.7.1.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -wget https://github.com/d3/d3/releases/download/v${D3_JS_VERSION}/d3.zip -O temp/d3_${D3_JS_VERSION}.zip +wget -q https://github.com/twbs/bootstrap/releases/download/v${BOOTSTRAP_VERSION}/bootstrap-${BOOTSTRAP_VERSION}-dist.zip -O temp/bootstrap${BOOTSTRAP_VERSION}.zip +wget -q https://github.com/FezVrasta/popper.js/archive/v1.14.3.zip -O temp/popper.zip +wget -q https://github.com/BlackrockDigital/startbootstrap-sb-admin/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}.zip +wget -q https://github.com/BlackrockDigital/startbootstrap-sb-admin-2/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}-2.zip +wget -q https://github.com/FortAwesome/Font-Awesome/archive/v4.7.0.zip -O temp/FONT_AWESOME_4.7.0.zip +wget -q https://github.com/FortAwesome/Font-Awesome/archive/5.7.1.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip +wget -q https://github.com/d3/d3/releases/download/v${D3_JS_VERSION}/d3.zip -O temp/d3_${D3_JS_VERSION}.zip # dateRangePicker -wget https://github.com/moment/moment/archive/2.22.2.zip -O temp/moment_2.22.2.zip -wget https://github.com/longbill/jquery-date-range-picker/archive/v0.18.0.zip -O temp/daterangepicker_v0.18.0.zip +wget -q https://github.com/moment/moment/archive/2.22.2.zip -O temp/moment_2.22.2.zip +wget -q https://github.com/longbill/jquery-date-range-picker/archive/v0.18.0.zip -O temp/daterangepicker_v0.18.0.zip -unzip temp/bootstrap${BOOTSTRAP_VERSION}.zip -d temp/ -unzip temp/popper.zip -d temp/ -unzip temp/${SBADMIN_VERSION}.zip -d temp/ -unzip temp/${SBADMIN_VERSION}-2.zip -d temp/ -unzip temp/FONT_AWESOME_4.7.0.zip -d temp/ -unzip temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/ -unzip temp/d3_${D3_JS_VERSION}.zip -d temp/ +unzip -qq temp/bootstrap${BOOTSTRAP_VERSION}.zip -d temp/ +unzip -qq temp/popper.zip -d temp/ +unzip -qq temp/${SBADMIN_VERSION}.zip -d temp/ +unzip -qq temp/${SBADMIN_VERSION}-2.zip -d temp/ +unzip -qq temp/FONT_AWESOME_4.7.0.zip -d temp/ +unzip -qq temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/ +unzip -qq temp/d3_${D3_JS_VERSION}.zip -d temp/ -unzip temp/moment_2.22.2.zip -d temp/ -unzip temp/daterangepicker_v0.18.0.zip -d temp/ +unzip -qq temp/moment_2.22.2.zip -d temp/ +unzip -qq temp/daterangepicker_v0.18.0.zip -d temp/ mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/js/bootstrap.min.js ./static/js/bootstrap4.min.js mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/js/bootstrap.min.js.map ./static/js/bootstrap.min.js.map @@ -70,38 +70,38 @@ mv temp/jquery-date-range-picker-0.18.0/dist/jquery.daterangepicker.min.js ./sta rm -rf temp JQVERSION="1.12.4" -wget http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js +wget -q http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js #Ressources for dataTable -wget https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min.js -O ./static/js/jquery.dataTables.min.js -wget https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTables.bootstrap.css -O ./static/css/dataTables.bootstrap.css -wget https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTables.bootstrap.js -O ./static/js/dataTables.bootstrap.js +wget -q https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min.js -O ./static/js/jquery.dataTables.min.js +wget -q https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTables.bootstrap.css -O ./static/css/dataTables.bootstrap.css +wget -q https://cdn.datatables.net/plug-ins/1.10.7/integration/bootstrap/3/dataTables.bootstrap.js -O ./static/js/dataTables.bootstrap.js -wget https://cdn.datatables.net/1.10.18/css/dataTables.bootstrap4.min.css -O ./static/css/dataTables.bootstrap.min.css -wget https://cdn.datatables.net/1.10.18/js/dataTables.bootstrap4.min.js -O ./static/js/dataTables.bootstrap.min.js +wget -q https://cdn.datatables.net/1.10.18/css/dataTables.bootstrap4.min.css -O ./static/css/dataTables.bootstrap.min.css +wget -q https://cdn.datatables.net/1.10.18/js/dataTables.bootstrap4.min.js -O ./static/js/dataTables.bootstrap.min.js #Ressource for graph -wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.js -O ./static/js/jquery.flot.js -wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.pie.js -O ./static/js/jquery.flot.pie.js -wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.time.js -O ./static/js/jquery.flot.time.js -wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.stack.js -O ./static/js/jquery.flot.stack.js +wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.js -O ./static/js/jquery.flot.js +wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.pie.js -O ./static/js/jquery.flot.pie.js +wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.time.js -O ./static/js/jquery.flot.time.js +wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.stack.js -O ./static/js/jquery.flot.stack.js #Ressources for sparkline and canvasJS and slider -wget http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js +wget -q http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js mkdir temp -wget http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip -unzip temp/canvasjs.zip -d temp/ +wget -q http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip +unzip -qq temp/canvasjs.zip -d temp/ mv temp/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js -wget https://jqueryui.com/resources/download/jquery-ui-1.12.0.zip -O temp/jquery-ui.zip -unzip temp/jquery-ui.zip -d temp/ +wget -q https://jqueryui.com/resources/download/jquery-ui-1.12.0.zip -O temp/jquery-ui.zip +unzip -qq temp/jquery-ui.zip -d temp/ mv temp/jquery-ui-1.12.0/jquery-ui.min.js ./static/js/jquery-ui.min.js mv temp/jquery-ui-1.12.0/jquery-ui.min.css ./static/css/jquery-ui.min.css rm -rf temp mkdir -p ./static/image pushd static/image -wget https://www.circl.lu/assets/images/logos/AIL.png -O AIL.png +wget -q https://www.circl.lu/assets/images/logos/AIL.png -O AIL.png popd if ! [[ -n "$AIL_HOME" ]]