diff --git a/.gitignore b/.gitignore index 785419e5..28ed8063 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,8 @@ indexdir/ logs/ old/ +DEFAULT_PASSWORD + # Webstuff var/www/static/ !var/www/static/css/dygraph_gallery.css diff --git a/OVERVIEW.md b/OVERVIEW.md index 97991b57..74674bd4 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -42,8 +42,6 @@ Redis and ARDB overview | | **background update name** | | | **...** | | | | -| ail:update_date_v1.5 | **update date** | -| | | | ail:update_error | **update message error** | | | | | ail:update_in_progress | **update version in progress** | @@ -52,6 +50,10 @@ Redis and ARDB overview | ail:current_background_script | **name of the background script currently executed** | | ail:current_background_script_stat | **progress in % of the background script** | +| Hset Key | Field | Value | +| ------ | ------ | ------ | +| ail:update_date | **update tag** | **update date** | + ##### User Management: | Hset Key | Field | Value | | ------ | ------ | ------ | diff --git a/installing_deps.sh b/installing_deps.sh index e8de3fd6..bf72cf4b 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -101,6 +101,7 @@ fi pushd var/www/ ./update_thirdparty.sh +python3 create_default_user.py popd mkdir -p $AIL_HOME/PASTES diff --git a/installing_deps_archlinux.sh b/installing_deps_archlinux.sh deleted file mode 100755 index 38ac4ca7..00000000 --- a/installing_deps_archlinux.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - - -echo "Currently unmaintained, continue at your own risk of not having a working AIL at the end :( Will be merged into main install deps later on." -exit 1 - -set -e -set -x - -sudo pacman -Syu - -sudo pacman -S python2-pip screen gcc unzip freetype2 python2 git --needed -sudo yaourt -S snappy --needed -sudo pip2 install virtualenv - -#Needed for bloom filters -sudo pacman -S openssl python2-numpy --needed - -# DNS deps -sudo pacman -S adns --needed - -#Needed for redis-lvlDB -sudo pacman -S libev gmp --needed - -#needed for mathplotlib -test ! -L /usr/include/ft2build.h && sudo ln -s freetype2/ft2build.h /usr/include/ -sudo easy_install-2.7 -U distribute - -# REDIS # -test ! -d redis/ && git clone https://github.com/antirez/redis.git -pushd redis/ -git checkout 2.8 -make -popd - -# REDIS LEVEL DB # -test ! -d redis-leveldb/ && git clone https://github.com/KDr2/redis-leveldb.git -pushd redis-leveldb/ -git submodule init -git submodule update -make -popd - -# Faup -test ! -d faup/ && git clone https://github.com/stricaud/faup.git -pushd faup/ -test ! -d build && mkdir build -cd build -cmake .. && make -sudo make install -echo '/usr/local/lib' | sudo tee -a /etc/ld.so.conf.d/faup.conf -sudo ldconfig -popd - -# tlsh -test ! -d tlsh && git clone git://github.com/trendmicro/tlsh.git -pushd tlsh/ -./make.sh -pushd build/release/ -sudo make install -sudo ldconfig -popd -popd - - - -if [ ! -f bin/packages/config.cfg ]; then - cp bin/packages/config.cfg.sample bin/packages/config.cfg -fi - -pushd var/www/ -./update_thirdparty.sh -popd - -virtualenv AILENV - -echo export AIL_HOME=$(pwd) >> ./AILENV/bin/activate -echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate -echo export AIL_FLASK=$(pwd)/var/www/ >> ./AILENV/bin/activate -echo export AIL_REDIS=$(pwd)/redis/src/ >> ./AILENV/bin/activate -echo export AIL_LEVELDB=$(pwd)/redis-leveldb/ >> ./AILENV/bin/activate - -. ./AILENV/bin/activate - -mkdir -p $AIL_HOME/{PASTES,Blooms,dumps} -mkdir -p $AIL_HOME/LEVEL_DB_DATA/2017 -mkdir -p $AIL_HOME/LEVEL_DB_DATA/3017 - -pip install -U pip -pip install -U -r pip_packages_requirement.txt - -# Pyfaup -pushd faup/src/lib/bindings/python/ -python setup.py install -popd - -# Py tlsh -pushd tlsh/py_ext -python setup.py build -python setup.py install - -# Download the necessary NLTK corpora and sentiment vader -HOME=$(pwd) python -m textblob.download_corpora -python -m nltk.downloader vader_lexicon -python -m nltk.downloader punkt - -#Create the file all_module and update the graph in doc -$AIL_HOME/doc/generate_modules_data_flow_graph.sh diff --git a/update/v2.0/Update.py b/update/v2.0/Update.py new file mode 100755 index 00000000..4e026e1f --- /dev/null +++ b/update/v2.0/Update.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import time +import redis +import datetime +import configparser + +if __name__ == '__main__': + + start_deb = time.time() + + configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + cfg = configparser.ConfigParser() + cfg.read(configfile) + + r_serv = redis.StrictRedis( + host=cfg.get("ARDB_DB", "host"), + port=cfg.getint("ARDB_DB", "port"), + db=cfg.getint("ARDB_DB", "db"), + decode_responses=True) + + #Set current ail version + r_serv.set('ail:version', 'v2.0') + + # use new update_date format + date_tag_to_replace = ['v1.5', 'v1.7'] + for tag in date_tag_to_replace: + if r_serv.exists('ail:update_date_{}'.format(tag)): + date_tag = r_serv.get('ail:update_date_{}'.format(tag)) + r_serv.hset('ail:update_date', tag, date_tag) + r_serv.delete('ail:update_date_{}'.format(tag)) + + #Set current ail version + r_serv.hset('ail:update_date', 'v2.0', datetime.datetime.now().strftime("%Y%m%d")) diff --git a/update/v2.0/Update.sh b/update/v2.0/Update.sh new file mode 100755 index 00000000..bda6a127 --- /dev/null +++ b/update/v2.0/Update.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; + +export PATH=$AIL_HOME:$PATH +export PATH=$AIL_REDIS:$PATH +export PATH=$AIL_ARDB:$PATH +export PATH=$AIL_BIN:$PATH +export PATH=$AIL_FLASK:$PATH + +GREEN="\\033[1;32m" +DEFAULT="\\033[0;39m" + +echo -e $GREEN"Shutting down AIL ..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -k +wait + +echo "" +echo -e $GREEN"Update requirement"$DEFAULT +echo "" +pip3 install flask-login +wait +echo "" +pip3 install bcrypt +wait +echo "" +echo "" + +bash ${AIL_BIN}LAUNCH.sh -lav & +wait +echo "" + +echo "" +echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT +echo "" +python ${AIL_HOME}/update/v2.0/Update.py +wait +echo "" +echo "" + +echo "" +echo -e $GREEN"Update thirdparty ..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -t +wait +echo "" + +echo "" +echo -e $GREEN"Create Default User"$DEFAULT +echo "" +python3 ${AIL_FLASK}create_default_user.py + + +echo "" +echo -e $GREEN"Shutting down ARDB ..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -k +wait + +exit 0 diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 71827852..3dbab06a 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -36,18 +36,6 @@ import Flask_config from Role_Manager import create_user_db, check_password_strength from Role_Manager import login_admin, login_analyst -def flask_init(): - # # TODO: move this to update - # role init - if not r_serv_db.exists('ail:all_role'): - r_serv_db.zadd('ail:all_role', 1, 'admin') - r_serv_db.zadd('ail:all_role', 2, 'analyst') - - # check if an account exists - if not r_serv_db.exists('user:all'): - password = secrets.token_urlsafe() - create_user_db('admin@admin.test', password, role='admin',default=True) - # CONFIG # cfg = Flask_config.cfg baseUrl = cfg.get("Flask", "baseurl") diff --git a/var/www/create_default_user.py b/var/www/create_default_user.py index 9391af56..c0735611 100755 --- a/var/www/create_default_user.py +++ b/var/www/create_default_user.py @@ -3,20 +3,44 @@ import os import sys - +import redis import secrets +import configparser -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) -sys.path.append('./modules/') +sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules')) -from Role_Manager import create_user_db, get_default_admin_token +from Role_Manager import create_user_db, edit_user_db, get_default_admin_token + + + +configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') +if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') +cfg = configparser.ConfigParser() +cfg.read(configfile) + +r_serv = redis.StrictRedis( + host=cfg.get("ARDB_DB", "host"), + port=cfg.getint("ARDB_DB", "port"), + db=cfg.getint("ARDB_DB", "db"), + decode_responses=True) if __name__ == "__main__": + # create role_list + if not r_serv.exists('ail:all_role'): + r_serv.zadd('ail:all_role', 1, 'admin') + r_serv.zadd('ail:all_role', 2, 'analyst') + username = 'admin@admin.test' password = secrets.token_urlsafe() - create_user_db(username, password, role='admin', default=True) + if r_serv.exists('user_metadata:admin@admin.test'): + edit_user_db(username, password=password, role='admin') + else: + create_user_db(username, password, role='admin', default=True) token = get_default_admin_token() default_passwd_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD') diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index 82c5ba45..4535da28 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -93,10 +93,10 @@ def one(): # ============= ROUTES ============== -@restApi.route("/api", methods=['GET']) -@login_required -def api(): - return 'api doc' +# @restApi.route("/api", methods=['GET']) +# @login_required +# def api(): +# return 'api doc' @restApi.route("api/items", methods=['POST']) @token_required diff --git a/var/www/modules/restApi/templates/api_default.html b/var/www/modules/restApi/templates/api_default.html index 91d300af..cb2cc24f 100644 --- a/var/www/modules/restApi/templates/api_default.html +++ b/var/www/modules/restApi/templates/api_default.html @@ -26,82 +26,6 @@
-
-
- -
-
-
Onions Crawlers
-
-
- {{ statDomains_onion['domains_up'] }} UP - {{ statDomains_onion['domains_down'] }} DOWN -
-
- {{ statDomains_onion['total'] }} Crawled - {{ statDomains_onion['domains_queue'] }} Queue -
-
-
-
- - - {% for crawler in crawler_metadata_onion %} - - - - - - {% endfor %} - -
- {{crawler['crawler_info']}} - - {{crawler['crawling_domain']}} - - {{crawler['status_info']}} -
-
-
- -
-
-
-
-
Regular Crawlers
-
-
- {{ statDomains_regular['domains_up'] }} UP - {{ statDomains_regular['domains_down'] }} DOWN -
-
- {{ statDomains_regular['total'] }} Crawled - {{ statDomains_regular['domains_queue'] }} Queue -
-
-
-
- - - {% for crawler in crawler_metadata_regular %} - - - - - - {% endfor %} - -
- {{crawler['crawler_info']}} - - {{crawler['crawling_domain']}} - - {{crawler['status_info']}} -
-
-
-
-
@@ -109,114 +33,3 @@ - - diff --git a/var/www/modules/restApi/templates/header_api.html b/var/www/modules/restApi/templates/header_api.html deleted file mode 100644 index 34e20319..00000000 --- a/var/www/modules/restApi/templates/header_api.html +++ /dev/null @@ -1 +0,0 @@ -
  • hidden Services
    • diff --git a/var/www/templates/error/403.html b/var/www/templates/error/403.html index e90433ba..3193da28 100644 --- a/var/www/templates/error/403.html +++ b/var/www/templates/error/403.html @@ -7,6 +7,7 @@ + diff --git a/var/www/templates/error/404.html b/var/www/templates/error/404.html index 6943c674..f911a550 100644 --- a/var/www/templates/error/404.html +++ b/var/www/templates/error/404.html @@ -7,6 +7,7 @@ +