chg: [user_management 2.0] add update scripts + fix create_default_user

This commit is contained in:
Terrtia 2019-06-20 14:47:59 +02:00
parent 821cf3cbea
commit a9837c6e27
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
13 changed files with 145 additions and 319 deletions

2
.gitignore vendored
View file

@ -20,6 +20,8 @@ indexdir/
logs/
old/
DEFAULT_PASSWORD
# Webstuff
var/www/static/
!var/www/static/css/dygraph_gallery.css

View file

@ -42,8 +42,6 @@ Redis and ARDB overview
| | **background update name** |
| | **...** |
| | |
| ail:update_date_v1.5 | **update date** |
| | |
| ail:update_error | **update message error** |
| | |
| ail:update_in_progress | **update version in progress** |
@ -52,6 +50,10 @@ Redis and ARDB overview
| ail:current_background_script | **name of the background script currently executed** |
| ail:current_background_script_stat | **progress in % of the background script** |
| Hset Key | Field | Value |
| ------ | ------ | ------ |
| ail:update_date | **update tag** | **update date** |
##### User Management:
| Hset Key | Field | Value |
| ------ | ------ | ------ |

View file

@ -101,6 +101,7 @@ fi
pushd var/www/
./update_thirdparty.sh
python3 create_default_user.py
popd
mkdir -p $AIL_HOME/PASTES

View file

@ -1,108 +0,0 @@
#!/bin/bash
echo "Currently unmaintained, continue at your own risk of not having a working AIL at the end :( Will be merged into main install deps later on."
exit 1
set -e
set -x
sudo pacman -Syu
sudo pacman -S python2-pip screen gcc unzip freetype2 python2 git --needed
sudo yaourt -S snappy --needed
sudo pip2 install virtualenv
#Needed for bloom filters
sudo pacman -S openssl python2-numpy --needed
# DNS deps
sudo pacman -S adns --needed
#Needed for redis-lvlDB
sudo pacman -S libev gmp --needed
#needed for mathplotlib
test ! -L /usr/include/ft2build.h && sudo ln -s freetype2/ft2build.h /usr/include/
sudo easy_install-2.7 -U distribute
# REDIS #
test ! -d redis/ && git clone https://github.com/antirez/redis.git
pushd redis/
git checkout 2.8
make
popd
# REDIS LEVEL DB #
test ! -d redis-leveldb/ && git clone https://github.com/KDr2/redis-leveldb.git
pushd redis-leveldb/
git submodule init
git submodule update
make
popd
# Faup
test ! -d faup/ && git clone https://github.com/stricaud/faup.git
pushd faup/
test ! -d build && mkdir build
cd build
cmake .. && make
sudo make install
echo '/usr/local/lib' | sudo tee -a /etc/ld.so.conf.d/faup.conf
sudo ldconfig
popd
# tlsh
test ! -d tlsh && git clone git://github.com/trendmicro/tlsh.git
pushd tlsh/
./make.sh
pushd build/release/
sudo make install
sudo ldconfig
popd
popd
if [ ! -f bin/packages/config.cfg ]; then
cp bin/packages/config.cfg.sample bin/packages/config.cfg
fi
pushd var/www/
./update_thirdparty.sh
popd
virtualenv AILENV
echo export AIL_HOME=$(pwd) >> ./AILENV/bin/activate
echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate
echo export AIL_FLASK=$(pwd)/var/www/ >> ./AILENV/bin/activate
echo export AIL_REDIS=$(pwd)/redis/src/ >> ./AILENV/bin/activate
echo export AIL_LEVELDB=$(pwd)/redis-leveldb/ >> ./AILENV/bin/activate
. ./AILENV/bin/activate
mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}
mkdir -p $AIL_HOME/LEVEL_DB_DATA/2017
mkdir -p $AIL_HOME/LEVEL_DB_DATA/3017
pip install -U pip
pip install -U -r pip_packages_requirement.txt
# Pyfaup
pushd faup/src/lib/bindings/python/
python setup.py install
popd
# Py tlsh
pushd tlsh/py_ext
python setup.py build
python setup.py install
# Download the necessary NLTK corpora and sentiment vader
HOME=$(pwd) python -m textblob.download_corpora
python -m nltk.downloader vader_lexicon
python -m nltk.downloader punkt
#Create the file all_module and update the graph in doc
$AIL_HOME/doc/generate_modules_data_flow_graph.sh

41
update/v2.0/Update.py Executable file
View file

@ -0,0 +1,41 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import time
import redis
import datetime
import configparser
if __name__ == '__main__':
start_deb = time.time()
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \
Did you set environment variables? \
Or activate the virtualenv.')
cfg = configparser.ConfigParser()
cfg.read(configfile)
r_serv = redis.StrictRedis(
host=cfg.get("ARDB_DB", "host"),
port=cfg.getint("ARDB_DB", "port"),
db=cfg.getint("ARDB_DB", "db"),
decode_responses=True)
#Set current ail version
r_serv.set('ail:version', 'v2.0')
# use new update_date format
date_tag_to_replace = ['v1.5', 'v1.7']
for tag in date_tag_to_replace:
if r_serv.exists('ail:update_date_{}'.format(tag)):
date_tag = r_serv.get('ail:update_date_{}'.format(tag))
r_serv.hset('ail:update_date', tag, date_tag)
r_serv.delete('ail:update_date_{}'.format(tag))
#Set current ail version
r_serv.hset('ail:update_date', 'v2.0', datetime.datetime.now().strftime("%Y%m%d"))

62
update/v2.0/Update.sh Executable file
View file

@ -0,0 +1,62 @@
#!/bin/bash
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
export PATH=$AIL_HOME:$PATH
export PATH=$AIL_REDIS:$PATH
export PATH=$AIL_ARDB:$PATH
export PATH=$AIL_BIN:$PATH
export PATH=$AIL_FLASK:$PATH
GREEN="\\033[1;32m"
DEFAULT="\\033[0;39m"
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -k
wait
echo ""
echo -e $GREEN"Update requirement"$DEFAULT
echo ""
pip3 install flask-login
wait
echo ""
pip3 install bcrypt
wait
echo ""
echo ""
bash ${AIL_BIN}LAUNCH.sh -lav &
wait
echo ""
echo ""
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
echo ""
python ${AIL_HOME}/update/v2.0/Update.py
wait
echo ""
echo ""
echo ""
echo -e $GREEN"Update thirdparty ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -t
wait
echo ""
echo ""
echo -e $GREEN"Create Default User"$DEFAULT
echo ""
python3 ${AIL_FLASK}create_default_user.py
echo ""
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -k
wait
exit 0

View file

@ -36,18 +36,6 @@ import Flask_config
from Role_Manager import create_user_db, check_password_strength
from Role_Manager import login_admin, login_analyst
def flask_init():
# # TODO: move this to update
# role init
if not r_serv_db.exists('ail:all_role'):
r_serv_db.zadd('ail:all_role', 1, 'admin')
r_serv_db.zadd('ail:all_role', 2, 'analyst')
# check if an account exists
if not r_serv_db.exists('user:all'):
password = secrets.token_urlsafe()
create_user_db('admin@admin.test', password, role='admin',default=True)
# CONFIG #
cfg = Flask_config.cfg
baseUrl = cfg.get("Flask", "baseurl")

View file

@ -3,20 +3,44 @@
import os
import sys
import redis
import secrets
import configparser
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
sys.path.append('./modules/')
sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules'))
from Role_Manager import create_user_db, get_default_admin_token
from Role_Manager import create_user_db, edit_user_db, get_default_admin_token
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \
Did you set environment variables? \
Or activate the virtualenv.')
cfg = configparser.ConfigParser()
cfg.read(configfile)
r_serv = redis.StrictRedis(
host=cfg.get("ARDB_DB", "host"),
port=cfg.getint("ARDB_DB", "port"),
db=cfg.getint("ARDB_DB", "db"),
decode_responses=True)
if __name__ == "__main__":
# create role_list
if not r_serv.exists('ail:all_role'):
r_serv.zadd('ail:all_role', 1, 'admin')
r_serv.zadd('ail:all_role', 2, 'analyst')
username = 'admin@admin.test'
password = secrets.token_urlsafe()
create_user_db(username, password, role='admin', default=True)
if r_serv.exists('user_metadata:admin@admin.test'):
edit_user_db(username, password=password, role='admin')
else:
create_user_db(username, password, role='admin', default=True)
token = get_default_admin_token()
default_passwd_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD')

View file

@ -93,10 +93,10 @@ def one():
# ============= ROUTES ==============
@restApi.route("/api", methods=['GET'])
@login_required
def api():
return 'api doc'
# @restApi.route("/api", methods=['GET'])
# @login_required
# def api():
# return 'api doc'
@restApi.route("api/items", methods=['POST'])
@token_required

View file

@ -26,82 +26,6 @@
<div class="col-12 col-lg-10" id="core_content">
<div class="row">
<div class="col-xl-6">
<div class="card mt-1 mb-1">
<div class="card-header text-white bg-dark">
<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=onion"><i class="fas fa-user-secret"></i> Onions Crawlers</a></h5>
<div class="row">
<div class="col-6">
<span class="badge badge-success" id="stat_onion_domain_up">{{ statDomains_onion['domains_up'] }}</span> UP
<span class="badge badge-danger ml-md-3" id="stat_onion_domain_down">{{ statDomains_onion['domains_down'] }}</span> DOWN
</div>
<div class="col-6">
<span class="badge badge-success" id="stat_onion_total">{{ statDomains_onion['total'] }}</span> Crawled
<span class="badge badge-warning ml-md-3" id="stat_onion_queue">{{ statDomains_onion['domains_queue'] }}</span> Queue
</div>
</div>
</div>
<div class="card-body px-0 py-0 ">
<table class="table">
<tbody id="tbody_crawler_onion_info">
{% for crawler in crawler_metadata_onion %}
<tr>
<td>
<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
</td>
<td>
{{crawler['crawling_domain']}}
</td>
<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
{{crawler['status_info']}}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
<div class="col-xl-6">
<div class="card mt-1 mb-1">
<div class="card-header text-white bg-dark">
<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=regular"><i class="fab fa-html5"></i> Regular Crawlers</a></h5>
<div class="row">
<div class="col-6">
<span class="badge badge-success" id="stat_regular_domain_up">{{ statDomains_regular['domains_up'] }}</span> UP
<span class="badge badge-danger ml-md-3" id="stat_regular_domain_down">{{ statDomains_regular['domains_down'] }}</span> DOWN
</div>
<div class="col-6">
<span class="badge badge-success" id="stat_regular_total">{{ statDomains_regular['total'] }}</span> Crawled
<span class="badge badge-warning ml-md-3" id="stat_regular_queue">{{ statDomains_regular['domains_queue'] }}</span> Queue
</div>
</div>
</div>
<div class="card-body px-0 py-0 ">
<table class="table">
<tbody id="tbody_crawler_regular_info">
{% for crawler in crawler_metadata_regular %}
<tr>
<td>
<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
</td>
<td>
{{crawler['crawling_domain']}}
</td>
<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
{{crawler['status_info']}}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
</div>
@ -109,114 +33,3 @@
</body>
<script>
var to_refresh = false
$(document).ready(function(){
$("#page-Crawler").addClass("active");
$("#nav_dashboard").addClass("active");
$( window ).focus(function() {
to_refresh = true
refresh_crawler_status();
});
$( window ).blur(function() {
to_refresh = false
});
to_refresh = true
refresh_crawler_status();
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
function refresh_crawler_status(){
$.getJSON("{{ url_for('hiddenServices.crawler_dashboard_json') }}",
function(data) {
$('#stat_onion_domain_up').text(data.statDomains_onion['domains_up']);
$('#stat_onion_domain_down').text(data.statDomains_onion['domains_down']);
$('#stat_onion_total').text(data.statDomains_onion['total']);
$('#stat_onion_queue').text(data.statDomains_onion['domains_queue']);
$('#stat_regular_domain_up').text(data.statDomains_regular['domains_up']);
$('#stat_regular_domain_down').text(data.statDomains_regular['domains_down']);
$('#stat_regular_total').text(data.statDomains_regular['total']);
$('#stat_regular_queue').text(data.statDomains_regular['domains_queue']);
if(data.crawler_metadata_onion.length!=0){
$("#tbody_crawler_onion_info").empty();
var tableRef = document.getElementById('tbody_crawler_onion_info');
for (var i = 0; i < data.crawler_metadata_onion.length; i++) {
var crawler = data.crawler_metadata_onion[i];
var newRow = tableRef.insertRow(tableRef.rows.length);
var text_color;
var icon;
if(crawler['status']){
text_color = 'Green';
icon = 'check';
} else {
text_color = 'Red';
icon = 'times';
}
var newCell = newRow.insertCell(0);
newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
newCell = newRow.insertCell(1);
newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
newCell = newRow.insertCell(2);
newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
//$("#panel_crawler").show();
}
}
if(data.crawler_metadata_regular.length!=0){
$("#tbody_crawler_regular_info").empty();
var tableRef = document.getElementById('tbody_crawler_regular_info');
for (var i = 0; i < data.crawler_metadata_regular.length; i++) {
var crawler = data.crawler_metadata_regular[i];
var newRow = tableRef.insertRow(tableRef.rows.length);
var text_color;
var icon;
if(crawler['status']){
text_color = 'Green';
icon = 'check';
} else {
text_color = 'Red';
icon = 'times';
}
var newCell = newRow.insertCell(0);
newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
newCell = newRow.insertCell(1);
newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
newCell = newRow.insertCell(2);
newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
//$("#panel_crawler").show();
}
}
}
);
if (to_refresh) {
setTimeout("refresh_crawler_status()", 10000);
}
}
</script>

View file

@ -1 +0,0 @@
<li id='page-hiddenServices'><a href="{{ url_for('hiddenServices.dashboard') }}"><i class="fa fa-user-secret"></i> hidden Services </a></li>

View file

@ -7,6 +7,7 @@
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
</head>
<body>

View file

@ -7,6 +7,7 @@
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
</head>
<body>