From 1ab1a55a4f809cc531b719133c7c0bb1c5cbc0d2 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Thu, 2 May 2019 17:31:14 +0200
Subject: [PATCH 01/22] chg: [UI] add basic user management

---
 bin/packages/User.py                          | 36 +++++++++
 pip3_packages_requirement.txt                 |  4 +-
 var/www/Flask_server.py                       | 54 ++++++++++++-
 .../modules/PasteSubmit/Flask_PasteSubmit.py  | 14 ++++
 var/www/modules/Tags/Flask_Tags.py            | 22 +++++
 .../modules/hashDecoded/Flask_hashDecoded.py  | 17 ++++
 .../hiddenServices/Flask_hiddenServices.py    | 20 +++++
 .../modules/rawSkeleton/Flask_rawSkeleton.py  |  2 +
 var/www/modules/search/Flask_search.py        |  3 +
 var/www/modules/sentiment/Flask_sentiment.py  |  5 ++
 var/www/modules/settings/Flask_settings.py    |  3 +
 var/www/modules/showpaste/Flask_showpaste.py  | 10 +++
 var/www/modules/terms/Flask_terms.py          | 15 ++++
 .../trendingcharts/Flask_trendingcharts.py    |  5 ++
 .../trendingmodules/Flask_trendingmodules.py  |  4 +
 var/www/templates/login.html                  | 81 +++++++++++++++++++
 16 files changed, 293 insertions(+), 2 deletions(-)
 create mode 100755 bin/packages/User.py
 create mode 100644 var/www/templates/login.html

diff --git a/bin/packages/User.py b/bin/packages/User.py
new file mode 100755
index 00000000..b46069f1
--- /dev/null
+++ b/bin/packages/User.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import redis
+
+from flask_login import UserMixin
+
+class User(UserMixin):
+
+    def __init__(self, id):
+        self.id = 'abcdef'
+
+    # return True or False
+    #def is_authenticated():
+
+    # return True or False
+    #def is_active():
+
+    # return True or False
+    #def is_anonymous():
+
+    @classmethod
+    def get(self_class, id):
+        print(id)
+        return self_class(id)
+
+    def check_password(self, password):
+        print(self.id)
+        if password=='admin':
+            print('password ok')
+            return True
+        else:
+            return False
+
+    def set_password(self):
+        return True
diff --git a/pip3_packages_requirement.txt b/pip3_packages_requirement.txt
index 3991e158..4f9d1f87 100644
--- a/pip3_packages_requirement.txt
+++ b/pip3_packages_requirement.txt
@@ -43,9 +43,11 @@ psutil
 phonenumbers
 
 ipython
-flask
 texttable
 
+flask
+flask-login
+
 #DomainClassifier
 DomainClassifier
 #Indexer requirements
diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py
index 95433757..907f1fd4 100755
--- a/var/www/Flask_server.py
+++ b/var/www/Flask_server.py
@@ -3,11 +3,14 @@
 
 import redis
 import configparser
+import random
 import json
 import datetime
 import time
 import calendar
-from flask import Flask, render_template, jsonify, request, Request
+from flask import Flask, render_template, jsonify, request, Request, session, redirect, url_for
+from flask_login import LoginManager, current_user, login_user, logout_user, login_required
+
 import flask
 import importlib
 import os
@@ -18,6 +21,8 @@ sys.path.append('./modules/')
 import Paste
 from Date import Date
 
+from User import User
+
 from pytaxonomies import Taxonomies
 
 # Import config
@@ -34,6 +39,18 @@ Flask_config.app = Flask(__name__, static_url_path=baseUrl+'/static/')
 app = Flask_config.app
 app.config['MAX_CONTENT_LENGTH'] = 900 * 1024 * 1024
 
+# ========= session ========
+app.secret_key = str(random.getrandbits(256))
+login_manager = LoginManager()
+login_manager.login_view = 'login'
+login_manager.init_app(app)
+
+# ========= LOGIN MANAGER ========
+
+@login_manager.user_loader
+def load_user(user_id):
+    return User.get(user_id)
+
 # ========= HEADER GENERATION ========
 
 # Get headers items that should be ignored (not displayed)
@@ -118,6 +135,41 @@ def add_header(response):
     return response
 
 # ========== ROUTES ============
+@app.route('/login', methods=['POST', 'GET'])
+def login():
+    if request.method == 'POST':
+        username = request.form.get('username')
+        password = request.form.get('password')
+        next_page = request.form.get('next_page')
+
+        print(username)
+        print(password)
+
+        if username is not None:
+            user = User.get(username)
+            #print(user.is_anonymous)
+            #print('auth') # TODO: overwrite
+            #print(user.is_authenticated)
+            if user and user.check_password(password):
+                login_user(user) ## TODO: use remember me ?
+                return redirect(url_for('dashboard.index'))
+            else:
+                return 'incorrect password'
+
+        return 'none'
+
+    else:
+        next_page = request.args.get('next')
+        print(next_page)
+        return render_template("login.html", next_page=next_page)
+
+@app.route('/logout')
+@login_required
+def logout():
+    logout_user()
+    return redirect(url_for('dashboard.index'))
+
+
 @app.route('/searchbox/')
 def searchbox():
     return render_template("searchbox.html")
diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py
index cc38de77..eba8541a 100644
--- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py
+++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py
@@ -6,6 +6,7 @@
 '''
 import redis
 from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect
+from flask_login import login_required
 
 import unicodedata
 import string
@@ -273,6 +274,7 @@ def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title,
 # ============= ROUTES ==============
 
 @PasteSubmit.route("/PasteSubmit/", methods=['GET'])
+@login_required
 def PasteSubmit_page():
     #active taxonomies
     active_taxonomies = r_serv_tags.smembers('active_taxonomies')
@@ -285,6 +287,7 @@ def PasteSubmit_page():
                             active_galaxies = active_galaxies)
 
 @PasteSubmit.route("/PasteSubmit/submit", methods=['POST'])
+@login_required
 def submit():
 
     #paste_name = request.form['paste_name']
@@ -385,6 +388,7 @@ def submit():
     return PasteSubmit_page()
 
 @PasteSubmit.route("/PasteSubmit/submit_status", methods=['GET'])
+@login_required
 def submit_status():
     UUID = request.args.get('UUID')
 
@@ -451,6 +455,7 @@ def submit_status():
 
 
 @PasteSubmit.route("/PasteSubmit/create_misp_event", methods=['POST'])
+@login_required
 def create_misp_event():
 
     distribution = int(request.form['misp_data[Event][distribution]'])
@@ -473,6 +478,7 @@ def create_misp_event():
     return 'error0'
 
 @PasteSubmit.route("/PasteSubmit/create_hive_case", methods=['POST'])
+@login_required
 def create_hive_case():
 
     hive_tlp = int(request.form['hive_tlp'])
@@ -495,6 +501,7 @@ def create_hive_case():
     return 'error'
 
 @PasteSubmit.route("/PasteSubmit/edit_tag_export")
+@login_required
 def edit_tag_export():
     misp_auto_events = r_serv_db.get('misp:auto-events')
     hive_auto_alerts = r_serv_db.get('hive:auto-alerts')
@@ -559,6 +566,7 @@ def edit_tag_export():
                             flag_hive=flag_hive)
 
 @PasteSubmit.route("/PasteSubmit/tag_export_edited", methods=['POST'])
+@login_required
 def tag_export_edited():
     tag_enabled_misp = request.form.getlist('tag_enabled_misp')
     tag_enabled_hive = request.form.getlist('tag_enabled_hive')
@@ -583,26 +591,31 @@ def tag_export_edited():
     return redirect(url_for('PasteSubmit.edit_tag_export'))
 
 @PasteSubmit.route("/PasteSubmit/enable_misp_auto_event")
+@login_required
 def enable_misp_auto_event():
     r_serv_db.set('misp:auto-events', 1)
     return edit_tag_export()
 
 @PasteSubmit.route("/PasteSubmit/disable_misp_auto_event")
+@login_required
 def disable_misp_auto_event():
     r_serv_db.set('misp:auto-events', 0)
     return edit_tag_export()
 
 @PasteSubmit.route("/PasteSubmit/enable_hive_auto_alert")
+@login_required
 def enable_hive_auto_alert():
     r_serv_db.set('hive:auto-alerts', 1)
     return edit_tag_export()
 
 @PasteSubmit.route("/PasteSubmit/disable_hive_auto_alert")
+@login_required
 def disable_hive_auto_alert():
     r_serv_db.set('hive:auto-alerts', 0)
     return edit_tag_export()
 
 @PasteSubmit.route("/PasteSubmit/add_push_tag")
+@login_required
 def add_push_tag():
     tag = request.args.get('tag')
     if tag is not None:
@@ -620,6 +633,7 @@ def add_push_tag():
         return 'None args', 400
 
 @PasteSubmit.route("/PasteSubmit/delete_push_tag")
+@login_required
 def delete_push_tag():
     tag = request.args.get('tag')
 
diff --git a/var/www/modules/Tags/Flask_Tags.py b/var/www/modules/Tags/Flask_Tags.py
index 3cc08159..ec329b30 100644
--- a/var/www/modules/Tags/Flask_Tags.py
+++ b/var/www/modules/Tags/Flask_Tags.py
@@ -6,6 +6,7 @@
 '''
 import redis
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
+from flask_login import login_required
 
 import json
 import datetime
@@ -218,6 +219,7 @@ def update_tag_last_seen(tag, tag_first_seen, tag_last_seen):
 # ============= ROUTES ==============
 
 @Tags.route("/tags/", methods=['GET'])
+@login_required
 def Tags_page():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -351,6 +353,7 @@ def Tags_page():
 
 
 @Tags.route("/Tags/get_all_tags")
+@login_required
 def get_all_tags():
 
     all_tags = r_serv_tags.smembers('list_tags')
@@ -373,6 +376,7 @@ def get_all_tags():
     return jsonify(list_tags)
 
 @Tags.route("/Tags/get_all_tags_taxonomies")
+@login_required
 def get_all_tags_taxonomies():
 
     taxonomies = Taxonomies()
@@ -390,6 +394,7 @@ def get_all_tags_taxonomies():
     return jsonify(list_tags)
 
 @Tags.route("/Tags/get_all_tags_galaxies")
+@login_required
 def get_all_tags_galaxy():
 
     active_galaxies = r_serv_tags.smembers('active_galaxies')
@@ -403,6 +408,7 @@ def get_all_tags_galaxy():
     return jsonify(list_tags)
 
 @Tags.route("/Tags/get_tags_taxonomie")
+@login_required
 def get_tags_taxonomie():
 
     taxonomie = request.args.get('taxonomie')
@@ -429,6 +435,7 @@ def get_tags_taxonomie():
         return 'INCORRECT INPUT'
 
 @Tags.route("/Tags/get_tags_galaxy")
+@login_required
 def get_tags_galaxy():
 
     galaxy = request.args.get('galaxy')
@@ -449,6 +456,7 @@ def get_tags_galaxy():
         return 'this galaxy is disable'
 
 @Tags.route("/Tags/remove_tag")
+@login_required
 def remove_tag():
 
     #TODO verify input
@@ -460,6 +468,7 @@ def remove_tag():
     return redirect(url_for('showsavedpastes.showsavedpaste', paste=path))
 
 @Tags.route("/Tags/confirm_tag")
+@login_required
 def confirm_tag():
 
     #TODO verify input
@@ -478,6 +487,7 @@ def confirm_tag():
     return 'incompatible tag'
 
 @Tags.route("/Tags/tag_validation")
+@login_required
 def tag_validation():
 
     path = request.args.get('paste')
@@ -498,6 +508,7 @@ def tag_validation():
         return 'input error'
 
 @Tags.route("/Tags/addTags")
+@login_required
 def addTags():
 
     tags = request.args.get('tags')
@@ -547,6 +558,7 @@ def addTags():
 
 
 @Tags.route("/Tags/taxonomies")
+@login_required
 def taxonomies():
 
     active_taxonomies = r_serv_tags.smembers('active_taxonomies')
@@ -583,6 +595,7 @@ def taxonomies():
                             n_tags=n_tags)
 
 @Tags.route("/Tags/edit_taxonomie")
+@login_required
 def edit_taxonomie():
 
     taxonomies = Taxonomies()
@@ -631,6 +644,7 @@ def edit_taxonomie():
         return 'INVALID TAXONOMIE'
 
 @Tags.route("/Tags/disable_taxonomie")
+@login_required
 def disable_taxonomie():
 
     taxonomies = Taxonomies()
@@ -651,6 +665,7 @@ def disable_taxonomie():
 
 
 @Tags.route("/Tags/active_taxonomie")
+@login_required
 def active_taxonomie():
 
     taxonomies = Taxonomies()
@@ -670,6 +685,7 @@ def active_taxonomie():
         return "INCORRECT INPUT"
 
 @Tags.route("/Tags/edit_taxonomie_tag")
+@login_required
 def edit_taxonomie_tag():
 
     taxonomies = Taxonomies()
@@ -712,6 +728,7 @@ def edit_taxonomie_tag():
         return "INCORRECT INPUT"
 
 @Tags.route("/Tags/galaxies")
+@login_required
 def galaxies():
 
     active_galaxies = r_serv_tags.smembers('active_galaxies')
@@ -758,6 +775,7 @@ def galaxies():
 
 
 @Tags.route("/Tags/edit_galaxy")
+@login_required
 def edit_galaxy():
 
     id = request.args.get('galaxy')
@@ -825,6 +843,7 @@ def edit_galaxy():
 
 
 @Tags.route("/Tags/active_galaxy")
+@login_required
 def active_galaxy():
 
     id = request.args.get('galaxy')
@@ -869,6 +888,7 @@ def active_galaxy():
 
 
 @Tags.route("/Tags/disable_galaxy")
+@login_required
 def disable_galaxy():
 
     id = request.args.get('galaxy')
@@ -889,6 +909,7 @@ def disable_galaxy():
 
 
 @Tags.route("/Tags/edit_galaxy_tag")
+@login_required
 def edit_galaxy_tag():
 
     arg1 = request.args.getlist('tag_enabled')
@@ -961,6 +982,7 @@ def edit_galaxy_tag():
         return "INCORRECT INPUT"
 
 @Tags.route("/Tags/tag_galaxy_info")
+@login_required
 def tag_galaxy_info():
 
     galaxy = request.args.get('galaxy')
diff --git a/var/www/modules/hashDecoded/Flask_hashDecoded.py b/var/www/modules/hashDecoded/Flask_hashDecoded.py
index 8a7945d2..db60e0c8 100644
--- a/var/www/modules/hashDecoded/Flask_hashDecoded.py
+++ b/var/www/modules/hashDecoded/Flask_hashDecoded.py
@@ -15,6 +15,7 @@ import zipfile
 
 import requests
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, send_file
+from flask_login import login_required
 
 # ============ VARIABLES ============
 import Flask_config
@@ -97,6 +98,7 @@ def one():
 
 # ============= ROUTES ==============
 @hashDecoded.route("/hashDecoded/all_hash_search", methods=['POST'])
+@login_required
 def all_hash_search():
     date_from = request.form.get('date_from')
     date_to = request.form.get('date_to')
@@ -107,6 +109,7 @@ def all_hash_search():
 
 
 @hashDecoded.route("/hashDecoded/", methods=['GET'])
+@login_required
 def hashDecoded_page():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -224,6 +227,7 @@ def hashDecoded_page():
 
 
 @hashDecoded.route('/hashDecoded/hash_by_type')
+@login_required
 def hash_by_type():
     type = request.args.get('type')
     type = 'text/plain'
@@ -231,12 +235,14 @@ def hash_by_type():
 
 
 @hashDecoded.route('/hashDecoded/hash_hash')
+@login_required
 def hash_hash():
     hash = request.args.get('hash')
     return render_template('hash_hash.html')
 
 
 @hashDecoded.route('/hashDecoded/showHash')
+@login_required
 def showHash():
     hash = request.args.get('hash')
     #hash = 'e02055d3efaad5d656345f6a8b1b6be4fe8cb5ea'
@@ -290,6 +296,7 @@ def showHash():
 
 
 @hashDecoded.route('/hashDecoded/downloadHash')
+@login_required
 def downloadHash():
     hash = request.args.get('hash')
     # sanitize hash
@@ -326,6 +333,7 @@ def downloadHash():
 
 
 @hashDecoded.route('/hashDecoded/hash_by_type_json')
+@login_required
 def hash_by_type_json():
     type = request.args.get('type')
 
@@ -359,6 +367,7 @@ def hash_by_type_json():
 
 
 @hashDecoded.route('/hashDecoded/decoder_type_json')
+@login_required
 def decoder_type_json():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -414,6 +423,7 @@ def decoder_type_json():
 
 
 @hashDecoded.route('/hashDecoded/top5_type_json')
+@login_required
 def top5_type_json():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -472,6 +482,7 @@ def top5_type_json():
 
 
 @hashDecoded.route('/hashDecoded/daily_type_json')
+@login_required
 def daily_type_json():
     date = request.args.get('date')
 
@@ -491,6 +502,7 @@ def daily_type_json():
 
 
 @hashDecoded.route('/hashDecoded/range_type_json')
+@login_required
 def range_type_json():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -547,6 +559,7 @@ def range_type_json():
 
 
 @hashDecoded.route('/hashDecoded/hash_graph_line_json')
+@login_required
 def hash_graph_line_json():
     hash = request.args.get('hash')
     date_from = request.args.get('date_from')
@@ -576,6 +589,7 @@ def hash_graph_line_json():
 
 
 @hashDecoded.route('/hashDecoded/hash_graph_node_json')
+@login_required
 def hash_graph_node_json():
     hash = request.args.get('hash')
 
@@ -643,6 +657,7 @@ def hash_graph_node_json():
 
 
 @hashDecoded.route('/hashDecoded/hash_types')
+@login_required
 def hash_types():
     date_from = 20180701
     date_to = 20180706
@@ -650,6 +665,7 @@ def hash_types():
 
 
 @hashDecoded.route('/hashDecoded/send_file_to_vt_js')
+@login_required
 def send_file_to_vt_js():
     hash = request.args.get('hash')
 
@@ -673,6 +689,7 @@ def send_file_to_vt_js():
 
 
 @hashDecoded.route('/hashDecoded/update_vt_result')
+@login_required
 def update_vt_result():
     hash = request.args.get('hash')
 
diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py
index fd68dc93..76c61667 100644
--- a/var/www/modules/hiddenServices/Flask_hiddenServices.py
+++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py
@@ -12,6 +12,7 @@ import time
 import json
 from pyfaup.faup import Faup
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
+from flask_login import login_required
 
 from Date import Date
 from HiddenServices import HiddenServices
@@ -232,6 +233,7 @@ def delete_auto_crawler(url):
 # ============= ROUTES ==============
 
 @hiddenServices.route("/crawlers/", methods=['GET'])
+@login_required
 def dashboard():
     crawler_metadata_onion = get_crawler_splash_status('onion')
     crawler_metadata_regular = get_crawler_splash_status('regular')
@@ -246,14 +248,17 @@ def dashboard():
                                 statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular)
 
 @hiddenServices.route("/hiddenServices/2", methods=['GET'])
+@login_required
 def hiddenServices_page_test():
     return render_template("Crawler_index.html")
 
 @hiddenServices.route("/crawlers/manual", methods=['GET'])
+@login_required
 def manual():
     return render_template("Crawler_Splash_manual.html")
 
 @hiddenServices.route("/crawlers/crawler_splash_onion", methods=['GET'])
+@login_required
 def crawler_splash_onion():
     type = 'onion'
     last_onions = get_last_domains_crawled(type)
@@ -271,6 +276,7 @@ def crawler_splash_onion():
                             crawler_metadata=crawler_metadata, date_from=date_string, date_to=date_string)
 
 @hiddenServices.route("/crawlers/Crawler_Splash_last_by_type", methods=['GET'])
+@login_required
 def Crawler_Splash_last_by_type():
     type = request.args.get('type')
     # verify user input
@@ -293,6 +299,7 @@ def Crawler_Splash_last_by_type():
                             crawler_metadata=crawler_metadata, date_from=date_string, date_to=date_string)
 
 @hiddenServices.route("/crawlers/blacklisted_domains", methods=['GET'])
+@login_required
 def blacklisted_domains():
     blacklist_domain = request.args.get('blacklist_domain')
     unblacklist_domain = request.args.get('unblacklist_domain')
@@ -327,6 +334,7 @@ def blacklisted_domains():
         return 'Incorrect Type'
 
 @hiddenServices.route("/crawler/blacklist_domain", methods=['GET'])
+@login_required
 def blacklist_domain():
     domain = request.args.get('domain')
     type = request.args.get('type')
@@ -348,6 +356,7 @@ def blacklist_domain():
         return 'Incorrect type'
 
 @hiddenServices.route("/crawler/unblacklist_domain", methods=['GET'])
+@login_required
 def unblacklist_domain():
     domain = request.args.get('domain')
     type = request.args.get('type')
@@ -369,6 +378,7 @@ def unblacklist_domain():
         return 'Incorrect type'
 
 @hiddenServices.route("/crawlers/create_spider_splash", methods=['POST'])
+@login_required
 def create_spider_splash():
     url = request.form.get('url_to_crawl')
     automatic = request.form.get('crawler_type')
@@ -444,6 +454,7 @@ def create_spider_splash():
     return redirect(url_for('hiddenServices.manual'))
 
 @hiddenServices.route("/crawlers/auto_crawler", methods=['GET'])
+@login_required
 def auto_crawler():
     nb_element_to_display = 100
     try:
@@ -495,6 +506,7 @@ def auto_crawler():
                                 auto_crawler_domain_regular_metadata=auto_crawler_domain_regular_metadata)
 
 @hiddenServices.route("/crawlers/remove_auto_crawler", methods=['GET'])
+@login_required
 def remove_auto_crawler():
     url = request.args.get('url')
     page = request.args.get('page')
@@ -504,6 +516,7 @@ def remove_auto_crawler():
     return redirect(url_for('hiddenServices.auto_crawler', page=page))
 
 @hiddenServices.route("/crawlers/crawler_dashboard_json", methods=['GET'])
+@login_required
 def crawler_dashboard_json():
 
     crawler_metadata_onion = get_crawler_splash_status('onion')
@@ -520,6 +533,7 @@ def crawler_dashboard_json():
 
 # # TODO: refractor
 @hiddenServices.route("/hiddenServices/last_crawled_domains_with_stats_json", methods=['GET'])
+@login_required
 def last_crawled_domains_with_stats_json():
     last_onions = r_serv_onion.lrange('last_onion', 0 ,-1)
     list_onion = []
@@ -569,6 +583,7 @@ def last_crawled_domains_with_stats_json():
     return jsonify({'last_onions': list_onion, 'statDomains': statDomains, 'crawler_metadata':crawler_metadata})
 
 @hiddenServices.route("/hiddenServices/get_onions_by_daterange", methods=['POST'])
+@login_required
 def get_onions_by_daterange():
     date_from = request.form.get('date_from')
     date_to = request.form.get('date_to')
@@ -580,6 +595,7 @@ def get_onions_by_daterange():
     return redirect(url_for('hiddenServices.show_domains_by_daterange', date_from=date_from, date_to=date_to, service_type=service_type, domains_up=domains_up, domains_down=domains_down, domains_tags=domains_tags))
 
 @hiddenServices.route("/hiddenServices/show_domains_by_daterange", methods=['GET'])
+@login_required
 def show_domains_by_daterange():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -684,6 +700,7 @@ def show_domains_by_daterange():
                                 domains_tags=domains_tags, type=service_type, bootstrap_label=bootstrap_label)
 
 @hiddenServices.route("/crawlers/show_domain", methods=['GET'])
+@login_required
 def show_domain():
     domain = request.args.get('domain')
     epoch = request.args.get('epoch')
@@ -754,6 +771,7 @@ def show_domain():
                             domain_tags=domain_tags, screenshot=screenshot)
 
 @hiddenServices.route("/hiddenServices/onion_son", methods=['GET'])
+@login_required
 def onion_son():
     onion_domain = request.args.get('onion_domain')
 
@@ -764,6 +782,7 @@ def onion_son():
 
 # ============= JSON ==============
 @hiddenServices.route("/hiddenServices/domain_crawled_7days_json", methods=['GET'])
+@login_required
 def domain_crawled_7days_json():
     type = 'onion'
         ## TODO: # FIXME: 404 error
@@ -782,6 +801,7 @@ def domain_crawled_7days_json():
     return jsonify(json_domain_stats)
 
 @hiddenServices.route('/hiddenServices/domain_crawled_by_type_json')
+@login_required
 def domain_crawled_by_type_json():
     current_date = request.args.get('date')
     type = request.args.get('type')
diff --git a/var/www/modules/rawSkeleton/Flask_rawSkeleton.py b/var/www/modules/rawSkeleton/Flask_rawSkeleton.py
index d17e2b33..fe6e1f66 100644
--- a/var/www/modules/rawSkeleton/Flask_rawSkeleton.py
+++ b/var/www/modules/rawSkeleton/Flask_rawSkeleton.py
@@ -6,6 +6,7 @@
 '''
 import redis
 from flask import Flask, render_template, jsonify, request, Blueprint
+from flask_login import login_required
 
 # ============ VARIABLES ============
 import Flask_config
@@ -22,6 +23,7 @@ def one():
 # ============= ROUTES ==============
 
 @rawSkeleton.route("/rawSkeleton/", methods=['GET'])
+@login_required
 def skeleton_page():
     return render_template("rawSkeleton.html")
 
diff --git a/var/www/modules/search/Flask_search.py b/var/www/modules/search/Flask_search.py
index 7405b1e9..866c0bfc 100644
--- a/var/www/modules/search/Flask_search.py
+++ b/var/www/modules/search/Flask_search.py
@@ -10,6 +10,7 @@ import os
 import datetime
 import flask
 from flask import Flask, render_template, jsonify, request, Blueprint
+from flask_login import login_required
 
 import Paste
 from whoosh import index
@@ -93,6 +94,7 @@ def to_iso_date(timestamp):
 # ============ ROUTES ============
 
 @searches.route("/search", methods=['POST'])
+@login_required
 def search():
     query = request.form['query']
     q = []
@@ -180,6 +182,7 @@ def search():
 
 
 @searches.route("/get_more_search_result", methods=['POST'])
+@login_required
 def get_more_search_result():
     query = request.form['query']
     q = []
diff --git a/var/www/modules/sentiment/Flask_sentiment.py b/var/www/modules/sentiment/Flask_sentiment.py
index 9a86eaa4..14904558 100644
--- a/var/www/modules/sentiment/Flask_sentiment.py
+++ b/var/www/modules/sentiment/Flask_sentiment.py
@@ -10,6 +10,7 @@ import calendar
 from Date import Date
 import flask
 from flask import Flask, render_template, jsonify, request, Blueprint
+from flask_login import login_required
 
 import Paste
 
@@ -39,11 +40,13 @@ def get_date_range(num_day):
 # ============ ROUTES ============
 
 @sentiments.route("/sentiment_analysis_trending/")
+@login_required
 def sentiment_analysis_trending():
     return render_template("sentiment_analysis_trending.html")
 
 
 @sentiments.route("/sentiment_analysis_getplotdata/", methods=['GET'])
+@login_required
 def sentiment_analysis_getplotdata():
     # Get the top providers based on number of pastes
     oneHour = 60*60
@@ -94,12 +97,14 @@ def sentiment_analysis_getplotdata():
 
 
 @sentiments.route("/sentiment_analysis_plot_tool/")
+@login_required
 def sentiment_analysis_plot_tool():
     return render_template("sentiment_analysis_plot_tool.html")
 
 
 
 @sentiments.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET'])
+@login_required
 def sentiment_analysis_plot_tool_getdata():
     getProviders = request.args.get('getProviders')
 
diff --git a/var/www/modules/settings/Flask_settings.py b/var/www/modules/settings/Flask_settings.py
index f8600f58..0563056a 100644
--- a/var/www/modules/settings/Flask_settings.py
+++ b/var/www/modules/settings/Flask_settings.py
@@ -5,6 +5,7 @@
     Flask functions and routes for the settings modules page
 '''
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
+from flask_login import login_required
 
 import json
 import datetime
@@ -74,6 +75,7 @@ def get_update_metadata():
 # ============= ROUTES ==============
 
 @settings.route("/settings/", methods=['GET'])
+@login_required
 def settings_page():
     git_metadata = get_git_metadata()
     current_version = r_serv_db.get('ail:version')
@@ -85,6 +87,7 @@ def settings_page():
 
 
 @settings.route("/settings/get_background_update_stats_json", methods=['GET'])
+@login_required
 def get_background_update_stats_json():
     # handle :end, error
     update_stats = {}
diff --git a/var/www/modules/showpaste/Flask_showpaste.py b/var/www/modules/showpaste/Flask_showpaste.py
index 474280b5..c73d93c8 100644
--- a/var/www/modules/showpaste/Flask_showpaste.py
+++ b/var/www/modules/showpaste/Flask_showpaste.py
@@ -9,6 +9,8 @@ import json
 import os
 import flask
 from flask import Flask, render_template, jsonify, request, Blueprint, make_response, Response, send_from_directory, redirect, url_for
+from flask_login import login_required
+
 import difflib
 import ssdeep
 
@@ -378,16 +380,19 @@ def show_item_min(requested_path , content_range=0):
 # ============ ROUTES ============
 
 @showsavedpastes.route("/showsavedpaste/") #completely shows the paste in a new tab
+@login_required
 def showsavedpaste():
     requested_path = request.args.get('paste', '')
     return showpaste(0, requested_path)
 
 @showsavedpastes.route("/showsaveditem_min/") #completely shows the paste in a new tab
+@login_required
 def showsaveditem_min():
     requested_path = request.args.get('paste', '')
     return show_item_min(requested_path)
 
 @showsavedpastes.route("/showsavedrawpaste/") #shows raw
+@login_required
 def showsavedrawpaste():
     requested_path = request.args.get('paste', '')
     paste = Paste.Paste(requested_path)
@@ -395,6 +400,7 @@ def showsavedrawpaste():
     return Response(content, mimetype='text/plain')
 
 @showsavedpastes.route("/showpreviewpaste/")
+@login_required
 def showpreviewpaste():
     num = request.args.get('num', '')
     requested_path = request.args.get('paste', '')
@@ -402,6 +408,7 @@ def showpreviewpaste():
 
 
 @showsavedpastes.route("/getmoredata/")
+@login_required
 def getmoredata():
     requested_path = request.args.get('paste', '')
     paste = Paste.Paste(requested_path)
@@ -410,6 +417,7 @@ def getmoredata():
     return to_return
 
 @showsavedpastes.route("/showDiff/")
+@login_required
 def showDiff():
     s1 = request.args.get('s1', '')
     s2 = request.args.get('s2', '')
@@ -426,10 +434,12 @@ def showDiff():
     return the_html
 
 @showsavedpastes.route('/screenshot/<path:filename>')
+@login_required
 def screenshot(filename):
     return send_from_directory(SCREENSHOT_FOLDER, filename+'.png', as_attachment=True)
 
 @showsavedpastes.route('/send_file_to_vt/', methods=['POST'])
+@login_required
 def send_file_to_vt():
     b64_path = request.form['b64_path']
     paste = request.form['paste']
diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py
index 1fb83bcb..fd42ec4d 100644
--- a/var/www/modules/terms/Flask_terms.py
+++ b/var/www/modules/terms/Flask_terms.py
@@ -11,6 +11,8 @@ import datetime
 import calendar
 import flask
 from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect
+from flask_login import login_required
+
 import re
 import Paste
 from pprint import pprint
@@ -143,6 +145,7 @@ def save_tag_to_auto_push(list_tag):
 # ============ ROUTES ============
 
 @terms.route("/terms_management/")
+@login_required
 def terms_management():
     per_paste = request.args.get('per_paste')
     if per_paste == "1" or per_paste is None:
@@ -261,6 +264,7 @@ def terms_management():
 
 
 @terms.route("/terms_management_query_paste/")
+@login_required
 def terms_management_query_paste():
     term =  request.args.get('term')
     paste_info = []
@@ -293,6 +297,7 @@ def terms_management_query_paste():
 
 
 @terms.route("/terms_management_query/")
+@login_required
 def terms_management_query():
     TrackedTermsDate_Name = "TrackedTermDate"
     BlackListTermsDate_Name = "BlackListTermDate"
@@ -315,6 +320,7 @@ def terms_management_query():
 
 
 @terms.route("/terms_management_action/", methods=['GET'])
+@login_required
 def terms_management_action():
     today = datetime.datetime.now()
     today = today.replace(microsecond=0)
@@ -440,6 +446,7 @@ def terms_management_action():
         return jsonify(to_return)
 
 @terms.route("/terms_management/delete_terms_tags", methods=['POST'])
+@login_required
 def delete_terms_tags():
     term = request.form.get('term')
     tags_to_delete = request.form.getlist('tags_to_delete')
@@ -452,6 +459,7 @@ def delete_terms_tags():
         return 'None args', 400
 
 @terms.route("/terms_management/delete_terms_email", methods=['GET'])
+@login_required
 def delete_terms_email():
     term =  request.args.get('term')
     email =  request.args.get('email')
@@ -464,6 +472,7 @@ def delete_terms_email():
 
 
 @terms.route("/terms_plot_tool/")
+@login_required
 def terms_plot_tool():
     term =  request.args.get('term')
     if term is not None:
@@ -473,6 +482,7 @@ def terms_plot_tool():
 
 
 @terms.route("/terms_plot_tool_data/")
+@login_required
 def terms_plot_tool_data():
     oneDay = 60*60*24
     range_start =  datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
@@ -503,6 +513,7 @@ def terms_plot_tool_data():
 
 
 @terms.route("/terms_plot_top/")
+@login_required
 def terms_plot_top():
     per_paste = request.args.get('per_paste')
     per_paste = per_paste if per_paste is not None else 1
@@ -510,6 +521,7 @@ def terms_plot_top():
 
 
 @terms.route("/terms_plot_top_data/")
+@login_required
 def terms_plot_top_data():
     oneDay = 60*60*24
     today = datetime.datetime.now()
@@ -556,10 +568,12 @@ def terms_plot_top_data():
 
 
 @terms.route("/credentials_tracker/")
+@login_required
 def credentials_tracker():
     return render_template("credentials_tracker.html")
 
 @terms.route("/credentials_management_query_paste/", methods=['GET', 'POST'])
+@login_required
 def credentials_management_query_paste():
     cred =  request.args.get('cred')
     allPath = request.json['allPath']
@@ -583,6 +597,7 @@ def credentials_management_query_paste():
     return jsonify(paste_info)
 
 @terms.route("/credentials_management_action/", methods=['GET'])
+@login_required
 def cred_management_action():
 
     supplied =  request.args.get('term')
diff --git a/var/www/modules/trendingcharts/Flask_trendingcharts.py b/var/www/modules/trendingcharts/Flask_trendingcharts.py
index ad2e5b76..bad6a353 100644
--- a/var/www/modules/trendingcharts/Flask_trendingcharts.py
+++ b/var/www/modules/trendingcharts/Flask_trendingcharts.py
@@ -9,6 +9,7 @@ import datetime
 from Date import Date
 import flask
 from flask import Flask, render_template, jsonify, request, Blueprint
+from flask_login import login_required
 
 # ============ VARIABLES ============
 import Flask_config
@@ -36,6 +37,7 @@ def get_date_range(num_day):
 # ============ ROUTES ============
 
 @trendings.route("/_progressionCharts", methods=['GET'])
+@login_required
 def progressionCharts():
     attribute_name = request.args.get('attributeName')
     trending_name = request.args.get('trendingName')
@@ -61,18 +63,21 @@ def progressionCharts():
         return jsonify(keyw_value)
 
 @trendings.route("/wordstrending/")
+@login_required
 def wordstrending():
     default_display = cfg.get("Flask", "default_display")
     return render_template("Wordstrending.html", default_display = default_display)
 
 
 @trendings.route("/protocolstrending/")
+@login_required
 def protocolstrending():
     default_display = cfg.get("Flask", "default_display")
     return render_template("Protocolstrending.html", default_display = default_display)
 
 
 @trendings.route("/trending/")
+@login_required
 def trending():
     default_display = cfg.get("Flask", "default_display")
     return render_template("Trending.html", default_display = default_display)
diff --git a/var/www/modules/trendingmodules/Flask_trendingmodules.py b/var/www/modules/trendingmodules/Flask_trendingmodules.py
index aeec0eb9..a53066b9 100644
--- a/var/www/modules/trendingmodules/Flask_trendingmodules.py
+++ b/var/www/modules/trendingmodules/Flask_trendingmodules.py
@@ -9,6 +9,7 @@ import datetime
 from Date import Date
 import flask
 from flask import Flask, render_template, jsonify, request, Blueprint
+from flask_login import login_required
 
 # ============ VARIABLES ============
 import Flask_config
@@ -49,6 +50,7 @@ def get_date_range(num_day):
 # ============ ROUTES ============
 
 @trendingmodules.route("/_moduleCharts", methods=['GET'])
+@login_required
 def modulesCharts():
     keyword_name = request.args.get('keywordName')
     module_name = request.args.get('moduleName')
@@ -75,6 +77,7 @@ def modulesCharts():
 
 
 @trendingmodules.route("/_providersChart", methods=['GET'])
+@login_required
 def providersChart():
     keyword_name = request.args.get('keywordName')
     module_name = request.args.get('moduleName')
@@ -121,6 +124,7 @@ def providersChart():
 
 
 @trendingmodules.route("/moduletrending/")
+@login_required
 def moduletrending():
     return render_template("Moduletrending.html")
 
diff --git a/var/www/templates/login.html b/var/www/templates/login.html
new file mode 100644
index 00000000..2e413435
--- /dev/null
+++ b/var/www/templates/login.html
@@ -0,0 +1,81 @@
+<!DOCTYPE html>
+
+<html>
+<head>
+	 <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+	<title>AIL-Framework</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
+	<!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+
+	<!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+
+
+	<style>
+	html,
+	body {
+	height: 100%;
+	}
+
+	body {
+	display: -ms-flexbox;
+	display: flex;
+	-ms-flex-align: center;
+	align-items: center;
+	padding-top: 40px;
+	padding-bottom: 40px;
+	background-color: #f5f5f5;
+	}
+
+	.form-signin {
+	width: 100%;
+	max-width: 330px;
+	padding: 15px;
+	margin: auto;
+	}
+	.form-signin .checkbox {
+	font-weight: 400;
+	}
+	.form-signin .form-control {
+	position: relative;
+	box-sizing: border-box;
+	height: auto;
+	padding: 10px;
+	font-size: 16px;
+	}
+	.form-signin .form-control:focus {
+	z-index: 2;
+	}
+	.form-signin input[type="email"] {
+	margin-bottom: -1px;
+	border-bottom-right-radius: 0;
+	border-bottom-left-radius: 0;
+	}
+	.form-signin input[type="password"] {
+	margin-bottom: 10px;
+	border-top-left-radius: 0;
+	border-top-right-radius: 0;
+	}
+	</style>
+
+</head>
+
+<body class="text-center">
+
+
+				<form class="form-signin" action="{{ url_for('login')}}"  method="post">
+		      <img class="mb-4" src="{{ url_for('static', filename='image/AIL.png')}}" width="300">
+		      <h1 class="h3 mb-3 text-secondary">Please sign in</h1>
+		      <label for="inputEmail" class="sr-only">Email address</label>
+		      <input type="email" id="inputEmail" name="username" class="form-control" placeholder="Email address" required autofocus>
+		      <label for="inputPassword" class="sr-only">Password</label>
+		      <input type="password" id="inputPassword" name="password" class="form-control" placeholder="Password" required>
+		      <button class="btn btn-lg btn-primary btn-block" type="submit">Sign in</button>
+					<input type="text" name="next_page" hidden>{{next_page}}</input>
+		    </form>
+
+
+</body>

From 64ff94ce5f1b5730ab5aa30b5efaeee60788d1b8 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Fri, 3 May 2019 16:52:05 +0200
Subject: [PATCH 02/22] chg: [user_management] create + check user password

---
 OVERVIEW.md                                   | 12 +++
 bin/packages/User.py                          | 47 ++++++++--
 pip3_packages_requirement.txt                 |  1 +
 var/www/Flask_server.py                       | 85 ++++++++++++++++---
 var/www/modules/Decorator.py                  | 17 ++++
 var/www/modules/dashboard/Flask_dashboard.py  |  5 ++
 .../hiddenServices/Flask_hiddenServices.py    |  4 +
 7 files changed, 151 insertions(+), 20 deletions(-)
 create mode 100644 var/www/modules/Decorator.py

diff --git a/OVERVIEW.md b/OVERVIEW.md
index 5d6dbc99..ecf16bf3 100644
--- a/OVERVIEW.md
+++ b/OVERVIEW.md
@@ -52,6 +52,18 @@ Redis and ARDB overview
 | ail:current_background_script | **name of the background script currently executed** |
 | ail:current_background_script_stat | **progress in % of the background script** |
 
+##### User Management:
+| Key | Field | Value |
+| ------ | ------ | ------ |
+| user:all | **user id** | **password hash**  |
+
+| Key | Value |
+| ------ | ------ |
+| user:request_password_change | **user id** |
+| user:admin | **user id** |
+| | |
+| user_role:**role** | **user id** |
+
 ## DB2 - TermFreq:
 
 ##### Set:
diff --git a/bin/packages/User.py b/bin/packages/User.py
index b46069f1..3dfe31a5 100755
--- a/bin/packages/User.py
+++ b/bin/packages/User.py
@@ -1,36 +1,65 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 
+import os
 import redis
+import bcrypt
+import configparser
 
 from flask_login import UserMixin
 
 class User(UserMixin):
 
     def __init__(self, id):
-        self.id = 'abcdef'
+
+        configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+        if not os.path.exists(configfile):
+            raise Exception('Unable to find the configuration file. \
+                            Did you set environment variables? \
+                            Or activate the virtualenv.')
+
+        cfg = configparser.ConfigParser()
+        cfg.read(configfile)
+
+        self.r_serv_db = redis.StrictRedis(
+            host=cfg.get("ARDB_DB", "host"),
+            port=cfg.getint("ARDB_DB", "port"),
+            db=cfg.getint("ARDB_DB", "db"),
+            decode_responses=True)
+
+        if self.r_serv_db.hexists('user:all', id):
+            self.id = id
+        else:
+            self.id = "__anonymous__"
 
     # return True or False
     #def is_authenticated():
 
-    # return True or False
-    #def is_active():
-
     # return True or False
     #def is_anonymous():
 
     @classmethod
     def get(self_class, id):
-        print(id)
         return self_class(id)
 
     def check_password(self, password):
+        password = password.encode()
         print(self.id)
-        if password=='admin':
-            print('password ok')
+        hashed_password = self.r_serv_db.hget('user:all', self.id).encode()
+        if bcrypt.checkpw(password, hashed_password):
+            print('password correct')
             return True
         else:
             return False
 
-    def set_password(self):
-        return True
+    def request_password_change(self):
+        if self.r_serv_db.sismember('user:request_password_change', self.id):
+            return True
+        else:
+            return False
+
+    def is_in_role(self, role):
+        if self.r_serv_db.sismember('user_role:{}'.format(role), self.id):
+            return True
+        else:
+            return False
diff --git a/pip3_packages_requirement.txt b/pip3_packages_requirement.txt
index 4f9d1f87..4f5372bc 100644
--- a/pip3_packages_requirement.txt
+++ b/pip3_packages_requirement.txt
@@ -47,6 +47,7 @@ texttable
 
 flask
 flask-login
+bcrypt
 
 #DomainClassifier
 DomainClassifier
diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py
index 907f1fd4..a689089f 100755
--- a/var/www/Flask_server.py
+++ b/var/www/Flask_server.py
@@ -11,6 +11,8 @@ import calendar
 from flask import Flask, render_template, jsonify, request, Request, session, redirect, url_for
 from flask_login import LoginManager, current_user, login_user, logout_user, login_required
 
+import bcrypt
+
 import flask
 import importlib
 import os
@@ -28,6 +30,41 @@ from pytaxonomies import Taxonomies
 # Import config
 import Flask_config
 
+def flask_init():
+    int_user_management()
+
+def int_user_management():
+    # # TODO: check for admin account
+    # check if an account exists
+    if not r_serv_db.hexists('user:all'):
+        password = 'admin@admin.test'
+        create_user_db('admin', password, default=True)
+
+def hashing_password(bytes_password):
+    hashed = bcrypt.hashpw(bytes_password, bcrypt.gensalt())
+    return hashed
+
+def verify_password(id, bytes_password):
+    hashed_password = r_serv_db.hget('user:all', id)
+    if bcrypt.checkpw(password, hashed):
+        return True
+    else:
+        return False
+
+def create_user_db(username_id , password, default=False):
+    ## TODO: validate username
+    ## TODO: validate password
+
+    if username_id == '__anonymous__':
+        ## TODO: return 500
+        return 'ERROR'
+
+    password = password.encode()
+    password_hash = hashing_password(password)
+    r_serv_db.hset('user:all', username_id, password_hash)
+    if default:
+        r_serv_db.set('user:request_password_change', username_id)
+
 # CONFIG #
 cfg = Flask_config.cfg
 baseUrl = cfg.get("Flask", "baseurl")
@@ -35,6 +72,20 @@ baseUrl = baseUrl.replace('/', '')
 if baseUrl != '':
     baseUrl = '/'+baseUrl
 
+# ========= REDIS =========#
+r_serv_db = redis.StrictRedis(
+    host=cfg.get("ARDB_DB", "host"),
+    port=cfg.getint("ARDB_DB", "port"),
+    db=cfg.getint("ARDB_DB", "db"),
+    decode_responses=True)
+r_serv_tags = redis.StrictRedis(
+    host=cfg.get("ARDB_Tags", "host"),
+    port=cfg.getint("ARDB_Tags", "port"),
+    db=cfg.getint("ARDB_Tags", "db"),
+    decode_responses=True)
+
+# =========       =========#
+
 Flask_config.app = Flask(__name__, static_url_path=baseUrl+'/static/')
 app = Flask_config.app
 app.config['MAX_CONTENT_LENGTH'] = 900 * 1024 * 1024
@@ -152,6 +203,8 @@ def login():
             #print(user.is_authenticated)
             if user and user.check_password(password):
                 login_user(user) ## TODO: use remember me ?
+                #print(user.request_password_change())
+                print(user.is_active)
                 return redirect(url_for('dashboard.index'))
             else:
                 return 'incorrect password'
@@ -167,7 +220,27 @@ def login():
 @login_required
 def logout():
     logout_user()
-    return redirect(url_for('dashboard.index'))
+    return redirect(url_for('login'))
+
+@app.route('/create_user')
+@login_required
+def create_user():
+    username = request.form.get('username')
+    password = request.form.get('password')
+    #role = request.form.get('role') ## TODO: create role
+
+    ## TODO: validate username
+    ## TODO: validate password
+
+    username = 'admin@admin.test'
+    password = 'admin'
+
+    if r_serv_db.hexists('user:all', username):
+        return 'this id is not available'
+
+    create_user_db(username, password)
+
+    return 'True'
 
 
 @app.route('/searchbox/')
@@ -176,11 +249,6 @@ def searchbox():
 
 
 # ========== INITIAL taxonomies ============
-r_serv_tags = redis.StrictRedis(
-    host=cfg.get("ARDB_Tags", "host"),
-    port=cfg.getint("ARDB_Tags", "port"),
-    db=cfg.getint("ARDB_Tags", "db"),
-    decode_responses=True)
 # add default ail taxonomies
 r_serv_tags.sadd('active_taxonomies', 'infoleak')
 r_serv_tags.sadd('active_taxonomies', 'gdpr')
@@ -195,11 +263,6 @@ for tag in taxonomies.get('fpf').machinetags():
     r_serv_tags.sadd('active_tag_fpf', tag)
 
 # ========== INITIAL tags auto export ============
-r_serv_db = redis.StrictRedis(
-    host=cfg.get("ARDB_DB", "host"),
-    port=cfg.getint("ARDB_DB", "port"),
-    db=cfg.getint("ARDB_DB", "db"),
-    decode_responses=True)
 infoleak_tags = taxonomies.get('infoleak').machinetags()
 infoleak_automatic_tags = []
 for tag in taxonomies.get('infoleak').machinetags():
diff --git a/var/www/modules/Decorator.py b/var/www/modules/Decorator.py
new file mode 100644
index 00000000..c1ab7755
--- /dev/null
+++ b/var/www/modules/Decorator.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+from functools import wraps
+from flask_login import LoginManager, current_user, login_user, logout_user, login_required
+
+from flask import request
+
+def login_required(role="ANY"):
+    @wraps(role)
+    def decorated_view(*args, **kwargs):
+        if not current_user.is_authenticated:
+            return current_app.login_manager.unauthorized()
+        elif (not current_user.is_in_role(role)) and (role != "ANY"):
+            return login_manager.unauthorized()
+        return func(*args, **kwargs)
+    return decorated_view
diff --git a/var/www/modules/dashboard/Flask_dashboard.py b/var/www/modules/dashboard/Flask_dashboard.py
index 7ebf6747..0bbcccac 100644
--- a/var/www/modules/dashboard/Flask_dashboard.py
+++ b/var/www/modules/dashboard/Flask_dashboard.py
@@ -13,6 +13,7 @@ import flask
 from Date import Date
 
 from flask import Flask, render_template, jsonify, request, Blueprint, url_for
+from flask_login import login_required
 
 # ============ VARIABLES ============
 import Flask_config
@@ -109,10 +110,12 @@ def datetime_from_utc_to_local(utc_str):
 # ============ ROUTES ============
 
 @dashboard.route("/_logs")
+@login_required
 def logs():
     return flask.Response(event_stream(), mimetype="text/event-stream")
 
 @dashboard.route("/_get_last_logs_json")
+@login_required
 def get_last_logs_json():
     date = datetime.datetime.now().strftime("%Y%m%d")
 
@@ -154,11 +157,13 @@ def get_last_logs_json():
 
 
 @dashboard.route("/_stuff", methods=['GET'])
+@login_required
 def stuff():
     return jsonify(row1=get_queues(r_serv))
 
 
 @dashboard.route("/")
+@login_required
 def index():
     default_minute = cfg.get("Flask", "minute_processed_paste")
     threshold_stucked_module = cfg.getint("Module_ModuleInformation", "threshold_stucked_module")
diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py
index 76c61667..42b36006 100644
--- a/var/www/modules/hiddenServices/Flask_hiddenServices.py
+++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py
@@ -17,6 +17,8 @@ from flask_login import login_required
 from Date import Date
 from HiddenServices import HiddenServices
 
+from Decorator import login_required
+
 # ============ VARIABLES ============
 import Flask_config
 
@@ -35,6 +37,7 @@ list_types=['onion', 'regular']
 dic_type_name={'onion':'Onion', 'regular':'Website'}
 
 # ============ FUNCTIONS ============
+
 def one():
     return 1
 
@@ -233,6 +236,7 @@ def delete_auto_crawler(url):
 # ============= ROUTES ==============
 
 @hiddenServices.route("/crawlers/", methods=['GET'])
+#@login_required(role="ADMIN")
 @login_required
 def dashboard():
     crawler_metadata_onion = get_crawler_splash_status('onion')

From 1dac8177fe5a0d47a7c74c46a9e0b3483d76c67d Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Mon, 6 May 2019 16:58:36 +0200
Subject: [PATCH 03/22] chg: [user_management] add user role_management

---
 var/www/Flask_server.py                       |  4 +++
 var/www/modules/Decorator.py                  | 17 -----------
 var/www/modules/Role_Manager.py               | 30 +++++++++++++++++++
 var/www/modules/Tags/Flask_Tags.py            |  6 +++-
 .../hiddenServices/Flask_hiddenServices.py    |  3 --
 5 files changed, 39 insertions(+), 21 deletions(-)
 delete mode 100644 var/www/modules/Decorator.py
 create mode 100644 var/www/modules/Role_Manager.py

diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py
index a689089f..2e8c8375 100755
--- a/var/www/Flask_server.py
+++ b/var/www/Flask_server.py
@@ -216,6 +216,10 @@ def login():
         print(next_page)
         return render_template("login.html", next_page=next_page)
 
+@app.route('/role', methods=['POST', 'GET'])
+def role():
+    return 'ERROR role'
+
 @app.route('/logout')
 @login_required
 def logout():
diff --git a/var/www/modules/Decorator.py b/var/www/modules/Decorator.py
deleted file mode 100644
index c1ab7755..00000000
--- a/var/www/modules/Decorator.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env python3
-# -*-coding:UTF-8 -*
-
-from functools import wraps
-from flask_login import LoginManager, current_user, login_user, logout_user, login_required
-
-from flask import request
-
-def login_required(role="ANY"):
-    @wraps(role)
-    def decorated_view(*args, **kwargs):
-        if not current_user.is_authenticated:
-            return current_app.login_manager.unauthorized()
-        elif (not current_user.is_in_role(role)) and (role != "ANY"):
-            return login_manager.unauthorized()
-        return func(*args, **kwargs)
-    return decorated_view
diff --git a/var/www/modules/Role_Manager.py b/var/www/modules/Role_Manager.py
new file mode 100644
index 00000000..19314003
--- /dev/null
+++ b/var/www/modules/Role_Manager.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+from functools import wraps
+from flask_login import LoginManager, current_user, login_user, logout_user, login_required
+
+from flask import request, current_app
+
+login_manager = LoginManager()
+login_manager.login_view = 'role'
+
+def login_admin(func):
+    @wraps(func)
+    def decorated_view(*args, **kwargs):
+        if not current_user.is_authenticated:
+            return login_manager.unauthorized()
+        elif (not current_user.is_in_role('admin')):
+            return login_manager.unauthorized()
+        return func(*args, **kwargs)
+    return decorated_view
+
+def login_analyst(func):
+    @wraps(func)
+    def decorated_view(*args, **kwargs):
+        if not current_user.is_authenticated:
+            return login_manager.unauthorized()
+        elif (not current_user.is_in_role('analyst')):
+            return login_manager.unauthorized()
+        return func(*args, **kwargs)
+    return decorated_view
diff --git a/var/www/modules/Tags/Flask_Tags.py b/var/www/modules/Tags/Flask_Tags.py
index ec329b30..307f6ed3 100644
--- a/var/www/modules/Tags/Flask_Tags.py
+++ b/var/www/modules/Tags/Flask_Tags.py
@@ -5,7 +5,9 @@
     Flask functions and routes for the trending modules page
 '''
 import redis
-from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
+from flask import Flask, render_template, jsonify, request, Blueprint, current_app, redirect, url_for
+
+from Role_Manager import login_admin, login_analyst
 from flask_login import login_required
 
 import json
@@ -220,6 +222,7 @@ def update_tag_last_seen(tag, tag_first_seen, tag_last_seen):
 
 @Tags.route("/tags/", methods=['GET'])
 @login_required
+@login_admin
 def Tags_page():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -469,6 +472,7 @@ def remove_tag():
 
 @Tags.route("/Tags/confirm_tag")
 @login_required
+@login_analyst
 def confirm_tag():
 
     #TODO verify input
diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py
index 42b36006..c7bf2ea2 100644
--- a/var/www/modules/hiddenServices/Flask_hiddenServices.py
+++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py
@@ -17,8 +17,6 @@ from flask_login import login_required
 from Date import Date
 from HiddenServices import HiddenServices
 
-from Decorator import login_required
-
 # ============ VARIABLES ============
 import Flask_config
 
@@ -236,7 +234,6 @@ def delete_auto_crawler(url):
 # ============= ROUTES ==============
 
 @hiddenServices.route("/crawlers/", methods=['GET'])
-#@login_required(role="ADMIN")
 @login_required
 def dashboard():
     crawler_metadata_onion = get_crawler_splash_status('onion')

From f2ca241e4f4d9b19630b1b29093fe6e470ae3437 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Wed, 8 May 2019 14:58:41 +0200
Subject: [PATCH 04/22] chg: [restapi] add rest api authentification  + create
 default user

---
 OVERVIEW.md                                   |   4 +
 var/www/create_default_user.py                |  53 +++++
 var/www/modules/Tags/Flask_Tags.py            |   2 +-
 var/www/modules/restApi/Flask_restApi.py      | 109 +++++++++
 .../restApi/templates/api_default.html        | 222 ++++++++++++++++++
 .../modules/restApi/templates/header_api.html |   1 +
 6 files changed, 390 insertions(+), 1 deletion(-)
 create mode 100755 var/www/create_default_user.py
 create mode 100644 var/www/modules/restApi/Flask_restApi.py
 create mode 100644 var/www/modules/restApi/templates/api_default.html
 create mode 100644 var/www/modules/restApi/templates/header_api.html

diff --git a/OVERVIEW.md b/OVERVIEW.md
index ecf16bf3..38488a74 100644
--- a/OVERVIEW.md
+++ b/OVERVIEW.md
@@ -56,6 +56,10 @@ Redis and ARDB overview
 | Key | Field | Value |
 | ------ | ------ | ------ |
 | user:all | **user id** | **password hash**  |
+| | | |
+| user:tokens | **token** | **user id** |
+| | | |
+| user_metadata:**user id** | **user token** | **token**  |
 
 | Key | Value |
 | ------ | ------ |
diff --git a/var/www/create_default_user.py b/var/www/create_default_user.py
new file mode 100755
index 00000000..c249ec0c
--- /dev/null
+++ b/var/www/create_default_user.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import redis
+import configparser
+
+import bcrypt
+import secrets
+
+# Import config
+sys.path.append('./modules/')
+
+def hashing_password(bytes_password):
+    hashed = bcrypt.hashpw(bytes_password, bcrypt.gensalt())
+    return hashed
+
+def create_user_db(username_id , password, default=False):
+    password = password.encode()
+    password_hash = hashing_password(password)
+    r_serv_db.hset('user:all', username_id, password_hash)
+    if default:
+        r_serv_db.set('user:request_password_change', username_id)
+
+
+if __name__ == "__main__":
+    configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+    if not os.path.exists(configfile):
+        raise Exception('Unable to find the configuration file. \
+                        Did you set environment variables? \
+                        Or activate the virtualenv.')
+
+    cfg = configparser.ConfigParser()
+    cfg.read(configfile)
+
+    r_serv_db = redis.StrictRedis(
+        host=cfg.get("ARDB_DB", "host"),
+        port=cfg.getint("ARDB_DB", "port"),
+        db=cfg.getint("ARDB_DB", "db"),
+        decode_responses=True)
+
+    username = 'admin@admin.test'
+    # # TODO: create random password
+    password = 'admin'
+    create_user_db(username, password, default=True)
+
+    # create user token
+    token = secrets.token_urlsafe(41)
+    r_serv_db.hset('user:tokens', token, username)
+
+    print('new user created: {}'.format(username))
+    print('password: {}'.format(password))
diff --git a/var/www/modules/Tags/Flask_Tags.py b/var/www/modules/Tags/Flask_Tags.py
index 307f6ed3..f68491e6 100644
--- a/var/www/modules/Tags/Flask_Tags.py
+++ b/var/www/modules/Tags/Flask_Tags.py
@@ -5,7 +5,7 @@
     Flask functions and routes for the trending modules page
 '''
 import redis
-from flask import Flask, render_template, jsonify, request, Blueprint, current_app, redirect, url_for
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
 
 from Role_Manager import login_admin, login_analyst
 from flask_login import login_required
diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py
new file mode 100644
index 00000000..82c5ba45
--- /dev/null
+++ b/var/www/modules/restApi/Flask_restApi.py
@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+'''
+    Flask functions and routes for the rest api
+'''
+
+import os
+import re
+import sys
+import json
+import redis
+import datetime
+
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
+from flask_login import login_required
+
+from functools import wraps
+
+# ============ VARIABLES ============
+import Flask_config
+
+app = Flask_config.app
+cfg = Flask_config.cfg
+baseUrl = Flask_config.baseUrl
+r_cache = Flask_config.r_cache
+r_serv_db = Flask_config.r_serv_db
+r_serv_onion = Flask_config.r_serv_onion
+r_serv_metadata = Flask_config.r_serv_metadata
+
+restApi = Blueprint('restApi', __name__, template_folder='templates')
+
+# ============ AUTH FUNCTIONS ============
+
+def check_token_format(strg, search=re.compile(r'[^a-zA-Z0-9_-]').search):
+    return not bool(search(strg))
+
+def verify_token(token):
+    if len(token) != 55:
+        return False
+
+    if not check_token_format(token):
+        return False
+
+    if r_serv_db.hexists('user:tokens', token):
+        return True
+    else:
+        return False
+
+# ============ DECORATOR ============
+
+def token_required(funct):
+    @wraps(funct)
+    def api_token(*args, **kwargs):
+        data = authErrors()
+        if data:
+            return Response(json.dumps(data[0], indent=2, sort_keys=True), mimetype='application/json'), data[1]
+        else:
+            return funct(*args, **kwargs)
+    return api_token
+
+def get_auth_from_header():
+    token = request.headers.get('Authorization').replace(' ', '') # remove space
+    return token
+
+def authErrors():
+    # Check auth
+    if not request.headers.get('Authorization'):
+        return ({'status': 'error', 'reason': 'Authentication needed'}, 401)
+    token = get_auth_from_header()
+    data = None
+    # verify token format
+
+    try:
+        authenticated = False
+        if verify_token(token):
+            authenticated = True
+
+        if not authenticated:
+            data = ({'status': 'error', 'reason': 'Authentication failed'}, 401)
+    except Exception as e:
+        print(e)
+        data = ({'status': 'error', 'reason': 'Malformed Authentication String'}, 400)
+    if data:
+        return data
+    else:
+        return None
+
+# ============ FUNCTIONS ============
+
+def one():
+    return 1
+
+# ============= ROUTES ==============
+
+@restApi.route("/api", methods=['GET'])
+@login_required
+def api():
+    return 'api doc'
+
+@restApi.route("api/items", methods=['POST'])
+@token_required
+def items():
+    item = request.args.get('id')
+
+    return Response(json.dumps({'test': 2}), mimetype='application/json')
+
+# ========= REGISTRATION =========
+app.register_blueprint(restApi, url_prefix=baseUrl)
diff --git a/var/www/modules/restApi/templates/api_default.html b/var/www/modules/restApi/templates/api_default.html
new file mode 100644
index 00000000..91d300af
--- /dev/null
+++ b/var/www/modules/restApi/templates/api_default.html
@@ -0,0 +1,222 @@
+<!DOCTYPE html>
+
+<html>
+<head>
+	<title>AIL-Framework</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
+	<!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+
+	<!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+
+
+</head>
+
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'crawler/menu_sidebar.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+				<div class="row">
+					<div class="col-xl-6">
+
+						<div class="card mt-1 mb-1">
+						  <div class="card-header text-white bg-dark">
+								<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=onion"><i class="fas fa-user-secret"></i> Onions Crawlers</a></h5>
+								<div class="row">
+									<div class="col-6">
+										<span class="badge badge-success" id="stat_onion_domain_up">{{ statDomains_onion['domains_up'] }}</span> UP
+										<span class="badge badge-danger ml-md-3" id="stat_onion_domain_down">{{ statDomains_onion['domains_down'] }}</span> DOWN
+									</div>
+									<div class="col-6">
+										<span class="badge badge-success" id="stat_onion_total">{{ statDomains_onion['total'] }}</span> Crawled
+										<span class="badge badge-warning ml-md-3" id="stat_onion_queue">{{ statDomains_onion['domains_queue'] }}</span> Queue
+									</div>
+								</div>
+							</div>
+						  <div class="card-body px-0 py-0 ">
+								<table class="table">
+									<tbody id="tbody_crawler_onion_info">
+										{% for crawler in crawler_metadata_onion %}
+											<tr>
+						      			<td>
+													<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
+												</td>
+												<td>
+													{{crawler['crawling_domain']}}
+												</td>
+												<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
+													{{crawler['status_info']}}
+												</td>
+											</tr>
+										{% endfor %}
+									</tbody>
+								</table>
+							</div>
+						</div>
+
+					</div>
+					<div class="col-xl-6">
+						<div class="card mt-1 mb-1">
+							<div class="card-header text-white bg-dark">
+								<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=regular"><i class="fab fa-html5"></i> Regular Crawlers</a></h5>
+								<div class="row">
+									<div class="col-6">
+										<span class="badge badge-success" id="stat_regular_domain_up">{{ statDomains_regular['domains_up'] }}</span> UP
+										<span class="badge badge-danger ml-md-3" id="stat_regular_domain_down">{{ statDomains_regular['domains_down'] }}</span> DOWN
+									</div>
+									<div class="col-6">
+										<span class="badge badge-success" id="stat_regular_total">{{ statDomains_regular['total'] }}</span> Crawled
+										<span class="badge badge-warning ml-md-3" id="stat_regular_queue">{{ statDomains_regular['domains_queue'] }}</span> Queue
+									</div>
+								</div>
+							</div>
+							<div class="card-body px-0 py-0 ">
+								<table class="table">
+									<tbody id="tbody_crawler_regular_info">
+										{% for crawler in crawler_metadata_regular %}
+											<tr>
+												<td>
+													<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
+												</td>
+												<td>
+													{{crawler['crawling_domain']}}
+												</td>
+												<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
+													{{crawler['status_info']}}
+												</td>
+											</tr>
+										{% endfor %}
+									</tbody>
+								</table>
+							</div>
+						</div>
+					</div>
+				</div>
+
+			</div>
+		</div>
+	</div>
+
+
+</body>
+
+<script>
+var to_refresh = false
+$(document).ready(function(){
+	$("#page-Crawler").addClass("active");
+	$("#nav_dashboard").addClass("active");
+	$( window ).focus(function() {
+			to_refresh = true
+			refresh_crawler_status();
+	});
+	$( window ).blur(function() {
+			to_refresh = false
+	});
+
+	to_refresh = true
+	refresh_crawler_status();
+});
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+
+function refresh_crawler_status(){
+
+	$.getJSON("{{ url_for('hiddenServices.crawler_dashboard_json') }}",
+		function(data) {
+
+			$('#stat_onion_domain_up').text(data.statDomains_onion['domains_up']);
+			$('#stat_onion_domain_down').text(data.statDomains_onion['domains_down']);
+			$('#stat_onion_total').text(data.statDomains_onion['total']);
+			$('#stat_onion_queue').text(data.statDomains_onion['domains_queue']);
+
+			$('#stat_regular_domain_up').text(data.statDomains_regular['domains_up']);
+			$('#stat_regular_domain_down').text(data.statDomains_regular['domains_down']);
+			$('#stat_regular_total').text(data.statDomains_regular['total']);
+			$('#stat_regular_queue').text(data.statDomains_regular['domains_queue']);
+
+			if(data.crawler_metadata_onion.length!=0){
+				$("#tbody_crawler_onion_info").empty();
+				var tableRef = document.getElementById('tbody_crawler_onion_info');
+				for (var i = 0; i < data.crawler_metadata_onion.length; i++) {
+						var crawler = data.crawler_metadata_onion[i];
+						var newRow = tableRef.insertRow(tableRef.rows.length);
+						var text_color;
+						var icon;
+						if(crawler['status']){
+							text_color = 'Green';
+							icon = 'check';
+						} else {
+							text_color = 'Red';
+							icon = 'times';
+						}
+
+				    var newCell  = newRow.insertCell(0);
+						newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
+
+						newCell  = newRow.insertCell(1);
+						newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
+
+						newCell  = newRow.insertCell(2);
+						newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
+
+						//$("#panel_crawler").show();
+				}
+			}
+			if(data.crawler_metadata_regular.length!=0){
+				$("#tbody_crawler_regular_info").empty();
+				var tableRef = document.getElementById('tbody_crawler_regular_info');
+				for (var i = 0; i < data.crawler_metadata_regular.length; i++) {
+						var crawler = data.crawler_metadata_regular[i];
+						var newRow = tableRef.insertRow(tableRef.rows.length);
+						var text_color;
+						var icon;
+						if(crawler['status']){
+							text_color = 'Green';
+							icon = 'check';
+						} else {
+							text_color = 'Red';
+							icon = 'times';
+						}
+
+				    var newCell  = newRow.insertCell(0);
+						newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
+
+						newCell  = newRow.insertCell(1);
+						newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
+
+						newCell  = newRow.insertCell(2);
+						newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
+
+						//$("#panel_crawler").show();
+				}
+			}
+		}
+	);
+
+	if (to_refresh) {
+    setTimeout("refresh_crawler_status()", 10000);
+	}
+}
+</script>
diff --git a/var/www/modules/restApi/templates/header_api.html b/var/www/modules/restApi/templates/header_api.html
new file mode 100644
index 00000000..34e20319
--- /dev/null
+++ b/var/www/modules/restApi/templates/header_api.html
@@ -0,0 +1 @@
+<li id='page-hiddenServices'><a href="{{ url_for('hiddenServices.dashboard') }}"><i class="fa fa-user-secret"></i> hidden Services </a></li>

From 3fe9d14e9f6e86e0ee61de13629e8bf699cc67f7 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Thu, 6 Jun 2019 21:27:13 +0200
Subject: [PATCH 05/22] chg: [user_management] create default admin user (temp
 passwd save in AIL_HOME) + change password UI + logout UI + create random
 password

---
 OVERVIEW.md                            | 10 ++-
 bin/packages/User.py                   |  2 +-
 var/www/Flask_server.py                | 87 +++++++++++++++-------
 var/www/create_default_user.py         | 27 +++++--
 var/www/templates/change_password.html | 99 ++++++++++++++++++++++++++
 var/www/templates/login.html           |  3 +-
 var/www/templates/nav_bar.html         |  3 +
 7 files changed, 192 insertions(+), 39 deletions(-)
 create mode 100644 var/www/templates/change_password.html

diff --git a/OVERVIEW.md b/OVERVIEW.md
index b84f6447..b9baf5ab 100644
--- a/OVERVIEW.md
+++ b/OVERVIEW.md
@@ -53,21 +53,27 @@ Redis and ARDB overview
 | ail:current_background_script_stat | **progress in % of the background script** |
 
 ##### User Management:
-| Key | Field | Value |
+| Hset Key | Field | Value |
 | ------ | ------ | ------ |
 | user:all | **user id** | **password hash**  |
 | | | |
 | user:tokens | **token** | **user id** |
 | | | |
 | user_metadata:**user id** | **user token** | **token**  |
+|                           | change_passwd  | **boolean** |
 
-| Key | Value |
+| Set Key | Value |
 | ------ | ------ |
 | user:request_password_change | **user id** |
 | user:admin | **user id** |
 | | |
 | user_role:**role** | **user id** |
 
+
+| Zrank Key | Field | Value |
+| ------ | ------ | ------ |
+| ail:all_role | **role** | **int, role priority (1=admin)** |
+
 ## DB2 - TermFreq:
 
 ##### Set:
diff --git a/bin/packages/User.py b/bin/packages/User.py
index 3dfe31a5..5be7021d 100755
--- a/bin/packages/User.py
+++ b/bin/packages/User.py
@@ -53,7 +53,7 @@ class User(UserMixin):
             return False
 
     def request_password_change(self):
-        if self.r_serv_db.sismember('user:request_password_change', self.id):
+        if self.r_serv_db.hget('user_metadata:{}'.format(self.id), 'change_passwd') == 'True':
             return True
         else:
             return False
diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py
index e004cb5a..a66c5229 100755
--- a/var/www/Flask_server.py
+++ b/var/www/Flask_server.py
@@ -16,6 +16,7 @@ import bcrypt
 import flask
 import importlib
 import os
+import re
 from os.path import join
 import sys
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
@@ -31,14 +32,14 @@ from pytaxonomies import Taxonomies
 import Flask_config
 
 def flask_init():
-    int_user_management()
-
-def int_user_management():
-    # # TODO: check for admin account
     # check if an account exists
-    if not r_serv_db.hexists('user:all'):
-        password = 'admin@admin.test'
-        create_user_db('admin', password, default=True)
+    if not r_serv_db.exists('user:all'):
+        password = secrets.token_urlsafe()
+        create_user_db('admin@admin.test', password, role='admin',default=True)
+    # add default roles
+    if not r_serv_db.exists('ail:all_role'):
+        r_serv_db.zadd('ail:all_role', 1, 'admin')
+        r_serv_db.zadd('ail:all_role', 2, 'analyst')
 
 def hashing_password(bytes_password):
     hashed = bcrypt.hashpw(bytes_password, bcrypt.gensalt())
@@ -51,19 +52,31 @@ def verify_password(id, bytes_password):
     else:
         return False
 
-def create_user_db(username_id , password, default=False):
-    ## TODO: validate username
-    ## TODO: validate password
+def check_password_strength(password):
+    result = regex_password.match(password)
+    if result:
+        return True
+    else:
+        return False
 
-    if username_id == '__anonymous__':
-        ## TODO: return 500
-        return 'ERROR'
 
+def create_user_db(username_id , password, default=False, role=None, update=False):
     password = password.encode()
     password_hash = hashing_password(password)
     r_serv_db.hset('user:all', username_id, password_hash)
-    if default:
-        r_serv_db.set('user:request_password_change', username_id)
+    if update:
+        r_serv_db.hdel('user_metadata:{}'.format(username_id), 'change_passwd')
+        if username_id=='admin@admin.test':
+            os.remove(default_passwd_file)
+    else:
+        if default:
+            r_serv_db.hset('user_metadata:{}'.format(username_id), 'change_passwd', True)
+        if role:
+            if role in get_all_role():
+                r_serv_db.sadd('user_role:{}'.format(role), username_id)
+
+def get_all_role():
+    return r_serv_db.zrange('ail:all_role', 0 , -1)
 
 # CONFIG #
 cfg = Flask_config.cfg
@@ -72,6 +85,11 @@ baseUrl = baseUrl.replace('/', '')
 if baseUrl != '':
     baseUrl = '/'+baseUrl
 
+default_passwd_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD')
+
+regex_password = r'^(?=(.*\d){2})(?=.*[a-z])(?=.*[A-Z]).{10,}$'
+regex_password = re.compile(regex_password)
+
 # ========= REDIS =========#
 r_serv_db = redis.StrictRedis(
     host=cfg.get("ARDB_DB", "host"),
@@ -163,6 +181,8 @@ modified_header = modified_header.replace('<!--insert here-->', '\n'.join(to_add
 with open('templates/header.html', 'w') as f:
     f.write(modified_header)
 
+flask_init()
+
 
 # ========= JINJA2 FUNCTIONS ========
 def list_len(s):
@@ -187,30 +207,43 @@ def login():
     if request.method == 'POST':
         username = request.form.get('username')
         password = request.form.get('password')
-        next_page = request.form.get('next_page')
-
-        print(username)
-        print(password)
+        #next_page = request.form.get('next_page')
 
         if username is not None:
             user = User.get(username)
-            #print(user.is_anonymous)
-            #print('auth') # TODO: overwrite
-            #print(user.is_authenticated)
             if user and user.check_password(password):
                 login_user(user) ## TODO: use remember me ?
-                #print(user.request_password_change())
                 print(user.is_active)
-                return redirect(url_for('dashboard.index'))
+                if user.request_password_change():
+                    return redirect(url_for('change_password'))
+                else:
+                    return redirect(url_for('dashboard.index'))
             else:
                 return 'incorrect password'
 
         return 'none'
 
     else:
-        next_page = request.args.get('next')
-        print(next_page)
-        return render_template("login.html", next_page=next_page)
+        #next_page = request.args.get('next')
+        return render_template("login.html")
+
+@app.route('/change_password', methods=['POST', 'GET'])
+@login_required
+def change_password():
+    password1 = request.form.get('password1')
+    password2 = request.form.get('password2')
+
+    # # TODO: display errors message
+
+    if current_user.is_authenticated and password1!=None and password1==password2:
+        if check_password_strength(password1):
+            user_id = current_user.get_id()
+            create_user_db(user_id , password1, update=True)
+            return redirect(url_for('dashboard.index'))
+        else:
+            return render_template("change_password.html")
+    else:
+        return render_template("change_password.html")
 
 @app.route('/role', methods=['POST', 'GET'])
 def role():
diff --git a/var/www/create_default_user.py b/var/www/create_default_user.py
index c249ec0c..07dca0d6 100755
--- a/var/www/create_default_user.py
+++ b/var/www/create_default_user.py
@@ -12,17 +12,25 @@ import secrets
 # Import config
 sys.path.append('./modules/')
 
+def get_all_role():
+    return r_serv_db.zrange('ail:all_role', 0 , -1)
+
 def hashing_password(bytes_password):
     hashed = bcrypt.hashpw(bytes_password, bcrypt.gensalt())
     return hashed
 
-def create_user_db(username_id , password, default=False):
+def create_user_db(username_id , password, default=False, role=None, update=False):
     password = password.encode()
     password_hash = hashing_password(password)
     r_serv_db.hset('user:all', username_id, password_hash)
-    if default:
-        r_serv_db.set('user:request_password_change', username_id)
-
+    if update:
+        r_serv_db.hdel('user_metadata:{}'.format(username_id), 'change_passwd')
+    else:
+        if default:
+            r_serv_db.hset('user_metadata:{}'.format(username_id), 'change_passwd', True)
+        if role:
+            if role in get_all_role():
+                r_serv_db.sadd('user_role:{}'.format(role), username_id)
 
 if __name__ == "__main__":
     configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
@@ -41,13 +49,18 @@ if __name__ == "__main__":
         decode_responses=True)
 
     username = 'admin@admin.test'
-    # # TODO: create random password
-    password = 'admin'
-    create_user_db(username, password, default=True)
+    password = secrets.token_urlsafe()
+    create_user_db(username, password, role='admin', default=True)
 
     # create user token
     token = secrets.token_urlsafe(41)
     r_serv_db.hset('user:tokens', token, username)
 
+    default_passwd_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD')
+    to_write_str = '# Password Generated by default\n# This file is deleted after the first login\n#\nemail=admin@admin.test\npassword='
+    to_write_str = to_write_str + password + '\nAPI_Key=' + token
+    with open(default_passwd_file, 'w') as f:
+        f.write(to_write_str)
+
     print('new user created: {}'.format(username))
     print('password: {}'.format(password))
diff --git a/var/www/templates/change_password.html b/var/www/templates/change_password.html
new file mode 100644
index 00000000..2107b7a1
--- /dev/null
+++ b/var/www/templates/change_password.html
@@ -0,0 +1,99 @@
+<!DOCTYPE html>
+
+<html>
+<head>
+	 <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+	<title>AIL-Framework</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
+	<!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+
+	<!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+
+
+	<style>
+	html,
+	body {
+	height: 100%;
+	}
+
+	body {
+	display: -ms-flexbox;
+	display: flex;
+	-ms-flex-align: center;
+	align-items: center;
+	padding-top: 40px;
+	padding-bottom: 40px;
+	background-color: #f5f5f5;
+	}
+
+	.form-signin {
+	width: 100%;
+	max-width: 330px;
+	padding: 15px;
+	margin: auto;
+	}
+	.form-signin .checkbox {
+	font-weight: 400;
+	}
+	.form-signin .form-control {
+	position: relative;
+	box-sizing: border-box;
+	height: auto;
+	padding: 10px;
+	font-size: 16px;
+	}
+	.form-signin .form-control:focus {
+	z-index: 2;
+	}
+	.form-signin input[type="password"] {
+	margin-bottom: 10px;
+	border-top-left-radius: 0;
+	border-top-right-radius: 0;
+	}
+	</style>
+
+</head>
+
+<body class="text-center">
+
+
+				<form class="form-signin" action="{{ url_for('change_password')}}" autocomplete="off" method="post">
+		      <img class="mb-4" src="{{ url_for('static', filename='image/logo-small.png')}}" width="300">
+		      <h1 class="h3 mb-3 text-secondary">Change Password</h1>
+		      <label for="inputPassword1" class="sr-only">Password</label>
+		      <input type="password" id="inputPassword1" name="password1" class="form-control" placeholder="Password" autocomplete="new-password" required autofocus>
+		      <label for="inputPassword2" class="sr-only">Confirm Password</label>
+		      <input type="password" id="inputPassword2" name="password2" class="form-control" placeholder="Confirm Password"  value="" autocomplete="new-password" required>
+		      <button class="btn btn-lg btn-primary btn-block" type="submit">Submit</button>
+
+					<br>
+					<br>
+					<br>
+					<h5 class="h3 mb-3 text-secondary">Password Requirements</h5>
+					<ul class="list-group">
+					 <li class="list-group-item d-flex justify-content-between align-items-center">
+						 Minimal length
+						 <span class="badge badge-primary badge-pill">10</span>
+					 </li>
+					 <li class="list-group-item d-flex justify-content-between align-items-center">
+						 Upper characters: A-Z
+						 <span class="badge badge-primary badge-pill">1</span>
+					 </li>
+					 <li class="list-group-item d-flex justify-content-between align-items-center">
+						 Lower characters: a-z
+						 <span class="badge badge-primary badge-pill">1</span>
+					 </li>
+					 <li class="list-group-item d-flex justify-content-between align-items-center">
+						 Digits: 0-9
+						 <span class="badge badge-primary badge-pill">2</span>
+					 </li>
+					</ul>
+
+		    </form>
+
+
+</body>
diff --git a/var/www/templates/login.html b/var/www/templates/login.html
index 2e413435..b846aaa9 100644
--- a/var/www/templates/login.html
+++ b/var/www/templates/login.html
@@ -67,14 +67,13 @@
 
 
 				<form class="form-signin" action="{{ url_for('login')}}"  method="post">
-		      <img class="mb-4" src="{{ url_for('static', filename='image/AIL.png')}}" width="300">
+		      <img class="mb-4" src="{{ url_for('static', filename='image/logo-small.png')}}" width="300">
 		      <h1 class="h3 mb-3 text-secondary">Please sign in</h1>
 		      <label for="inputEmail" class="sr-only">Email address</label>
 		      <input type="email" id="inputEmail" name="username" class="form-control" placeholder="Email address" required autofocus>
 		      <label for="inputPassword" class="sr-only">Password</label>
 		      <input type="password" id="inputPassword" name="password" class="form-control" placeholder="Password" required>
 		      <button class="btn btn-lg btn-primary btn-block" type="submit">Sign in</button>
-					<input type="text" name="next_page" hidden>{{next_page}}</input>
 		    </form>
 
 
diff --git a/var/www/templates/nav_bar.html b/var/www/templates/nav_bar.html
index c878da4c..77d595b3 100644
--- a/var/www/templates/nav_bar.html
+++ b/var/www/templates/nav_bar.html
@@ -33,6 +33,9 @@
       <li class="nav-item mr-3">
         <a class="nav-link" id="page-options" href="{{ url_for('settings.settings_page') }}" aria-disabled="true"><i class="fas fa-cog"></i> Server Management</a>
       </li>
+      <li class="nav-item mr-3">
+        <a class="nav-link" id="page-options" href="{{ url_for('logout') }}" aria-disabled="true"><i class="fas fa-sign-out-alt"></i> Log Out</a>
+      </li>
     </ul>
 
     <form class="form-inline my-2 my-lg-0 ml-auto justify-content-center">

From 9c2d290580e08a1e0f63ec5c6b25c9d816484619 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Fri, 7 Jun 2019 17:14:11 +0200
Subject: [PATCH 06/22] chg: [user_management UI] edit my_profile + renew api
 tokens

---
 OVERVIEW.md                                   |  2 +-
 var/www/create_default_user.py                |  2 +
 var/www/modules/settings/Flask_settings.py    | 32 +++++++-
 .../settings/templates/edit_profile.html      | 82 +++++++++++++++++++
 .../settings/templates/settings_index.html    |  3 +-
 var/www/templates/settings/menu_sidebar.html  | 50 ++++++++++-
 6 files changed, 165 insertions(+), 6 deletions(-)
 create mode 100644 var/www/modules/settings/templates/edit_profile.html

diff --git a/OVERVIEW.md b/OVERVIEW.md
index b9baf5ab..97991b57 100644
--- a/OVERVIEW.md
+++ b/OVERVIEW.md
@@ -59,7 +59,7 @@ Redis and ARDB overview
 | | | |
 | user:tokens | **token** | **user id** |
 | | | |
-| user_metadata:**user id** | **user token** | **token**  |
+| user_metadata:**user id** | token | **token**  |
 |                           | change_passwd  | **boolean** |
 
 | Set Key | Value |
diff --git a/var/www/create_default_user.py b/var/www/create_default_user.py
index 07dca0d6..1dcc9aed 100755
--- a/var/www/create_default_user.py
+++ b/var/www/create_default_user.py
@@ -28,6 +28,7 @@ def create_user_db(username_id , password, default=False, role=None, update=Fals
     else:
         if default:
             r_serv_db.hset('user_metadata:{}'.format(username_id), 'change_passwd', True)
+            r_serv_db.hset('user_metadata:{}'.format(username_id), 'role', role)
         if role:
             if role in get_all_role():
                 r_serv_db.sadd('user_role:{}'.format(role), username_id)
@@ -55,6 +56,7 @@ if __name__ == "__main__":
     # create user token
     token = secrets.token_urlsafe(41)
     r_serv_db.hset('user:tokens', token, username)
+    r_serv_db.hset('user_metadata:{}'.format(username), 'token', token)
 
     default_passwd_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD')
     to_write_str = '# Password Generated by default\n# This file is deleted after the first login\n#\nemail=admin@admin.test\npassword='
diff --git a/var/www/modules/settings/Flask_settings.py b/var/www/modules/settings/Flask_settings.py
index 0563056a..b6254d98 100644
--- a/var/www/modules/settings/Flask_settings.py
+++ b/var/www/modules/settings/Flask_settings.py
@@ -5,9 +5,10 @@
     Flask functions and routes for the settings modules page
 '''
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
-from flask_login import login_required
+from flask_login import login_required, current_user
 
 import json
+import secrets
 import datetime
 
 import git_status
@@ -35,6 +36,14 @@ def one():
 #def get_v1.5_update_tags_backgroud_status():
 #    return '38%'
 
+def generate_new_token(user_id):
+    # create user token
+    current_token = r_serv_db.hget('user_metadata:{}'.format(user_id), 'token')
+    r_serv_db.hdel('user:tokens', current_token)
+    token = secrets.token_urlsafe(41)
+    r_serv_db.hset('user:tokens', token, user_id)
+    r_serv_db.hset('user_metadata:{}'.format(user_id), 'token', token)
+
 def get_git_metadata():
     dict_git = {}
     dict_git['current_branch'] = git_status.get_current_branch()
@@ -72,6 +81,14 @@ def get_update_metadata():
         dict_update['current_background_script_stat'] = r_serv_db.get('ail:current_background_script_stat')
 
     return dict_update
+
+def get_user_metadata(user_id):
+    user_metadata = {}
+    user_metadata['email'] = user_id
+    user_metadata['role'] = r_serv_db.hget('user_metadata:{}'.format(user_id), 'role')
+    user_metadata['api_key'] = r_serv_db.hget('user_metadata:{}'.format(user_id), 'token')
+    return user_metadata
+
 # ============= ROUTES ==============
 
 @settings.route("/settings/", methods=['GET'])
@@ -81,10 +98,21 @@ def settings_page():
     current_version = r_serv_db.get('ail:version')
     update_metadata = get_update_metadata()
 
-
     return render_template("settings_index.html", git_metadata=git_metadata,
                             current_version=current_version)
 
+@settings.route("/settings/edit_profile", methods=['GET'])
+@login_required
+def edit_profile():
+    user_metadata = get_user_metadata(current_user.get_id())
+    return render_template("edit_profile.html", user_metadata=user_metadata)
+
+@settings.route("/settings/new_token", methods=['GET'])
+@login_required
+def new_token():
+    generate_new_token(current_user.get_id())
+    return redirect(url_for('settings.edit_profile'))
+
 
 @settings.route("/settings/get_background_update_stats_json", methods=['GET'])
 @login_required
diff --git a/var/www/modules/settings/templates/edit_profile.html b/var/www/modules/settings/templates/edit_profile.html
new file mode 100644
index 00000000..f0b8d476
--- /dev/null
+++ b/var/www/modules/settings/templates/edit_profile.html
@@ -0,0 +1,82 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Server Management - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+</head>
+<body>
+
+  {% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'settings/menu_sidebar.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+				<div class="card mb-3 mt-1">
+					<div class="card-header text-white bg-dark pb-1">
+						<h5 class="card-title">AIL-framework Status :</h5>
+					</div>
+					<div class="card-body">
+
+            <div class="row">
+              <div class="col-xl-6">
+                <div class="card text-center border-secondary">
+                  <div class="card-body px-1 py-0">
+                    <table class="table table-sm">
+                      <tbody>
+                        <tr>
+                          <td>Email</td>
+                          <td>{{user_metadata['email']}}</td>
+                        </tr>
+                        <tr>
+                          <td>Role</td>
+                          <td>{{user_metadata['role']}}</td>
+                        </tr>
+                        <tr>
+                          <td>API Key</td>
+                          <td>
+                            {{user_metadata['api_key']}}
+                            <a class="ml-3" href="{{url_for('settings.new_token')}}"><i class="fa fa-random"></i></a>
+                          </td>
+                        </tr>
+                      </tbody>
+                    </table>
+                  </div>
+                </div>
+              </div>
+            </div>
+
+					</div>
+				</div>
+
+    	</div>
+		</div>
+	</div>
+
+</body>
+
+<script>
+$(document).ready(function(){
+  $("#nav_edit_profile").addClass("active");
+	$("#nav_my_profile").removeClass("text-muted");
+} );
+</script>
+
+</html>
diff --git a/var/www/modules/settings/templates/settings_index.html b/var/www/modules/settings/templates/settings_index.html
index 5b1e055a..5fc21dc5 100644
--- a/var/www/modules/settings/templates/settings_index.html
+++ b/var/www/modules/settings/templates/settings_index.html
@@ -142,7 +142,8 @@
 
 <script>
 $(document).ready(function(){
-	$("#page-options").addClass("active");
+  $("#nav_server_status").addClass("active");
+	$("#nav_server").removeClass("text-muted");
 } );
 
 function toggle_sidebar(){
diff --git a/var/www/templates/settings/menu_sidebar.html b/var/www/templates/settings/menu_sidebar.html
index c1b58bf3..558a901f 100644
--- a/var/www/templates/settings/menu_sidebar.html
+++ b/var/www/templates/settings/menu_sidebar.html
@@ -5,9 +5,55 @@
       	  <span>Toggle Sidebar</span>
       	</button>
 
-      	<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
-      		<h5 class="d-flex text-muted w-100">
+      	<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2">
+      		<h5 class="d-flex text-muted w-100" id="nav_server">
       		  <span>Diagnostic</span>
       		</h5>
+          <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
+      	    <li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('hashDecoded.hashDecoded_page')}}" id="nav_server_status">
+      	        <i class="fas fa-tools"></i>
+      	        <span>Server Status</span>
+      	      </a>
+      	    </li>
+          </ul>
+      	</nav>
+        <nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_profile">
+          <h5 class="d-flex text-muted w-100" id="nav_my_profile">
+            <span>My Profile</span>
+          </h5>
+          <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
+      	    <li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('settings.edit_profile')}}" id="nav_edit_profile">
+      	        <i class="fas fa-user-edit"></i>
+      	        <span>My Profile</span>
+      	      </a>
+      	    </li>
+      	    <li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('change_password')}}" id="nav_dashboard">
+      	        <i class="fas fa-key"></i>
+      	        <span>Change Password</span>
+      	      </a>
+      	    </li>
+          </ul>
+        </nav>
+        <nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_users">
+      		<h5 class="d-flex text-muted w-100">
+      		  <span>User Management</span>
+      		</h5>
+          <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
+      	    <li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('hashDecoded.hashDecoded_page')}}" id="nav_dashboard">
+      	        <i class="fas fa-user-plus"></i>
+      	        <span>Create User</span>
+      	      </a>
+      	    </li>
+            <li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('hashDecoded.hashDecoded_page')}}" id="nav_dashboard">
+      	        <i class="fas fa-users"></i>
+      	        <span>Users List</span>
+      	      </a>
+      	    </li>
+          </ul>
       	</nav>
       </div>

From 5b58872b156672e1bfd02ff48cd3b66f0a1f06d6 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Tue, 11 Jun 2019 17:37:20 +0200
Subject: [PATCH 07/22] chg: [user_management UI] add admin section: edit +
 create users

---
 var/www/modules/Flask_config.py               |   4 +
 var/www/modules/Role_Manager.py               | 107 ++++++++++++++++
 var/www/modules/settings/Flask_settings.py    | 113 +++++++++++++++-
 .../settings/templates/create_user.html       | 121 ++++++++++++++++++
 .../settings/templates/edit_profile.html      |   4 +-
 .../settings/templates/users_list.html        | 108 ++++++++++++++++
 var/www/templates/settings/menu_sidebar.html  |   6 +-
 7 files changed, 456 insertions(+), 7 deletions(-)
 create mode 100644 var/www/modules/settings/templates/create_user.html
 create mode 100644 var/www/modules/settings/templates/users_list.html

diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py
index 899a26b5..2219824a 100644
--- a/var/www/modules/Flask_config.py
+++ b/var/www/modules/Flask_config.py
@@ -7,6 +7,7 @@
 import configparser
 import redis
 import os
+import re
 import sys
 
 # FLASK #
@@ -175,6 +176,9 @@ max_dashboard_logs = int(cfg.get("Flask", "max_dashboard_logs"))
 
 crawler_enabled = cfg.getboolean("Crawler", "activate_crawler")
 
+regex_password = r'^(?=(.*\d){2})(?=.*[a-z])(?=.*[A-Z]).{10,}$'
+regex_password = re.compile(regex_password)
+
 # VT
 try:
     from virusTotalKEYS import vt_key
diff --git a/var/www/modules/Role_Manager.py b/var/www/modules/Role_Manager.py
index 19314003..85440d57 100644
--- a/var/www/modules/Role_Manager.py
+++ b/var/www/modules/Role_Manager.py
@@ -1,6 +1,12 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 
+import os
+import redis
+import bcrypt
+import secrets
+import configparser
+
 from functools import wraps
 from flask_login import LoginManager, current_user, login_user, logout_user, login_required
 
@@ -9,6 +15,22 @@ from flask import request, current_app
 login_manager = LoginManager()
 login_manager.login_view = 'role'
 
+# CONFIG #
+configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+if not os.path.exists(configfile):
+    raise Exception('Unable to find the configuration file. \
+                    Did you set environment variables? \
+                    Or activate the virtualenv.')
+
+cfg = configparser.ConfigParser()
+cfg.read(configfile)
+
+r_serv_db = redis.StrictRedis(
+    host=cfg.get("ARDB_DB", "host"),
+    port=cfg.getint("ARDB_DB", "port"),
+    db=cfg.getint("ARDB_DB", "db"),
+    decode_responses=True)
+
 def login_admin(func):
     @wraps(func)
     def decorated_view(*args, **kwargs):
@@ -28,3 +50,88 @@ def login_analyst(func):
             return login_manager.unauthorized()
         return func(*args, **kwargs)
     return decorated_view
+
+
+
+###############################################################
+###############################################################
+###############################################################
+
+
+
+def create_user_db(username_id , password, default=False, role=None, update=False):
+    password = password.encode()
+    password_hash = hashing_password(password)
+
+    # create user token
+    token = secrets.token_urlsafe(41)
+    r_serv_db.hset('user:tokens', token, username_id)
+    r_serv_db.hset('user_metadata:{}'.format(username_id), 'token', token)
+
+    if update:
+        r_serv_db.hdel('user_metadata:{}'.format(username_id), 'change_passwd')
+    else:
+        if default:
+            r_serv_db.hset('user_metadata:{}'.format(username_id), 'change_passwd', True)
+        if role:
+            if role in get_all_role():
+                for role_to_add in get_all_user_role(role):
+                    r_serv_db.sadd('user_role:{}'.format(role_to_add), username_id)
+                r_serv_db.hset('user_metadata:{}'.format(username_id), 'role', role)
+
+    r_serv_db.hset('user:all', username_id, password_hash)
+
+def edit_user_db(user_id, role, password=None):
+    if password:
+        password_hash = hashing_password(password.encode())
+        r_serv_db.hset('user:all', user_id, password_hash)
+
+    current_role = r_serv_db.hget('user_metadata:{}'.format(user_id), 'role')
+    if role != current_role:
+        request_level = get_role_level(role)
+        current_role = get_role_level(current_role)
+
+        if current_role < request_level:
+            role_to_remove = get_user_role_by_range(current_role -1, request_level - 2)
+            print('to remove')
+            print(role_to_remove)
+            for role_id in role_to_remove:
+                r_serv_db.srem('user_role:{}'.format(role_id), user_id)
+            r_serv_db.hset('user_metadata:{}'.format(user_id), 'role', role)
+        else:
+            role_to_add = get_user_role_by_range(request_level -1, current_role)
+            print('to add')
+            print(role_to_add)
+            for role_id in role_to_add:
+                r_serv_db.sadd('user_role:{}'.format(role_id), user_id)
+            r_serv_db.hset('user_metadata:{}'.format(user_id), 'role', role)
+
+def delete_user_db(user_id):
+    if r_serv_db.exists('user_metadata:{}'.format(user_id)):
+        print('r')
+        role_to_remove =get_all_role()
+        for role_id in role_to_remove:
+            r_serv_db.srem('user_role:{}'.format(role_id), user_id)
+        user_token = r_serv_db.hget('user_metadata:{}'.format(user_id), 'token')
+        r_serv_db.hdel('user:tokens', user_token)
+        r_serv_db.delete('user_metadata:{}'.format(user_id))
+        r_serv_db.hdel('user:all', user_id)
+
+def hashing_password(bytes_password):
+    hashed = bcrypt.hashpw(bytes_password, bcrypt.gensalt())
+    return hashed
+
+def get_all_role():
+    return r_serv_db.zrange('ail:all_role', 0, -1)
+
+def get_role_level(role):
+    return int(r_serv_db.zscore('ail:all_role', role))
+
+def get_all_user_role(user_role):
+    current_role_val = get_role_level(user_role)
+    return r_serv_db.zrange('ail:all_role', current_role_val -1, -1)
+
+def get_user_role_by_range(inf, sup):
+    print(inf)
+    print(sup)
+    return r_serv_db.zrange('ail:all_role', inf, sup)
diff --git a/var/www/modules/settings/Flask_settings.py b/var/www/modules/settings/Flask_settings.py
index b6254d98..683d5b01 100644
--- a/var/www/modules/settings/Flask_settings.py
+++ b/var/www/modules/settings/Flask_settings.py
@@ -7,6 +7,8 @@
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
 from flask_login import login_required, current_user
 
+from Role_Manager import login_admin, login_analyst, create_user_db, edit_user_db, delete_user_db
+
 import json
 import secrets
 import datetime
@@ -24,6 +26,7 @@ max_preview_char = Flask_config.max_preview_char
 max_preview_modal = Flask_config.max_preview_modal
 REPO_ORIGIN = Flask_config.REPO_ORIGIN
 dict_update_description = Flask_config.dict_update_description
+regex_password = Flask_config.regex_password
 
 settings = Blueprint('settings', __name__, template_folder='templates')
 
@@ -33,8 +36,12 @@ settings = Blueprint('settings', __name__, template_folder='templates')
 def one():
     return 1
 
-#def get_v1.5_update_tags_backgroud_status():
-#    return '38%'
+def check_password_strength(password):
+    result = regex_password.match(password)
+    if result:
+        return True
+    else:
+        return False
 
 def generate_new_token(user_id):
     # create user token
@@ -89,6 +96,18 @@ def get_user_metadata(user_id):
     user_metadata['api_key'] = r_serv_db.hget('user_metadata:{}'.format(user_id), 'token')
     return user_metadata
 
+def get_users_metadata(list_users):
+    users = []
+    for user in list_users:
+        users.append(get_user_metadata(user))
+    return users
+
+def get_all_users():
+    return r_serv_db.hkeys('user:all')
+
+def get_all_roles():
+    return r_serv_db.zrange('ail:all_role', 0, -1)
+
 # ============= ROUTES ==============
 
 @settings.route("/settings/", methods=['GET'])
@@ -113,6 +132,96 @@ def new_token():
     generate_new_token(current_user.get_id())
     return redirect(url_for('settings.edit_profile'))
 
+@settings.route("/settings/new_token_user", methods=['GET'])
+@login_required
+def new_token_user():
+    user_id = request.args.get('user_id')
+    if r_serv_db.exists('user_metadata:{}'.format(user_id)):
+        generate_new_token(user_id)
+    return redirect(url_for('settings.users_list'))
+
+@settings.route("/settings/create_user", methods=['GET'])
+@login_required
+def create_user():
+    user_id = request.args.get('user_id')
+    role = None
+    if r_serv_db.exists('user_metadata:{}'.format(user_id)):
+        role = r_serv_db.hget('user_metadata:{}'.format(user_id), 'role')
+    else:
+        user_id = None
+    all_roles = get_all_roles()
+    return render_template("create_user.html", all_roles=all_roles, user_id=user_id, user_role=role)
+
+@settings.route("/settings/create_user_post", methods=['POST'])
+@login_required
+def create_user_post():
+    email = request.form.get('username')
+    role = request.form.get('user_role')
+    password1 = request.form.get('password1')
+    password2 = request.form.get('password2')
+
+    all_roles = get_all_roles()
+
+    if email and role:
+        if role in all_roles:
+            # password set
+            if password1 and password2:
+                if password1==password2:
+                    if check_password_strength(password1):
+                        password = password1
+                    else:
+                        return render_template("create_user.html", all_roles=all_roles)
+                else:
+                    return render_template("create_user.html", all_roles=all_roles)
+            # generate password
+            else:
+                password = secrets.token_urlsafe()
+
+            if current_user.is_in_role('admin'):
+                # edit user
+                if r_serv_db.exists('user_metadata:{}'.format(email)):
+                    if password1 and password2:
+                        edit_user_db(email, password=password, role=role)
+                        return redirect(url_for('settings.users_list', new_user=email, new_user_password=password, new_user_edited=True))
+                    else:
+                        edit_user_db(email, role=role)
+                        return redirect(url_for('settings.users_list', new_user=email, new_user_password='Password not changed', new_user_edited=True))
+                # create user
+                else:
+                    create_user_db(email, password, default=True, role=role)
+                    return redirect(url_for('settings.users_list', new_user=email, new_user_password=password, new_user_edited=False))
+
+        else:
+            return render_template("create_user.html", all_roles=all_roles)
+    else:
+        return render_template("create_user.html", all_roles=all_roles)
+
+@settings.route("/settings/users_list", methods=['GET'])
+@login_required
+def users_list():
+    all_users = get_users_metadata(get_all_users())
+    new_user = request.args.get('new_user')
+    new_user_dict = {}
+    if new_user:
+        new_user_dict['email'] = new_user
+        new_user_dict['edited'] = request.args.get('new_user_edited')
+        new_user_dict['password'] = request.args.get('new_user_password')
+    print(new_user)
+    return render_template("users_list.html", all_users=all_users, new_user=new_user_dict)
+
+@settings.route("/settings/edit_user", methods=['GET'])
+@login_required
+def edit_user():
+    user_id = request.args.get('user_id')
+    return redirect(url_for('settings.create_user', user_id=user_id))
+
+@settings.route("/settings/delete_user", methods=['GET'])
+@login_required
+def delete_user():
+    user_id = request.args.get('user_id')
+    delete_user_db(user_id)
+    return redirect(url_for('settings.users_list'))
+
 
 @settings.route("/settings/get_background_update_stats_json", methods=['GET'])
 @login_required
diff --git a/var/www/modules/settings/templates/create_user.html b/var/www/modules/settings/templates/create_user.html
new file mode 100644
index 00000000..2a24094d
--- /dev/null
+++ b/var/www/modules/settings/templates/create_user.html
@@ -0,0 +1,121 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Server Management - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+</head>
+<body>
+
+  {% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'settings/menu_sidebar.html' %}
+
+      <div class="col-12 col-xl-10" id="core_content">
+
+        <form class="form-signin" action="{{ url_for('settings.create_user_post')}}" autocomplete="off" method="post">
+
+          <h1 class="h3 mt-1 mb-3 text-center text-secondary">Create User</h1>
+          <label for="inputEmail" class="sr-only">Email address</label>
+          <input type="email" id="inputEmail" name="username" class="form-control" placeholder="Email address" autocomplete="off" required {% if user_id %}value="{{user_id}}"{% else %}{% endif %}>
+
+          <label  class="mt-3" for="role_selector">User Role</label>
+          <select class="custom-select" id="role_selector" name="user_role">
+            {% for role in all_roles %}
+              {% if role ==  user_role %}
+                <option value="{{role}}" selected>{{role}}</option>
+              {% else %}
+                <option value="{{role}}">{{role}}</option>
+              {% endif %}
+            {% endfor %}
+          </select>
+
+          <div class="custom-control custom-switch mt-4 mb-3">
+  				  <input type="checkbox" class="custom-control-input" id="set_manual_password" value="" onclick="toggle_password_fields();">
+  				  <label class="custom-control-label" for="set_manual_password">Set Password</label>
+				  </div>
+
+          <div id="password-section">
+    	      <h1 class="h3 mb-3 text-center text-secondary">Create Password</h1>
+    	      <label for="inputPassword1" class="sr-only">Password</label>
+    	      <input type="password" id="inputPassword1" name="password1" class="form-control" placeholder="Password" autocomplete="new-password">
+    	      <label for="inputPassword2" class="sr-only">Confirm Password</label>
+    	      <input type="password" id="inputPassword2" name="password2" class="form-control" placeholder="Confirm Password"  value="" autocomplete="new-password">
+
+          </div>
+          <button class="btn btn-lg btn-primary btn-block mt-3" type="submit">Submit</button>
+
+          <div id="password-section-info">
+    				<br>
+    				<br>
+    				<br>
+    				<h5 class="h3 mb-3 text-center text-secondary">Password Requirements</h5>
+    				<ul class="list-group">
+    				 <li class="list-group-item d-flex justify-content-between align-items-center">
+    					 Minimal length
+    					 <span class="badge badge-primary badge-pill">10</span>
+    				 </li>
+    				 <li class="list-group-item d-flex justify-content-between align-items-center">
+    					 Upper characters: A-Z
+    					 <span class="badge badge-primary badge-pill">1</span>
+    				 </li>
+    				 <li class="list-group-item d-flex justify-content-between align-items-center">
+    					 Lower characters: a-z
+    					 <span class="badge badge-primary badge-pill">1</span>
+    				 </li>
+    				 <li class="list-group-item d-flex justify-content-between align-items-center">
+    					 Digits: 0-9
+    					 <span class="badge badge-primary badge-pill">2</span>
+    				 </li>
+    				</ul>
+          </div>
+
+  		  </form>
+
+    	</div>
+		</div>
+	</div>
+
+</body>
+
+<script>
+$(document).ready(function(){
+  $("#password-section").hide();
+  $("#password-section-info").hide();
+  $("#nav_create_user").addClass("active");
+	$("#nav_user_management").removeClass("text-muted");
+} );
+
+function toggle_password_fields() {
+  var password_div = $("#password-section");
+  if(password_div.is(":visible")){
+    $("#password-section").hide();
+    $("#password-section-info").hide();
+    $("#inputPassword1").prop('required',false);
+    $("#inputPassword2").prop('required',false);
+  } else {
+    $("#password-section").show();
+    $("#password-section-info").show();
+    $("#inputPassword1").prop('required',true);
+    $("#inputPassword2").prop('required',true);
+  }
+}
+</script>
+
+</html>
diff --git a/var/www/modules/settings/templates/edit_profile.html b/var/www/modules/settings/templates/edit_profile.html
index f0b8d476..8dfe4cf5 100644
--- a/var/www/modules/settings/templates/edit_profile.html
+++ b/var/www/modules/settings/templates/edit_profile.html
@@ -27,11 +27,11 @@
 
 			{% include 'settings/menu_sidebar.html' %}
 
-			<div class="col-12 col-lg-10" id="core_content">
+			<div class="col-12 col-xl-10" id="core_content">
 
 				<div class="card mb-3 mt-1">
 					<div class="card-header text-white bg-dark pb-1">
-						<h5 class="card-title">AIL-framework Status :</h5>
+						<h5 class="card-title">My Profile :</h5>
 					</div>
 					<div class="card-body">
 
diff --git a/var/www/modules/settings/templates/users_list.html b/var/www/modules/settings/templates/users_list.html
new file mode 100644
index 00000000..d29dbb8e
--- /dev/null
+++ b/var/www/modules/settings/templates/users_list.html
@@ -0,0 +1,108 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Server Management - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+
+  <style>
+	.edit_icon:hover{
+		cursor: pointer;
+		color: #17a2b8;
+	}
+  .trash_icon:hover{
+		cursor: pointer;
+		color: #c82333;
+	}
+	</style>
+
+</head>
+<body>
+
+  {% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'settings/menu_sidebar.html' %}
+
+			<div class="col-12 col-xl-10" id="core_content">
+
+        {% if new_user %}
+        <div class="text-center my-3 ">
+          <div class="card">
+            <div class="card-header">
+              {% if new_user['edited']=='True' %}
+                <h5 class="card-title">User Edited</h5>
+              {% else %}
+                <h5 class="card-title">User Created</h5>
+              {% endif %}
+            </div>
+            <div class="card-body">
+              <p>User: {{new_user['email']}}</p>
+              <p>Password: {{new_user['password']}}</p>
+              <a href="{{url_for('settings.users_list')}}" class="btn btn-primary"><i class="fas fa-eye-slash"></i> Hide</a>
+            </div>
+          </div>
+        </div>
+        {% endif %}
+
+        <div class="table-responsive mt-1 table-hover table-borderless table-striped">
+          <table class="table">
+            <thead class="thead-dark">
+                <tr>
+                  <th>Email</th>
+                  <th>Role</th>
+                  <th>Api Key</th>
+                  <th>Actions</th>
+                </tr>
+            </thead>
+            <tbody id="tbody_last_crawled">
+                {% for user in all_users %}
+                  <tr>
+                    <td>{{user['email']}}</td>
+                    <td>{{user['role']}}</td>
+                    <td>
+                      {{user['api_key']}}
+                      <a class="ml-3" href="{{url_for('settings.new_token_user')}}?user_id={{user['email']}}"><i class="fa fa-random"></i></a>
+                    </td>
+                    <td>
+                      <a href="{{ url_for('settings.edit_user')}}?user_id={{user['email']}}">
+                          <i class="fas fa-pencil-alt edit_icon"></i>
+											</a>
+                      <a href="{{ url_for('settings.delete_user')}}?user_id={{user['email']}}" class="ml-4">
+                          <i class="fas fa-trash-alt trash_icon"></i>
+											</a>
+                    </td>
+                  </tr>
+                {% endfor %}
+            </tbody>
+          </table>
+        </div>
+
+    	</div>
+		</div>
+	</div>
+
+</body>
+
+<script>
+$(document).ready(function(){
+  $("#nav_edit_profile").addClass("active");
+	$("#nav_user_management").removeClass("text-muted");
+} );
+</script>
+
+</html>
diff --git a/var/www/templates/settings/menu_sidebar.html b/var/www/templates/settings/menu_sidebar.html
index 558a901f..e7aa7a7f 100644
--- a/var/www/templates/settings/menu_sidebar.html
+++ b/var/www/templates/settings/menu_sidebar.html
@@ -38,18 +38,18 @@
           </ul>
         </nav>
         <nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_users">
-      		<h5 class="d-flex text-muted w-100">
+      		<h5 class="d-flex text-muted w-100" id="nav_user_management">
       		  <span>User Management</span>
       		</h5>
           <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
       	    <li class="nav-item">
-      	      <a class="nav-link" href="{{url_for('hashDecoded.hashDecoded_page')}}" id="nav_dashboard">
+      	      <a class="nav-link" href="{{url_for('settings.create_user')}}" id="nav_create_user">
       	        <i class="fas fa-user-plus"></i>
       	        <span>Create User</span>
       	      </a>
       	    </li>
             <li class="nav-item">
-      	      <a class="nav-link" href="{{url_for('hashDecoded.hashDecoded_page')}}" id="nav_dashboard">
+      	      <a class="nav-link" href="{{url_for('settings.users_list')}}" id="nav_users_list">
       	        <i class="fas fa-users"></i>
       	        <span>Users List</span>
       	      </a>

From 34e57fe7af06358ee2f8afbb13d963f230bb7412 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Wed, 19 Jun 2019 15:00:25 +0200
Subject: [PATCH 08/22] chg: [user_managemant] clean code + check password and
 email length

---
 bin/packages/User.py                          |  2 -
 var/www/Flask_server.py                       | 91 +++----------------
 var/www/create_default_user.py                | 45 +--------
 var/www/modules/Flask_config.py               |  4 -
 var/www/modules/Role_Manager.py               | 33 +++++--
 var/www/modules/settings/Flask_settings.py    | 13 +--
 .../settings/templates/create_user.html       |  4 +
 .../settings/templates/users_list.html        |  2 +-
 var/www/templates/change_password.html        |  4 +
 var/www/templates/settings/menu_sidebar.html  |  2 +-
 10 files changed, 53 insertions(+), 147 deletions(-)

diff --git a/bin/packages/User.py b/bin/packages/User.py
index 5be7021d..cac5c688 100755
--- a/bin/packages/User.py
+++ b/bin/packages/User.py
@@ -44,10 +44,8 @@ class User(UserMixin):
 
     def check_password(self, password):
         password = password.encode()
-        print(self.id)
         hashed_password = self.r_serv_db.hget('user:all', self.id).encode()
         if bcrypt.checkpw(password, hashed_password):
-            print('password correct')
             return True
         else:
             return False
diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py
index a66c5229..5990ff6b 100755
--- a/var/www/Flask_server.py
+++ b/var/www/Flask_server.py
@@ -1,6 +1,10 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 
+import os
+import re
+import sys
+
 import redis
 import configparser
 import random
@@ -15,10 +19,7 @@ import bcrypt
 
 import flask
 import importlib
-import os
-import re
 from os.path import join
-import sys
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
 sys.path.append('./modules/')
 import Paste
@@ -31,52 +32,20 @@ from pytaxonomies import Taxonomies
 # Import config
 import Flask_config
 
+# Import Role_Manager
+from Role_Manager import create_user_db, check_password_strength
+
 def flask_init():
-    # check if an account exists
-    if not r_serv_db.exists('user:all'):
-        password = secrets.token_urlsafe()
-        create_user_db('admin@admin.test', password, role='admin',default=True)
-    # add default roles
+    # # TODO: move this to update
+    # role init
     if not r_serv_db.exists('ail:all_role'):
         r_serv_db.zadd('ail:all_role', 1, 'admin')
         r_serv_db.zadd('ail:all_role', 2, 'analyst')
 
-def hashing_password(bytes_password):
-    hashed = bcrypt.hashpw(bytes_password, bcrypt.gensalt())
-    return hashed
-
-def verify_password(id, bytes_password):
-    hashed_password = r_serv_db.hget('user:all', id)
-    if bcrypt.checkpw(password, hashed):
-        return True
-    else:
-        return False
-
-def check_password_strength(password):
-    result = regex_password.match(password)
-    if result:
-        return True
-    else:
-        return False
-
-
-def create_user_db(username_id , password, default=False, role=None, update=False):
-    password = password.encode()
-    password_hash = hashing_password(password)
-    r_serv_db.hset('user:all', username_id, password_hash)
-    if update:
-        r_serv_db.hdel('user_metadata:{}'.format(username_id), 'change_passwd')
-        if username_id=='admin@admin.test':
-            os.remove(default_passwd_file)
-    else:
-        if default:
-            r_serv_db.hset('user_metadata:{}'.format(username_id), 'change_passwd', True)
-        if role:
-            if role in get_all_role():
-                r_serv_db.sadd('user_role:{}'.format(role), username_id)
-
-def get_all_role():
-    return r_serv_db.zrange('ail:all_role', 0 , -1)
+    # check if an account exists
+    if not r_serv_db.exists('user:all'):
+        password = secrets.token_urlsafe()
+        create_user_db('admin@admin.test', password, role='admin',default=True)
 
 # CONFIG #
 cfg = Flask_config.cfg
@@ -85,11 +54,6 @@ baseUrl = baseUrl.replace('/', '')
 if baseUrl != '':
     baseUrl = '/'+baseUrl
 
-default_passwd_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD')
-
-regex_password = r'^(?=(.*\d){2})(?=.*[a-z])(?=.*[A-Z]).{10,}$'
-regex_password = re.compile(regex_password)
-
 # ========= REDIS =========#
 r_serv_db = redis.StrictRedis(
     host=cfg.get("ARDB_DB", "host"),
@@ -181,9 +145,6 @@ modified_header = modified_header.replace('<!--insert here-->', '\n'.join(to_add
 with open('templates/header.html', 'w') as f:
     f.write(modified_header)
 
-flask_init()
-
-
 # ========= JINJA2 FUNCTIONS ========
 def list_len(s):
     return len(s)
@@ -213,7 +174,6 @@ def login():
             user = User.get(username)
             if user and user.check_password(password):
                 login_user(user) ## TODO: use remember me ?
-                print(user.is_active)
                 if user.request_password_change():
                     return redirect(url_for('change_password'))
                 else:
@@ -245,37 +205,12 @@ def change_password():
     else:
         return render_template("change_password.html")
 
-@app.route('/role', methods=['POST', 'GET'])
-def role():
-    return 'ERROR role'
-
 @app.route('/logout')
 @login_required
 def logout():
     logout_user()
     return redirect(url_for('login'))
 
-@app.route('/create_user')
-@login_required
-def create_user():
-    username = request.form.get('username')
-    password = request.form.get('password')
-    #role = request.form.get('role') ## TODO: create role
-
-    ## TODO: validate username
-    ## TODO: validate password
-
-    username = 'admin@admin.test'
-    password = 'admin'
-
-    if r_serv_db.hexists('user:all', username):
-        return 'this id is not available'
-
-    create_user_db(username, password)
-
-    return 'True'
-
-
 @app.route('/searchbox/')
 def searchbox():
     return render_template("searchbox.html")
diff --git a/var/www/create_default_user.py b/var/www/create_default_user.py
index 1dcc9aed..9391af56 100755
--- a/var/www/create_default_user.py
+++ b/var/www/create_default_user.py
@@ -3,60 +3,21 @@
 
 import os
 import sys
-import redis
-import configparser
 
-import bcrypt
 import secrets
 
-# Import config
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
 sys.path.append('./modules/')
 
-def get_all_role():
-    return r_serv_db.zrange('ail:all_role', 0 , -1)
+from Role_Manager import create_user_db, get_default_admin_token
 
-def hashing_password(bytes_password):
-    hashed = bcrypt.hashpw(bytes_password, bcrypt.gensalt())
-    return hashed
-
-def create_user_db(username_id , password, default=False, role=None, update=False):
-    password = password.encode()
-    password_hash = hashing_password(password)
-    r_serv_db.hset('user:all', username_id, password_hash)
-    if update:
-        r_serv_db.hdel('user_metadata:{}'.format(username_id), 'change_passwd')
-    else:
-        if default:
-            r_serv_db.hset('user_metadata:{}'.format(username_id), 'change_passwd', True)
-            r_serv_db.hset('user_metadata:{}'.format(username_id), 'role', role)
-        if role:
-            if role in get_all_role():
-                r_serv_db.sadd('user_role:{}'.format(role), username_id)
 
 if __name__ == "__main__":
-    configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
-    if not os.path.exists(configfile):
-        raise Exception('Unable to find the configuration file. \
-                        Did you set environment variables? \
-                        Or activate the virtualenv.')
-
-    cfg = configparser.ConfigParser()
-    cfg.read(configfile)
-
-    r_serv_db = redis.StrictRedis(
-        host=cfg.get("ARDB_DB", "host"),
-        port=cfg.getint("ARDB_DB", "port"),
-        db=cfg.getint("ARDB_DB", "db"),
-        decode_responses=True)
 
     username = 'admin@admin.test'
     password = secrets.token_urlsafe()
     create_user_db(username, password, role='admin', default=True)
-
-    # create user token
-    token = secrets.token_urlsafe(41)
-    r_serv_db.hset('user:tokens', token, username)
-    r_serv_db.hset('user_metadata:{}'.format(username), 'token', token)
+    token = get_default_admin_token()
 
     default_passwd_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD')
     to_write_str = '# Password Generated by default\n# This file is deleted after the first login\n#\nemail=admin@admin.test\npassword='
diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py
index 2219824a..899a26b5 100644
--- a/var/www/modules/Flask_config.py
+++ b/var/www/modules/Flask_config.py
@@ -7,7 +7,6 @@
 import configparser
 import redis
 import os
-import re
 import sys
 
 # FLASK #
@@ -176,9 +175,6 @@ max_dashboard_logs = int(cfg.get("Flask", "max_dashboard_logs"))
 
 crawler_enabled = cfg.getboolean("Crawler", "activate_crawler")
 
-regex_password = r'^(?=(.*\d){2})(?=.*[a-z])(?=.*[A-Z]).{10,}$'
-regex_password = re.compile(regex_password)
-
 # VT
 try:
     from virusTotalKEYS import vt_key
diff --git a/var/www/modules/Role_Manager.py b/var/www/modules/Role_Manager.py
index 85440d57..aa524265 100644
--- a/var/www/modules/Role_Manager.py
+++ b/var/www/modules/Role_Manager.py
@@ -2,6 +2,7 @@
 # -*-coding:UTF-8 -*
 
 import os
+import re
 import redis
 import bcrypt
 import secrets
@@ -31,6 +32,15 @@ r_serv_db = redis.StrictRedis(
     db=cfg.getint("ARDB_DB", "db"),
     decode_responses=True)
 
+default_passwd_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD')
+
+regex_password = r'^(?=(.*\d){2})(?=.*[a-z])(?=.*[A-Z]).{10,100}$'
+regex_password = re.compile(regex_password)
+
+###############################################################
+###############       CHECK ROLE ACCESS      ##################
+###############################################################
+
 def login_admin(func):
     @wraps(func)
     def decorated_view(*args, **kwargs):
@@ -57,7 +67,11 @@ def login_analyst(func):
 ###############################################################
 ###############################################################
 
-
+def get_default_admin_token():
+    if r_serv_db.exists('user_metadata:admin@admin.test'):
+        return r_serv_db.hget('user_metadata:admin@admin.test', 'token')
+    else:
+        return ''
 
 def create_user_db(username_id , password, default=False, role=None, update=False):
     password = password.encode()
@@ -70,6 +84,9 @@ def create_user_db(username_id , password, default=False, role=None, update=Fals
 
     if update:
         r_serv_db.hdel('user_metadata:{}'.format(username_id), 'change_passwd')
+        # remove default user password file
+        if username_id=='admin@admin.test':
+            os.remove(default_passwd_file)
     else:
         if default:
             r_serv_db.hset('user_metadata:{}'.format(username_id), 'change_passwd', True)
@@ -93,22 +110,17 @@ def edit_user_db(user_id, role, password=None):
 
         if current_role < request_level:
             role_to_remove = get_user_role_by_range(current_role -1, request_level - 2)
-            print('to remove')
-            print(role_to_remove)
             for role_id in role_to_remove:
                 r_serv_db.srem('user_role:{}'.format(role_id), user_id)
             r_serv_db.hset('user_metadata:{}'.format(user_id), 'role', role)
         else:
             role_to_add = get_user_role_by_range(request_level -1, current_role)
-            print('to add')
-            print(role_to_add)
             for role_id in role_to_add:
                 r_serv_db.sadd('user_role:{}'.format(role_id), user_id)
             r_serv_db.hset('user_metadata:{}'.format(user_id), 'role', role)
 
 def delete_user_db(user_id):
     if r_serv_db.exists('user_metadata:{}'.format(user_id)):
-        print('r')
         role_to_remove =get_all_role()
         for role_id in role_to_remove:
             r_serv_db.srem('user_role:{}'.format(role_id), user_id)
@@ -121,6 +133,13 @@ def hashing_password(bytes_password):
     hashed = bcrypt.hashpw(bytes_password, bcrypt.gensalt())
     return hashed
 
+def check_password_strength(password):
+    result = regex_password.match(password)
+    if result:
+        return True
+    else:
+        return False
+
 def get_all_role():
     return r_serv_db.zrange('ail:all_role', 0, -1)
 
@@ -132,6 +151,4 @@ def get_all_user_role(user_role):
     return r_serv_db.zrange('ail:all_role', current_role_val -1, -1)
 
 def get_user_role_by_range(inf, sup):
-    print(inf)
-    print(sup)
     return r_serv_db.zrange('ail:all_role', inf, sup)
diff --git a/var/www/modules/settings/Flask_settings.py b/var/www/modules/settings/Flask_settings.py
index 683d5b01..8a5fb776 100644
--- a/var/www/modules/settings/Flask_settings.py
+++ b/var/www/modules/settings/Flask_settings.py
@@ -7,7 +7,7 @@
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
 from flask_login import login_required, current_user
 
-from Role_Manager import login_admin, login_analyst, create_user_db, edit_user_db, delete_user_db
+from Role_Manager import login_admin, login_analyst, create_user_db, edit_user_db, delete_user_db, check_password_strength
 
 import json
 import secrets
@@ -26,7 +26,6 @@ max_preview_char = Flask_config.max_preview_char
 max_preview_modal = Flask_config.max_preview_modal
 REPO_ORIGIN = Flask_config.REPO_ORIGIN
 dict_update_description = Flask_config.dict_update_description
-regex_password = Flask_config.regex_password
 
 settings = Blueprint('settings', __name__, template_folder='templates')
 
@@ -36,13 +35,6 @@ settings = Blueprint('settings', __name__, template_folder='templates')
 def one():
     return 1
 
-def check_password_strength(password):
-    result = regex_password.match(password)
-    if result:
-        return True
-    else:
-        return False
-
 def generate_new_token(user_id):
     # create user token
     current_token = r_serv_db.hget('user_metadata:{}'.format(user_id), 'token')
@@ -162,7 +154,7 @@ def create_user_post():
 
     all_roles = get_all_roles()
 
-    if email and role:
+    if email and len(email)< 300 and role:
         if role in all_roles:
             # password set
             if password1 and password2:
@@ -206,7 +198,6 @@ def users_list():
         new_user_dict['email'] = new_user
         new_user_dict['edited'] = request.args.get('new_user_edited')
         new_user_dict['password'] = request.args.get('new_user_password')
-    print(new_user)
     return render_template("users_list.html", all_users=all_users, new_user=new_user_dict)
 
 @settings.route("/settings/edit_user", methods=['GET'])
diff --git a/var/www/modules/settings/templates/create_user.html b/var/www/modules/settings/templates/create_user.html
index 2a24094d..6da31be3 100644
--- a/var/www/modules/settings/templates/create_user.html
+++ b/var/www/modules/settings/templates/create_user.html
@@ -83,6 +83,10 @@
     					 Digits: 0-9
     					 <span class="badge badge-primary badge-pill">2</span>
     				 </li>
+             <li class="list-group-item d-flex justify-content-between align-items-center">
+  						 Maximum length
+  						 <span class="badge badge-primary badge-pill">100</span>
+  					 </li>
     				</ul>
           </div>
 
diff --git a/var/www/modules/settings/templates/users_list.html b/var/www/modules/settings/templates/users_list.html
index d29dbb8e..ccf4003b 100644
--- a/var/www/modules/settings/templates/users_list.html
+++ b/var/www/modules/settings/templates/users_list.html
@@ -100,7 +100,7 @@
 
 <script>
 $(document).ready(function(){
-  $("#nav_edit_profile").addClass("active");
+  $("#nav_users_list").addClass("active");
 	$("#nav_user_management").removeClass("text-muted");
 } );
 </script>
diff --git a/var/www/templates/change_password.html b/var/www/templates/change_password.html
index 2107b7a1..e5f13a84 100644
--- a/var/www/templates/change_password.html
+++ b/var/www/templates/change_password.html
@@ -91,6 +91,10 @@
 						 Digits: 0-9
 						 <span class="badge badge-primary badge-pill">2</span>
 					 </li>
+					 <li class="list-group-item d-flex justify-content-between align-items-center">
+						 Maximum length
+						 <span class="badge badge-primary badge-pill">100</span>
+					 </li>
 					</ul>
 
 		    </form>
diff --git a/var/www/templates/settings/menu_sidebar.html b/var/www/templates/settings/menu_sidebar.html
index e7aa7a7f..f1af27d1 100644
--- a/var/www/templates/settings/menu_sidebar.html
+++ b/var/www/templates/settings/menu_sidebar.html
@@ -11,7 +11,7 @@
       		</h5>
           <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
       	    <li class="nav-item">
-      	      <a class="nav-link" href="{{url_for('hashDecoded.hashDecoded_page')}}" id="nav_server_status">
+      	      <a class="nav-link" href="{{url_for('settings.settings_page')}}" id="nav_server_status">
       	        <i class="fas fa-tools"></i>
       	        <span>Server Status</span>
       	      </a>

From ef716f22e51d5336a01b12c42512fac298adb4a2 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Wed, 19 Jun 2019 17:02:09 +0200
Subject: [PATCH 09/22] chg: [user_management endpoint] check user roles + add
 503 template

---
 var/www/Flask_server.py                       |  9 ++++
 .../modules/PasteSubmit/Flask_PasteSubmit.py  | 15 ++++++
 var/www/modules/Tags/Flask_Tags.py            | 21 +++++++-
 var/www/modules/dashboard/Flask_dashboard.py  |  6 +++
 .../modules/hashDecoded/Flask_hashDecoded.py  | 43 ++++++++++++++++
 .../hiddenServices/Flask_hiddenServices.py    | 27 ++++++++--
 .../modules/rawSkeleton/Flask_rawSkeleton.py  |  3 ++
 var/www/modules/search/Flask_search.py        |  4 ++
 var/www/modules/sentiment/Flask_sentiment.py  |  6 +++
 var/www/modules/settings/Flask_settings.py    | 13 ++++-
 var/www/modules/showpaste/Flask_showpaste.py  | 10 ++++
 var/www/modules/terms/Flask_terms.py          | 15 ++++++
 .../trendingcharts/Flask_trendingcharts.py    |  6 +++
 .../trendingmodules/Flask_trendingmodules.py  |  5 ++
 var/www/templates/error/403.html              | 49 +++++++++++++++++++
 15 files changed, 225 insertions(+), 7 deletions(-)
 create mode 100644 var/www/templates/error/403.html

diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py
index 5990ff6b..09f9ff41 100755
--- a/var/www/Flask_server.py
+++ b/var/www/Flask_server.py
@@ -34,6 +34,7 @@ import Flask_config
 
 # Import Role_Manager
 from Role_Manager import create_user_db, check_password_strength
+from Role_Manager import login_admin, login_analyst
 
 def flask_init():
     # # TODO: move this to update
@@ -211,7 +212,15 @@ def logout():
     logout_user()
     return redirect(url_for('login'))
 
+# role error template
+@app.route('/role', methods=['POST', 'GET'])
+@login_required
+def role():
+    return render_template("error/403.html"), 403
+
 @app.route('/searchbox/')
+@login_required
+@login_analyst
 def searchbox():
     return render_template("searchbox.html")
 
diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py
index 8a1ffaab..85ca57e2 100644
--- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py
+++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py
@@ -6,6 +6,8 @@
 '''
 import redis
 from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect
+
+from Role_Manager import login_admin, login_analyst
 from flask_login import login_required
 
 import unicodedata
@@ -275,6 +277,7 @@ def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title,
 
 @PasteSubmit.route("/PasteSubmit/", methods=['GET'])
 @login_required
+@login_analyst
 def PasteSubmit_page():
     #active taxonomies
     active_taxonomies = r_serv_tags.smembers('active_taxonomies')
@@ -288,6 +291,7 @@ def PasteSubmit_page():
 
 @PasteSubmit.route("/PasteSubmit/submit", methods=['POST'])
 @login_required
+@login_analyst
 def submit():
 
     #paste_name = request.form['paste_name']
@@ -398,6 +402,7 @@ def submit():
 
 @PasteSubmit.route("/PasteSubmit/submit_status", methods=['GET'])
 @login_required
+@login_analyst
 def submit_status():
     UUID = request.args.get('UUID')
 
@@ -465,6 +470,7 @@ def submit_status():
 
 @PasteSubmit.route("/PasteSubmit/create_misp_event", methods=['POST'])
 @login_required
+@login_analyst
 def create_misp_event():
 
     distribution = int(request.form['misp_data[Event][distribution]'])
@@ -488,6 +494,7 @@ def create_misp_event():
 
 @PasteSubmit.route("/PasteSubmit/create_hive_case", methods=['POST'])
 @login_required
+@login_analyst
 def create_hive_case():
 
     hive_tlp = int(request.form['hive_tlp'])
@@ -511,6 +518,7 @@ def create_hive_case():
 
 @PasteSubmit.route("/PasteSubmit/edit_tag_export")
 @login_required
+@login_analyst
 def edit_tag_export():
     misp_auto_events = r_serv_db.get('misp:auto-events')
     hive_auto_alerts = r_serv_db.get('hive:auto-alerts')
@@ -576,6 +584,7 @@ def edit_tag_export():
 
 @PasteSubmit.route("/PasteSubmit/tag_export_edited", methods=['POST'])
 @login_required
+@login_analyst
 def tag_export_edited():
     tag_enabled_misp = request.form.getlist('tag_enabled_misp')
     tag_enabled_hive = request.form.getlist('tag_enabled_hive')
@@ -601,30 +610,35 @@ def tag_export_edited():
 
 @PasteSubmit.route("/PasteSubmit/enable_misp_auto_event")
 @login_required
+@login_analyst
 def enable_misp_auto_event():
     r_serv_db.set('misp:auto-events', 1)
     return edit_tag_export()
 
 @PasteSubmit.route("/PasteSubmit/disable_misp_auto_event")
 @login_required
+@login_analyst
 def disable_misp_auto_event():
     r_serv_db.set('misp:auto-events', 0)
     return edit_tag_export()
 
 @PasteSubmit.route("/PasteSubmit/enable_hive_auto_alert")
 @login_required
+@login_analyst
 def enable_hive_auto_alert():
     r_serv_db.set('hive:auto-alerts', 1)
     return edit_tag_export()
 
 @PasteSubmit.route("/PasteSubmit/disable_hive_auto_alert")
 @login_required
+@login_analyst
 def disable_hive_auto_alert():
     r_serv_db.set('hive:auto-alerts', 0)
     return edit_tag_export()
 
 @PasteSubmit.route("/PasteSubmit/add_push_tag")
 @login_required
+@login_analyst
 def add_push_tag():
     tag = request.args.get('tag')
     if tag is not None:
@@ -643,6 +657,7 @@ def add_push_tag():
 
 @PasteSubmit.route("/PasteSubmit/delete_push_tag")
 @login_required
+@login_analyst
 def delete_push_tag():
     tag = request.args.get('tag')
 
diff --git a/var/www/modules/Tags/Flask_Tags.py b/var/www/modules/Tags/Flask_Tags.py
index f68491e6..8ab81297 100644
--- a/var/www/modules/Tags/Flask_Tags.py
+++ b/var/www/modules/Tags/Flask_Tags.py
@@ -222,7 +222,7 @@ def update_tag_last_seen(tag, tag_first_seen, tag_last_seen):
 
 @Tags.route("/tags/", methods=['GET'])
 @login_required
-@login_admin
+@login_analyst
 def Tags_page():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -357,6 +357,7 @@ def Tags_page():
 
 @Tags.route("/Tags/get_all_tags")
 @login_required
+@login_analyst
 def get_all_tags():
 
     all_tags = r_serv_tags.smembers('list_tags')
@@ -380,6 +381,7 @@ def get_all_tags():
 
 @Tags.route("/Tags/get_all_tags_taxonomies")
 @login_required
+@login_analyst
 def get_all_tags_taxonomies():
 
     taxonomies = Taxonomies()
@@ -398,6 +400,7 @@ def get_all_tags_taxonomies():
 
 @Tags.route("/Tags/get_all_tags_galaxies")
 @login_required
+@login_analyst
 def get_all_tags_galaxy():
 
     active_galaxies = r_serv_tags.smembers('active_galaxies')
@@ -412,6 +415,7 @@ def get_all_tags_galaxy():
 
 @Tags.route("/Tags/get_tags_taxonomie")
 @login_required
+@login_analyst
 def get_tags_taxonomie():
 
     taxonomie = request.args.get('taxonomie')
@@ -439,6 +443,7 @@ def get_tags_taxonomie():
 
 @Tags.route("/Tags/get_tags_galaxy")
 @login_required
+@login_analyst
 def get_tags_galaxy():
 
     galaxy = request.args.get('galaxy')
@@ -460,6 +465,7 @@ def get_tags_galaxy():
 
 @Tags.route("/Tags/remove_tag")
 @login_required
+@login_analyst
 def remove_tag():
 
     #TODO verify input
@@ -492,6 +498,7 @@ def confirm_tag():
 
 @Tags.route("/Tags/tag_validation")
 @login_required
+@login_analyst
 def tag_validation():
 
     path = request.args.get('paste')
@@ -513,6 +520,7 @@ def tag_validation():
 
 @Tags.route("/Tags/addTags")
 @login_required
+@login_analyst
 def addTags():
 
     tags = request.args.get('tags')
@@ -563,6 +571,7 @@ def addTags():
 
 @Tags.route("/Tags/taxonomies")
 @login_required
+@login_analyst
 def taxonomies():
 
     active_taxonomies = r_serv_tags.smembers('active_taxonomies')
@@ -600,6 +609,7 @@ def taxonomies():
 
 @Tags.route("/Tags/edit_taxonomie")
 @login_required
+@login_analyst
 def edit_taxonomie():
 
     taxonomies = Taxonomies()
@@ -649,6 +659,7 @@ def edit_taxonomie():
 
 @Tags.route("/Tags/disable_taxonomie")
 @login_required
+@login_analyst
 def disable_taxonomie():
 
     taxonomies = Taxonomies()
@@ -670,6 +681,7 @@ def disable_taxonomie():
 
 @Tags.route("/Tags/active_taxonomie")
 @login_required
+@login_analyst
 def active_taxonomie():
 
     taxonomies = Taxonomies()
@@ -690,6 +702,7 @@ def active_taxonomie():
 
 @Tags.route("/Tags/edit_taxonomie_tag")
 @login_required
+@login_analyst
 def edit_taxonomie_tag():
 
     taxonomies = Taxonomies()
@@ -733,6 +746,7 @@ def edit_taxonomie_tag():
 
 @Tags.route("/Tags/galaxies")
 @login_required
+@login_analyst
 def galaxies():
 
     active_galaxies = r_serv_tags.smembers('active_galaxies')
@@ -780,6 +794,7 @@ def galaxies():
 
 @Tags.route("/Tags/edit_galaxy")
 @login_required
+@login_analyst
 def edit_galaxy():
 
     id = request.args.get('galaxy')
@@ -848,6 +863,7 @@ def edit_galaxy():
 
 @Tags.route("/Tags/active_galaxy")
 @login_required
+@login_analyst
 def active_galaxy():
 
     id = request.args.get('galaxy')
@@ -893,6 +909,7 @@ def active_galaxy():
 
 @Tags.route("/Tags/disable_galaxy")
 @login_required
+@login_analyst
 def disable_galaxy():
 
     id = request.args.get('galaxy')
@@ -914,6 +931,7 @@ def disable_galaxy():
 
 @Tags.route("/Tags/edit_galaxy_tag")
 @login_required
+@login_analyst
 def edit_galaxy_tag():
 
     arg1 = request.args.getlist('tag_enabled')
@@ -987,6 +1005,7 @@ def edit_galaxy_tag():
 
 @Tags.route("/Tags/tag_galaxy_info")
 @login_required
+@login_analyst
 def tag_galaxy_info():
 
     galaxy = request.args.get('galaxy')
diff --git a/var/www/modules/dashboard/Flask_dashboard.py b/var/www/modules/dashboard/Flask_dashboard.py
index 0bbcccac..160d9edb 100644
--- a/var/www/modules/dashboard/Flask_dashboard.py
+++ b/var/www/modules/dashboard/Flask_dashboard.py
@@ -13,6 +13,8 @@ import flask
 from Date import Date
 
 from flask import Flask, render_template, jsonify, request, Blueprint, url_for
+
+from Role_Manager import login_admin, login_analyst
 from flask_login import login_required
 
 # ============ VARIABLES ============
@@ -111,11 +113,13 @@ def datetime_from_utc_to_local(utc_str):
 
 @dashboard.route("/_logs")
 @login_required
+@login_analyst
 def logs():
     return flask.Response(event_stream(), mimetype="text/event-stream")
 
 @dashboard.route("/_get_last_logs_json")
 @login_required
+@login_analyst
 def get_last_logs_json():
     date = datetime.datetime.now().strftime("%Y%m%d")
 
@@ -158,12 +162,14 @@ def get_last_logs_json():
 
 @dashboard.route("/_stuff", methods=['GET'])
 @login_required
+@login_analyst
 def stuff():
     return jsonify(row1=get_queues(r_serv))
 
 
 @dashboard.route("/")
 @login_required
+@login_analyst
 def index():
     default_minute = cfg.get("Flask", "minute_processed_paste")
     threshold_stucked_module = cfg.getint("Module_ModuleInformation", "threshold_stucked_module")
diff --git a/var/www/modules/hashDecoded/Flask_hashDecoded.py b/var/www/modules/hashDecoded/Flask_hashDecoded.py
index 83ce54b8..a3537b1a 100644
--- a/var/www/modules/hashDecoded/Flask_hashDecoded.py
+++ b/var/www/modules/hashDecoded/Flask_hashDecoded.py
@@ -17,6 +17,7 @@ from hashlib import sha256
 
 import requests
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, send_file
+from Role_Manager import login_admin, login_analyst
 from flask_login import login_required
 
 # ============ VARIABLES ============
@@ -476,6 +477,7 @@ def correlation_graph_node_json(correlation_type, type_id, key_id):
 # ============= ROUTES ==============
 @hashDecoded.route("/hashDecoded/all_hash_search", methods=['POST'])
 @login_required
+@login_analyst
 def all_hash_search():
     date_from = request.form.get('date_from')
     date_to = request.form.get('date_to')
@@ -486,6 +488,7 @@ def all_hash_search():
 
 @hashDecoded.route("/hashDecoded/", methods=['GET'])
 @login_required
+@login_analyst
 def hashDecoded_page():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -604,6 +607,7 @@ def hashDecoded_page():
 
 @hashDecoded.route('/hashDecoded/hash_by_type')
 @login_required
+@login_analyst
 def hash_by_type():
     type = request.args.get('type')
     type = 'text/plain'
@@ -612,6 +616,7 @@ def hash_by_type():
 
 @hashDecoded.route('/hashDecoded/hash_hash')
 @login_required
+@login_analyst
 def hash_hash():
     hash = request.args.get('hash')
     return render_template('hash_hash.html')
@@ -619,6 +624,7 @@ def hash_hash():
 
 @hashDecoded.route('/hashDecoded/showHash')
 @login_required
+@login_analyst
 def showHash():
     hash = request.args.get('hash')
     #hash = 'e02055d3efaad5d656345f6a8b1b6be4fe8cb5ea'
@@ -673,6 +679,7 @@ def showHash():
 
 @hashDecoded.route('/hashDecoded/downloadHash')
 @login_required
+@login_analyst
 def downloadHash():
     hash = request.args.get('hash')
     # sanitize hash
@@ -710,6 +717,7 @@ def downloadHash():
 
 @hashDecoded.route('/hashDecoded/hash_by_type_json')
 @login_required
+@login_analyst
 def hash_by_type_json():
     type = request.args.get('type')
 
@@ -744,6 +752,7 @@ def hash_by_type_json():
 
 @hashDecoded.route('/hashDecoded/decoder_type_json')
 @login_required
+@login_analyst
 def decoder_type_json():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -800,6 +809,7 @@ def decoder_type_json():
 
 @hashDecoded.route('/hashDecoded/top5_type_json')
 @login_required
+@login_analyst
 def top5_type_json():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -859,6 +869,7 @@ def top5_type_json():
 
 @hashDecoded.route('/hashDecoded/daily_type_json')
 @login_required
+@login_analyst
 def daily_type_json():
     date = request.args.get('date')
 
@@ -879,6 +890,7 @@ def daily_type_json():
 
 @hashDecoded.route('/hashDecoded/range_type_json')
 @login_required
+@login_analyst
 def range_type_json():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -936,6 +948,7 @@ def range_type_json():
 
 @hashDecoded.route('/hashDecoded/hash_graph_line_json')
 @login_required
+@login_analyst
 def hash_graph_line_json():
     hash = request.args.get('hash')
     date_from = request.args.get('date_from')
@@ -966,6 +979,7 @@ def hash_graph_line_json():
 
 @hashDecoded.route('/hashDecoded/hash_graph_node_json')
 @login_required
+@login_analyst
 def hash_graph_node_json():
     hash = request.args.get('hash')
 
@@ -1034,6 +1048,7 @@ def hash_graph_node_json():
 
 @hashDecoded.route('/hashDecoded/hash_types')
 @login_required
+@login_analyst
 def hash_types():
     date_from = 20180701
     date_to = 20180706
@@ -1042,6 +1057,7 @@ def hash_types():
 
 @hashDecoded.route('/hashDecoded/send_file_to_vt_js')
 @login_required
+@login_analyst
 def send_file_to_vt_js():
     hash = request.args.get('hash')
 
@@ -1066,6 +1082,7 @@ def send_file_to_vt_js():
 
 @hashDecoded.route('/hashDecoded/update_vt_result')
 @login_required
+@login_analyst
 def update_vt_result():
     hash = request.args.get('hash')
 
@@ -1102,6 +1119,8 @@ def update_vt_result():
 ############################ PGPDump ############################
 
 @hashDecoded.route('/decoded/pgp_by_type_json') ## TODO: REFRACTOR
+@login_required
+@login_analyst
 def pgp_by_type_json():
     type_id = request.args.get('type_id')
     date_from = request.args.get('date_from')
@@ -1146,6 +1165,8 @@ def pgp_by_type_json():
 
 ############################ Correlation ############################
 @hashDecoded.route("/correlation/pgpdump", methods=['GET'])
+@login_required
+@login_analyst
 def pgpdump_page():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -1156,6 +1177,8 @@ def pgpdump_page():
     return res
 
 @hashDecoded.route("/correlation/cryptocurrency", methods=['GET'])
+@login_required
+@login_analyst
 def cryptocurrency_page():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -1166,6 +1189,8 @@ def cryptocurrency_page():
     return res
 
 @hashDecoded.route("/correlation/all_pgpdump_search", methods=['POST'])
+@login_required
+@login_analyst
 def all_pgpdump_search():
     date_from = request.form.get('date_from')
     date_to = request.form.get('date_to')
@@ -1174,6 +1199,8 @@ def all_pgpdump_search():
     return redirect(url_for('hashDecoded.pgpdump_page', date_from=date_from, date_to=date_to, type_id=type_id, show_decoded_files=show_decoded_files))
 
 @hashDecoded.route("/correlation/all_cryptocurrency_search", methods=['POST'])
+@login_required
+@login_analyst
 def all_cryptocurrency_search():
     date_from = request.form.get('date_from')
     date_to = request.form.get('date_to')
@@ -1182,6 +1209,8 @@ def all_cryptocurrency_search():
     return redirect(url_for('hashDecoded.cryptocurrency_page', date_from=date_from, date_to=date_to, type_id=type_id, show_decoded_files=show_decoded_files))
 
 @hashDecoded.route('/correlation/show_pgpdump')
+@login_required
+@login_analyst
 def show_pgpdump():
     type_id = request.args.get('type_id')
     key_id = request.args.get('key_id')
@@ -1189,36 +1218,48 @@ def show_pgpdump():
 
 
 @hashDecoded.route('/correlation/show_cryptocurrency')
+@login_required
+@login_analyst
 def show_cryptocurrency():
     type_id = request.args.get('type_id')
     key_id = request.args.get('key_id')
     return show_correlation('cryptocurrency', type_id, key_id)
 
 @hashDecoded.route('/correlation/cryptocurrency_range_type_json')
+@login_required
+@login_analyst
 def cryptocurrency_range_type_json():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
     return correlation_type_range_type_json('cryptocurrency', date_from, date_to)
 
 @hashDecoded.route('/correlation/pgpdump_range_type_json')
+@login_required
+@login_analyst
 def pgpdump_range_type_json():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
     return correlation_type_range_type_json('pgpdump', date_from, date_to)
 
 @hashDecoded.route('/correlation/pgpdump_graph_node_json')
+@login_required
+@login_analyst
 def pgpdump_graph_node_json():
     type_id = request.args.get('type_id')
     key_id = request.args.get('key_id')
     return correlation_graph_node_json('pgpdump', type_id, key_id)
 
 @hashDecoded.route('/correlation/cryptocurrency_graph_node_json')
+@login_required
+@login_analyst
 def cryptocurrency_graph_node_json():
     type_id = request.args.get('type_id')
     key_id = request.args.get('key_id')
     return correlation_graph_node_json('cryptocurrency', type_id, key_id)
 
 @hashDecoded.route('/correlation/pgpdump_graph_line_json')
+@login_required
+@login_analyst
 def pgpdump_graph_line_json():
     type_id = request.args.get('type_id')
     key_id = request.args.get('key_id')
@@ -1251,6 +1292,8 @@ def correlation_graph_line_json(correlation_type, type_id, key_id, date_from, da
         return jsonify()
 
 @hashDecoded.route('/correlation/cryptocurrency_graph_line_json')
+@login_required
+@login_analyst
 def cryptocurrency_graph_line_json():
     type_id = request.args.get('type_id')
     key_id = request.args.get('key_id')
diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py
index 09d1df1f..12fe3177 100644
--- a/var/www/modules/hiddenServices/Flask_hiddenServices.py
+++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py
@@ -12,6 +12,8 @@ import time
 import json
 from pyfaup.faup import Faup
 from flask import Flask, render_template, jsonify, request, send_file, Blueprint, redirect, url_for
+
+from Role_Manager import login_admin, login_analyst
 from flask_login import login_required
 
 from Date import Date
@@ -241,6 +243,7 @@ def delete_auto_crawler(url):
 
 @hiddenServices.route("/crawlers/", methods=['GET'])
 @login_required
+@login_analyst
 def dashboard():
     crawler_metadata_onion = get_crawler_splash_status('onion')
     crawler_metadata_regular = get_crawler_splash_status('regular')
@@ -255,18 +258,15 @@ def dashboard():
                                 crawler_metadata_regular=crawler_metadata_regular,
                                 statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular)
 
-@hiddenServices.route("/hiddenServices/2", methods=['GET'])
-@login_required
-def hiddenServices_page_test():
-    return render_template("Crawler_index.html")
-
 @hiddenServices.route("/crawlers/manual", methods=['GET'])
 @login_required
+@login_analyst
 def manual():
     return render_template("Crawler_Splash_manual.html", crawler_enabled=crawler_enabled)
 
 @hiddenServices.route("/crawlers/crawler_splash_onion", methods=['GET'])
 @login_required
+@login_analyst
 def crawler_splash_onion():
     type = 'onion'
     last_onions = get_last_domains_crawled(type)
@@ -285,6 +285,7 @@ def crawler_splash_onion():
 
 @hiddenServices.route("/crawlers/Crawler_Splash_last_by_type", methods=['GET'])
 @login_required
+@login_analyst
 def Crawler_Splash_last_by_type():
     type = request.args.get('type')
     # verify user input
@@ -309,6 +310,7 @@ def Crawler_Splash_last_by_type():
 
 @hiddenServices.route("/crawlers/blacklisted_domains", methods=['GET'])
 @login_required
+@login_analyst
 def blacklisted_domains():
     blacklist_domain = request.args.get('blacklist_domain')
     unblacklist_domain = request.args.get('unblacklist_domain')
@@ -344,6 +346,7 @@ def blacklisted_domains():
 
 @hiddenServices.route("/crawler/blacklist_domain", methods=['GET'])
 @login_required
+@login_analyst
 def blacklist_domain():
     domain = request.args.get('domain')
     type = request.args.get('type')
@@ -366,6 +369,7 @@ def blacklist_domain():
 
 @hiddenServices.route("/crawler/unblacklist_domain", methods=['GET'])
 @login_required
+@login_analyst
 def unblacklist_domain():
     domain = request.args.get('domain')
     type = request.args.get('type')
@@ -388,6 +392,7 @@ def unblacklist_domain():
 
 @hiddenServices.route("/crawlers/create_spider_splash", methods=['POST'])
 @login_required
+@login_analyst
 def create_spider_splash():
     url = request.form.get('url_to_crawl')
     automatic = request.form.get('crawler_type')
@@ -475,6 +480,7 @@ def create_spider_splash():
 
 @hiddenServices.route("/crawlers/auto_crawler", methods=['GET'])
 @login_required
+@login_analyst
 def auto_crawler():
     nb_element_to_display = 100
     try:
@@ -528,6 +534,7 @@ def auto_crawler():
 
 @hiddenServices.route("/crawlers/remove_auto_crawler", methods=['GET'])
 @login_required
+@login_analyst
 def remove_auto_crawler():
     url = request.args.get('url')
     page = request.args.get('page')
@@ -538,6 +545,7 @@ def remove_auto_crawler():
 
 @hiddenServices.route("/crawlers/crawler_dashboard_json", methods=['GET'])
 @login_required
+@login_analyst
 def crawler_dashboard_json():
 
     crawler_metadata_onion = get_crawler_splash_status('onion')
@@ -555,6 +563,7 @@ def crawler_dashboard_json():
 # # TODO: refractor
 @hiddenServices.route("/hiddenServices/last_crawled_domains_with_stats_json", methods=['GET'])
 @login_required
+@login_analyst
 def last_crawled_domains_with_stats_json():
     last_onions = r_serv_onion.lrange('last_onion', 0 ,-1)
     list_onion = []
@@ -605,6 +614,7 @@ def last_crawled_domains_with_stats_json():
 
 @hiddenServices.route("/hiddenServices/get_onions_by_daterange", methods=['POST'])
 @login_required
+@login_analyst
 def get_onions_by_daterange():
     date_from = request.form.get('date_from')
     date_to = request.form.get('date_to')
@@ -617,6 +627,7 @@ def get_onions_by_daterange():
 
 @hiddenServices.route("/hiddenServices/show_domains_by_daterange", methods=['GET'])
 @login_required
+@login_analyst
 def show_domains_by_daterange():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
@@ -722,6 +733,7 @@ def show_domains_by_daterange():
 
 @hiddenServices.route("/crawlers/show_domain", methods=['GET'])
 @login_required
+@login_analyst
 def show_domain():
     domain = request.args.get('domain')
     epoch = request.args.get('epoch')
@@ -805,6 +817,8 @@ def show_domain():
                             domain_tags=domain_tags, screenshot=screenshot)
 
 @hiddenServices.route("/crawlers/download_domain", methods=['GET'])
+@login_required
+@login_analyst
 def download_domain():
     domain = request.args.get('domain')
     epoch = request.args.get('epoch')
@@ -857,6 +871,7 @@ def download_domain():
 
 @hiddenServices.route("/hiddenServices/onion_son", methods=['GET'])
 @login_required
+@login_analyst
 def onion_son():
     onion_domain = request.args.get('onion_domain')
 
@@ -868,6 +883,7 @@ def onion_son():
 # ============= JSON ==============
 @hiddenServices.route("/hiddenServices/domain_crawled_7days_json", methods=['GET'])
 @login_required
+@login_analyst
 def domain_crawled_7days_json():
     type = 'onion'
         ## TODO: # FIXME: 404 error
@@ -887,6 +903,7 @@ def domain_crawled_7days_json():
 
 @hiddenServices.route('/hiddenServices/domain_crawled_by_type_json')
 @login_required
+@login_analyst
 def domain_crawled_by_type_json():
     current_date = request.args.get('date')
     type = request.args.get('type')
diff --git a/var/www/modules/rawSkeleton/Flask_rawSkeleton.py b/var/www/modules/rawSkeleton/Flask_rawSkeleton.py
index fe6e1f66..d767a83c 100644
--- a/var/www/modules/rawSkeleton/Flask_rawSkeleton.py
+++ b/var/www/modules/rawSkeleton/Flask_rawSkeleton.py
@@ -6,6 +6,8 @@
 '''
 import redis
 from flask import Flask, render_template, jsonify, request, Blueprint
+
+from Role_Manager import login_admin, login_analyst
 from flask_login import login_required
 
 # ============ VARIABLES ============
@@ -24,6 +26,7 @@ def one():
 
 @rawSkeleton.route("/rawSkeleton/", methods=['GET'])
 @login_required
+@login_analyst
 def skeleton_page():
     return render_template("rawSkeleton.html")
 
diff --git a/var/www/modules/search/Flask_search.py b/var/www/modules/search/Flask_search.py
index 866c0bfc..67a518fb 100644
--- a/var/www/modules/search/Flask_search.py
+++ b/var/www/modules/search/Flask_search.py
@@ -10,6 +10,8 @@ import os
 import datetime
 import flask
 from flask import Flask, render_template, jsonify, request, Blueprint
+
+from Role_Manager import login_admin, login_analyst
 from flask_login import login_required
 
 import Paste
@@ -95,6 +97,7 @@ def to_iso_date(timestamp):
 
 @searches.route("/search", methods=['POST'])
 @login_required
+@login_analyst
 def search():
     query = request.form['query']
     q = []
@@ -183,6 +186,7 @@ def search():
 
 @searches.route("/get_more_search_result", methods=['POST'])
 @login_required
+@login_analyst
 def get_more_search_result():
     query = request.form['query']
     q = []
diff --git a/var/www/modules/sentiment/Flask_sentiment.py b/var/www/modules/sentiment/Flask_sentiment.py
index 14904558..af6c220c 100644
--- a/var/www/modules/sentiment/Flask_sentiment.py
+++ b/var/www/modules/sentiment/Flask_sentiment.py
@@ -10,6 +10,8 @@ import calendar
 from Date import Date
 import flask
 from flask import Flask, render_template, jsonify, request, Blueprint
+
+from Role_Manager import login_admin, login_analyst
 from flask_login import login_required
 
 import Paste
@@ -41,12 +43,14 @@ def get_date_range(num_day):
 
 @sentiments.route("/sentiment_analysis_trending/")
 @login_required
+@login_analyst
 def sentiment_analysis_trending():
     return render_template("sentiment_analysis_trending.html")
 
 
 @sentiments.route("/sentiment_analysis_getplotdata/", methods=['GET'])
 @login_required
+@login_analyst
 def sentiment_analysis_getplotdata():
     # Get the top providers based on number of pastes
     oneHour = 60*60
@@ -98,6 +102,7 @@ def sentiment_analysis_getplotdata():
 
 @sentiments.route("/sentiment_analysis_plot_tool/")
 @login_required
+@login_analyst
 def sentiment_analysis_plot_tool():
     return render_template("sentiment_analysis_plot_tool.html")
 
@@ -105,6 +110,7 @@ def sentiment_analysis_plot_tool():
 
 @sentiments.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET'])
 @login_required
+@login_analyst
 def sentiment_analysis_plot_tool_getdata():
     getProviders = request.args.get('getProviders')
 
diff --git a/var/www/modules/settings/Flask_settings.py b/var/www/modules/settings/Flask_settings.py
index 8a5fb776..09d5597d 100644
--- a/var/www/modules/settings/Flask_settings.py
+++ b/var/www/modules/settings/Flask_settings.py
@@ -7,7 +7,8 @@
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
 from flask_login import login_required, current_user
 
-from Role_Manager import login_admin, login_analyst, create_user_db, edit_user_db, delete_user_db, check_password_strength
+from Role_Manager import login_admin, login_analyst
+from Role_Manager import create_user_db, edit_user_db, delete_user_db, check_password_strength
 
 import json
 import secrets
@@ -104,6 +105,7 @@ def get_all_roles():
 
 @settings.route("/settings/", methods=['GET'])
 @login_required
+@login_analyst
 def settings_page():
     git_metadata = get_git_metadata()
     current_version = r_serv_db.get('ail:version')
@@ -114,18 +116,21 @@ def settings_page():
 
 @settings.route("/settings/edit_profile", methods=['GET'])
 @login_required
+@login_analyst
 def edit_profile():
     user_metadata = get_user_metadata(current_user.get_id())
     return render_template("edit_profile.html", user_metadata=user_metadata)
 
 @settings.route("/settings/new_token", methods=['GET'])
 @login_required
+@login_analyst
 def new_token():
     generate_new_token(current_user.get_id())
     return redirect(url_for('settings.edit_profile'))
 
 @settings.route("/settings/new_token_user", methods=['GET'])
 @login_required
+@login_admin
 def new_token_user():
     user_id = request.args.get('user_id')
     if r_serv_db.exists('user_metadata:{}'.format(user_id)):
@@ -134,6 +139,7 @@ def new_token_user():
 
 @settings.route("/settings/create_user", methods=['GET'])
 @login_required
+@login_admin
 def create_user():
     user_id = request.args.get('user_id')
     role = None
@@ -146,6 +152,7 @@ def create_user():
 
 @settings.route("/settings/create_user_post", methods=['POST'])
 @login_required
+@login_admin
 def create_user_post():
     email = request.form.get('username')
     role = request.form.get('user_role')
@@ -190,6 +197,7 @@ def create_user_post():
 
 @settings.route("/settings/users_list", methods=['GET'])
 @login_required
+@login_admin
 def users_list():
     all_users = get_users_metadata(get_all_users())
     new_user = request.args.get('new_user')
@@ -202,12 +210,14 @@ def users_list():
 
 @settings.route("/settings/edit_user", methods=['GET'])
 @login_required
+@login_admin
 def edit_user():
     user_id = request.args.get('user_id')
     return redirect(url_for('settings.create_user', user_id=user_id))
 
 @settings.route("/settings/delete_user", methods=['GET'])
 @login_required
+@login_admin
 def delete_user():
     user_id = request.args.get('user_id')
     delete_user_db(user_id)
@@ -216,6 +226,7 @@ def delete_user():
 
 @settings.route("/settings/get_background_update_stats_json", methods=['GET'])
 @login_required
+@login_analyst
 def get_background_update_stats_json():
     # handle :end, error
     update_stats = {}
diff --git a/var/www/modules/showpaste/Flask_showpaste.py b/var/www/modules/showpaste/Flask_showpaste.py
index dafba343..fb990bbf 100644
--- a/var/www/modules/showpaste/Flask_showpaste.py
+++ b/var/www/modules/showpaste/Flask_showpaste.py
@@ -9,6 +9,8 @@ import json
 import os
 import flask
 from flask import Flask, render_template, jsonify, request, Blueprint, make_response, Response, send_from_directory, redirect, url_for
+
+from Role_Manager import login_admin, login_analyst
 from flask_login import login_required
 
 import difflib
@@ -383,18 +385,21 @@ def show_item_min(requested_path , content_range=0):
 
 @showsavedpastes.route("/showsavedpaste/") #completely shows the paste in a new tab
 @login_required
+@login_analyst
 def showsavedpaste():
     requested_path = request.args.get('paste', '')
     return showpaste(0, requested_path)
 
 @showsavedpastes.route("/showsaveditem_min/") #completely shows the paste in a new tab
 @login_required
+@login_analyst
 def showsaveditem_min():
     requested_path = request.args.get('paste', '')
     return show_item_min(requested_path)
 
 @showsavedpastes.route("/showsavedrawpaste/") #shows raw
 @login_required
+@login_analyst
 def showsavedrawpaste():
     requested_path = request.args.get('paste', '')
     paste = Paste.Paste(requested_path)
@@ -403,6 +408,7 @@ def showsavedrawpaste():
 
 @showsavedpastes.route("/showpreviewpaste/")
 @login_required
+@login_analyst
 def showpreviewpaste():
     num = request.args.get('num', '')
     requested_path = request.args.get('paste', '')
@@ -411,6 +417,7 @@ def showpreviewpaste():
 
 @showsavedpastes.route("/getmoredata/")
 @login_required
+@login_analyst
 def getmoredata():
     requested_path = request.args.get('paste', '')
     paste = Paste.Paste(requested_path)
@@ -420,6 +427,7 @@ def getmoredata():
 
 @showsavedpastes.route("/showDiff/")
 @login_required
+@login_analyst
 def showDiff():
     s1 = request.args.get('s1', '')
     s2 = request.args.get('s2', '')
@@ -437,11 +445,13 @@ def showDiff():
 
 @showsavedpastes.route('/screenshot/<path:filename>')
 @login_required
+@login_analyst
 def screenshot(filename):
     return send_from_directory(SCREENSHOT_FOLDER, filename+'.png', as_attachment=True)
 
 @showsavedpastes.route('/send_file_to_vt/', methods=['POST'])
 @login_required
+@login_analyst
 def send_file_to_vt():
     b64_path = request.form['b64_path']
     paste = request.form['paste']
diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py
index fd42ec4d..f3b8c7de 100644
--- a/var/www/modules/terms/Flask_terms.py
+++ b/var/www/modules/terms/Flask_terms.py
@@ -11,6 +11,8 @@ import datetime
 import calendar
 import flask
 from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect
+
+from Role_Manager import login_admin, login_analyst
 from flask_login import login_required
 
 import re
@@ -146,6 +148,7 @@ def save_tag_to_auto_push(list_tag):
 
 @terms.route("/terms_management/")
 @login_required
+@login_analyst
 def terms_management():
     per_paste = request.args.get('per_paste')
     if per_paste == "1" or per_paste is None:
@@ -265,6 +268,7 @@ def terms_management():
 
 @terms.route("/terms_management_query_paste/")
 @login_required
+@login_analyst
 def terms_management_query_paste():
     term =  request.args.get('term')
     paste_info = []
@@ -298,6 +302,7 @@ def terms_management_query_paste():
 
 @terms.route("/terms_management_query/")
 @login_required
+@login_analyst
 def terms_management_query():
     TrackedTermsDate_Name = "TrackedTermDate"
     BlackListTermsDate_Name = "BlackListTermDate"
@@ -321,6 +326,7 @@ def terms_management_query():
 
 @terms.route("/terms_management_action/", methods=['GET'])
 @login_required
+@login_analyst
 def terms_management_action():
     today = datetime.datetime.now()
     today = today.replace(microsecond=0)
@@ -447,6 +453,7 @@ def terms_management_action():
 
 @terms.route("/terms_management/delete_terms_tags", methods=['POST'])
 @login_required
+@login_analyst
 def delete_terms_tags():
     term = request.form.get('term')
     tags_to_delete = request.form.getlist('tags_to_delete')
@@ -460,6 +467,7 @@ def delete_terms_tags():
 
 @terms.route("/terms_management/delete_terms_email", methods=['GET'])
 @login_required
+@login_analyst
 def delete_terms_email():
     term =  request.args.get('term')
     email =  request.args.get('email')
@@ -473,6 +481,7 @@ def delete_terms_email():
 
 @terms.route("/terms_plot_tool/")
 @login_required
+@login_analyst
 def terms_plot_tool():
     term =  request.args.get('term')
     if term is not None:
@@ -483,6 +492,7 @@ def terms_plot_tool():
 
 @terms.route("/terms_plot_tool_data/")
 @login_required
+@login_analyst
 def terms_plot_tool_data():
     oneDay = 60*60*24
     range_start =  datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
@@ -514,6 +524,7 @@ def terms_plot_tool_data():
 
 @terms.route("/terms_plot_top/")
 @login_required
+@login_analyst
 def terms_plot_top():
     per_paste = request.args.get('per_paste')
     per_paste = per_paste if per_paste is not None else 1
@@ -522,6 +533,7 @@ def terms_plot_top():
 
 @terms.route("/terms_plot_top_data/")
 @login_required
+@login_analyst
 def terms_plot_top_data():
     oneDay = 60*60*24
     today = datetime.datetime.now()
@@ -569,11 +581,13 @@ def terms_plot_top_data():
 
 @terms.route("/credentials_tracker/")
 @login_required
+@login_analyst
 def credentials_tracker():
     return render_template("credentials_tracker.html")
 
 @terms.route("/credentials_management_query_paste/", methods=['GET', 'POST'])
 @login_required
+@login_analyst
 def credentials_management_query_paste():
     cred =  request.args.get('cred')
     allPath = request.json['allPath']
@@ -598,6 +612,7 @@ def credentials_management_query_paste():
 
 @terms.route("/credentials_management_action/", methods=['GET'])
 @login_required
+@login_analyst
 def cred_management_action():
 
     supplied =  request.args.get('term')
diff --git a/var/www/modules/trendingcharts/Flask_trendingcharts.py b/var/www/modules/trendingcharts/Flask_trendingcharts.py
index bad6a353..a037e171 100644
--- a/var/www/modules/trendingcharts/Flask_trendingcharts.py
+++ b/var/www/modules/trendingcharts/Flask_trendingcharts.py
@@ -9,6 +9,8 @@ import datetime
 from Date import Date
 import flask
 from flask import Flask, render_template, jsonify, request, Blueprint
+
+from Role_Manager import login_admin, login_analyst
 from flask_login import login_required
 
 # ============ VARIABLES ============
@@ -38,6 +40,7 @@ def get_date_range(num_day):
 
 @trendings.route("/_progressionCharts", methods=['GET'])
 @login_required
+@login_analyst
 def progressionCharts():
     attribute_name = request.args.get('attributeName')
     trending_name = request.args.get('trendingName')
@@ -64,6 +67,7 @@ def progressionCharts():
 
 @trendings.route("/wordstrending/")
 @login_required
+@login_analyst
 def wordstrending():
     default_display = cfg.get("Flask", "default_display")
     return render_template("Wordstrending.html", default_display = default_display)
@@ -71,6 +75,7 @@ def wordstrending():
 
 @trendings.route("/protocolstrending/")
 @login_required
+@login_analyst
 def protocolstrending():
     default_display = cfg.get("Flask", "default_display")
     return render_template("Protocolstrending.html", default_display = default_display)
@@ -78,6 +83,7 @@ def protocolstrending():
 
 @trendings.route("/trending/")
 @login_required
+@login_analyst
 def trending():
     default_display = cfg.get("Flask", "default_display")
     return render_template("Trending.html", default_display = default_display)
diff --git a/var/www/modules/trendingmodules/Flask_trendingmodules.py b/var/www/modules/trendingmodules/Flask_trendingmodules.py
index a53066b9..80646ecb 100644
--- a/var/www/modules/trendingmodules/Flask_trendingmodules.py
+++ b/var/www/modules/trendingmodules/Flask_trendingmodules.py
@@ -9,6 +9,8 @@ import datetime
 from Date import Date
 import flask
 from flask import Flask, render_template, jsonify, request, Blueprint
+
+from Role_Manager import login_admin, login_analyst
 from flask_login import login_required
 
 # ============ VARIABLES ============
@@ -51,6 +53,7 @@ def get_date_range(num_day):
 
 @trendingmodules.route("/_moduleCharts", methods=['GET'])
 @login_required
+@login_analyst
 def modulesCharts():
     keyword_name = request.args.get('keywordName')
     module_name = request.args.get('moduleName')
@@ -78,6 +81,7 @@ def modulesCharts():
 
 @trendingmodules.route("/_providersChart", methods=['GET'])
 @login_required
+@login_analyst
 def providersChart():
     keyword_name = request.args.get('keywordName')
     module_name = request.args.get('moduleName')
@@ -125,6 +129,7 @@ def providersChart():
 
 @trendingmodules.route("/moduletrending/")
 @login_required
+@login_analyst
 def moduletrending():
     return render_template("Moduletrending.html")
 
diff --git a/var/www/templates/error/403.html b/var/www/templates/error/403.html
new file mode 100644
index 00000000..e90433ba
--- /dev/null
+++ b/var/www/templates/error/403.html
@@ -0,0 +1,49 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>403 - AIL</title>
+  <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+</head>
+
+<body>
+
+{% include 'nav_bar.html' %}
+
+<div>
+  <br>
+  <br>
+  <h1 class="text-center">403 Forbidden</h1>
+</div>
+<br>
+<br>
+<br>
+<br>
+<div class="d-flex justify-content-center">
+<pre>
+                                  ,d8       ,a8888a,     ad888888b,
+                                ,d888     ,8P"'  `"Y8,  d8"     "88
+                              ,d8" 88    ,8P        Y8,         a8P
+                            ,d8"   88    88          88      aad8"
+                          ,d8"     88    88          88      ""Y8,
+                          8888888888888  `8b        d8'         "8b
+                                   88     `8ba,  ,ad8'  Y8,     a88
+                                   88       "Y8888P"     "Y888888P'
+
+88888888888                       88           88           88           88
+88                                88           ""           88           88
+88                                88                        88           88
+88aaaaa   ,adPPYba,   8b,dPPYba,  88,dPPYba,   88   ,adPPYb,88   ,adPPYb,88   ,adPPYba,  8b,dPPYba,
+88"""""  a8"     "8a  88P'   "Y8  88P'    "8a  88  a8"    `Y88  a8"    `Y88  a8P_____88  88P'   `"8a
+88       8b       d8  88          88       d8  88  8b       88  8b       88  8PP"""""""  88       88
+88       "8a,   ,a8"  88          88b,   ,a8"  88  "8a,   ,d88  "8a,   ,d88  "8b,   ,aa  88       88
+88        `"YbbdP"'   88          8Y"Ybbd8"'   88   `"8bbdP"Y8   `"8bbdP"Y8   `"Ybbd8"'  88       88
+</pre>
+</div>
+
+<body>
+
+</html>

From e4ab9b6a054163245b1451ae74314764de0109d0 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Thu, 20 Jun 2019 10:11:23 +0200
Subject: [PATCH 10/22] fix: [server endpoint] unknow users: avoid endpoint
 enumeration

---
 var/www/Flask_server.py          |  7 ++++++
 var/www/templates/error/404.html | 40 ++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 var/www/templates/error/404.html

diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py
index 09f9ff41..a47264ee 100755
--- a/var/www/Flask_server.py
+++ b/var/www/Flask_server.py
@@ -224,6 +224,13 @@ def role():
 def searchbox():
     return render_template("searchbox.html")
 
+# ========== ERROR HANDLER ============
+
+@app.errorhandler(404)
+@login_required
+def page_not_found(e):
+    # note that we set the 404 status explicitly
+    return render_template('error/404.html'), 404
 
 # ========== INITIAL taxonomies ============
 # add default ail taxonomies
diff --git a/var/www/templates/error/404.html b/var/www/templates/error/404.html
new file mode 100644
index 00000000..9b25da48
--- /dev/null
+++ b/var/www/templates/error/404.html
@@ -0,0 +1,40 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>403 - AIL</title>
+  <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+</head>
+
+<body>
+
+{% include 'nav_bar.html' %}
+
+<div>
+  <br>
+  <br>
+  <h1 class="text-center">404 Not Found</h1>
+</div>
+<br>
+<br>
+<br>
+<br>
+<div class="d-flex justify-content-center">
+<pre>
+        ,d8       ,a8888a,              ,d8
+      ,d888     ,8P"'  `"Y8,          ,d888
+    ,d8" 88    ,8P        Y8,       ,d8" 88
+  ,d8"   88    88          88     ,d8"   88
+,d8"     88    88          88   ,d8"     88
+8888888888888  `8b        d8'   8888888888888
+         88     `8ba,  ,ad8'             88
+         88       "Y8888P"               88
+</pre>
+</div>
+
+<body>
+
+</html>

From 7ecd43db99209738fadc4959f0c7231844e55dd8 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Thu, 20 Jun 2019 10:56:31 +0200
Subject: [PATCH 11/22] chg: [UI user_management] incorrect passwords: display
 errors

---
 var/www/Flask_server.py                       | 26 ++++++++++++-------
 var/www/modules/Flask_config.py               |  4 +++
 var/www/modules/settings/Flask_settings.py    | 20 ++++++++++----
 .../settings/templates/create_user.html       | 20 +++++++++++---
 var/www/templates/change_password.html        | 11 +++++---
 var/www/templates/error/404.html              |  2 +-
 6 files changed, 62 insertions(+), 21 deletions(-)

diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py
index a47264ee..71827852 100755
--- a/var/www/Flask_server.py
+++ b/var/www/Flask_server.py
@@ -193,18 +193,26 @@ def login():
 def change_password():
     password1 = request.form.get('password1')
     password2 = request.form.get('password2')
+    error = request.args.get('error')
 
-    # # TODO: display errors message
+    if error:
+        return render_template("change_password.html", error=error)
 
-    if current_user.is_authenticated and password1!=None and password1==password2:
-        if check_password_strength(password1):
-            user_id = current_user.get_id()
-            create_user_db(user_id , password1, update=True)
-            return redirect(url_for('dashboard.index'))
+    if current_user.is_authenticated and password1!=None:
+        if password1==password2:
+            if check_password_strength(password1):
+                user_id = current_user.get_id()
+                create_user_db(user_id , password1, update=True)
+                return redirect(url_for('dashboard.index'))
+            else:
+                error = 'Incorrect password'
+                return render_template("change_password.html", error=error)
         else:
-            return render_template("change_password.html")
+            error = "Passwords don't match"
+            return render_template("change_password.html", error=error)
     else:
-        return render_template("change_password.html")
+        error = 'Please choose a new password'
+        return render_template("change_password.html", error=error)
 
 @app.route('/logout')
 @login_required
@@ -229,7 +237,7 @@ def searchbox():
 @app.errorhandler(404)
 @login_required
 def page_not_found(e):
-    # note that we set the 404 status explicitly
+    # avoid endpoint enumeration
     return render_template('error/404.html'), 404
 
 # ========== INITIAL taxonomies ============
diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py
index 899a26b5..29ae9eae 100644
--- a/var/www/modules/Flask_config.py
+++ b/var/www/modules/Flask_config.py
@@ -7,6 +7,7 @@
 import configparser
 import redis
 import os
+import re
 import sys
 
 # FLASK #
@@ -175,6 +176,9 @@ max_dashboard_logs = int(cfg.get("Flask", "max_dashboard_logs"))
 
 crawler_enabled = cfg.getboolean("Crawler", "activate_crawler")
 
+email_regex = r'[^@]+@[^@]+\.[^@]+'
+email_regex = re.compile(email_regex)
+
 # VT
 try:
     from virusTotalKEYS import vt_key
diff --git a/var/www/modules/settings/Flask_settings.py b/var/www/modules/settings/Flask_settings.py
index 09d5597d..2914213d 100644
--- a/var/www/modules/settings/Flask_settings.py
+++ b/var/www/modules/settings/Flask_settings.py
@@ -27,6 +27,7 @@ max_preview_char = Flask_config.max_preview_char
 max_preview_modal = Flask_config.max_preview_modal
 REPO_ORIGIN = Flask_config.REPO_ORIGIN
 dict_update_description = Flask_config.dict_update_description
+email_regex = Flask_config.email_regex
 
 settings = Blueprint('settings', __name__, template_folder='templates')
 
@@ -36,6 +37,13 @@ settings = Blueprint('settings', __name__, template_folder='templates')
 def one():
     return 1
 
+def check_email(email):
+    result = email_regex.match(email)
+    if result:
+        return True
+    else:
+        return False
+
 def generate_new_token(user_id):
     # create user token
     current_token = r_serv_db.hget('user_metadata:{}'.format(user_id), 'token')
@@ -142,13 +150,15 @@ def new_token_user():
 @login_admin
 def create_user():
     user_id = request.args.get('user_id')
+    error = request.args.get('error')
+    error_mail = request.args.get('error_mail')
     role = None
     if r_serv_db.exists('user_metadata:{}'.format(user_id)):
         role = r_serv_db.hget('user_metadata:{}'.format(user_id), 'role')
     else:
         user_id = None
     all_roles = get_all_roles()
-    return render_template("create_user.html", all_roles=all_roles, user_id=user_id, user_role=role)
+    return render_template("create_user.html", all_roles=all_roles, user_id=user_id, user_role=role, error=error, error_mail=error_mail)
 
 @settings.route("/settings/create_user_post", methods=['POST'])
 @login_required
@@ -161,7 +171,7 @@ def create_user_post():
 
     all_roles = get_all_roles()
 
-    if email and len(email)< 300 and role:
+    if email and len(email)< 300 and check_email(email) and role:
         if role in all_roles:
             # password set
             if password1 and password2:
@@ -169,9 +179,9 @@ def create_user_post():
                     if check_password_strength(password1):
                         password = password1
                     else:
-                        return render_template("create_user.html", all_roles=all_roles)
+                        return render_template("create_user.html", all_roles=all_roles, error="Incorrect Password")
                 else:
-                    return render_template("create_user.html", all_roles=all_roles)
+                    return render_template("create_user.html", all_roles=all_roles, error="Passwords don't match")
             # generate password
             else:
                 password = secrets.token_urlsafe()
@@ -193,7 +203,7 @@ def create_user_post():
         else:
             return render_template("create_user.html", all_roles=all_roles)
     else:
-        return render_template("create_user.html", all_roles=all_roles)
+        return render_template("create_user.html", all_roles=all_roles, error_mail=True)
 
 @settings.route("/settings/users_list", methods=['GET'])
 @login_required
diff --git a/var/www/modules/settings/templates/create_user.html b/var/www/modules/settings/templates/create_user.html
index 6da31be3..139a2a90 100644
--- a/var/www/modules/settings/templates/create_user.html
+++ b/var/www/modules/settings/templates/create_user.html
@@ -33,7 +33,12 @@
 
           <h1 class="h3 mt-1 mb-3 text-center text-secondary">Create User</h1>
           <label for="inputEmail" class="sr-only">Email address</label>
-          <input type="email" id="inputEmail" name="username" class="form-control" placeholder="Email address" autocomplete="off" required {% if user_id %}value="{{user_id}}"{% else %}{% endif %}>
+          <input type="email" id="inputEmail" name="username" class="form-control {% if error_mail %}is-invalid{% endif %}" placeholder="Email address" autocomplete="off" required {% if user_id %}value="{{user_id}}"{% else %}{% endif %}>
+          {% if error_mail %}
+            <div class="invalid-feedback">
+              Please provide a valid email address
+            </div>
+          {% endif %}
 
           <label  class="mt-3" for="role_selector">User Role</label>
           <select class="custom-select" id="role_selector" name="user_role">
@@ -54,9 +59,14 @@
           <div id="password-section">
     	      <h1 class="h3 mb-3 text-center text-secondary">Create Password</h1>
     	      <label for="inputPassword1" class="sr-only">Password</label>
-    	      <input type="password" id="inputPassword1" name="password1" class="form-control" placeholder="Password" autocomplete="new-password">
+    	      <input type="password" id="inputPassword1" name="password1" class="form-control {% if error %}is-invalid{% endif %}" placeholder="Password" autocomplete="new-password">
     	      <label for="inputPassword2" class="sr-only">Confirm Password</label>
-    	      <input type="password" id="inputPassword2" name="password2" class="form-control" placeholder="Confirm Password"  value="" autocomplete="new-password">
+    	      <input type="password" id="inputPassword2" name="password2" class="form-control {% if error %}is-invalid{% endif %}" placeholder="Confirm Password"  value="" autocomplete="new-password">
+            {% if error %}
+  						<div class="invalid-feedback">
+  							{{error}}
+  						</div>
+  					{% endif %}
 
           </div>
           <button class="btn btn-lg btn-primary btn-block mt-3" type="submit">Submit</button>
@@ -104,6 +114,10 @@ $(document).ready(function(){
   $("#password-section-info").hide();
   $("#nav_create_user").addClass("active");
 	$("#nav_user_management").removeClass("text-muted");
+
+  {% if error %}
+    toggle_password_fields();
+  {% endif %}
 } );
 
 function toggle_password_fields() {
diff --git a/var/www/templates/change_password.html b/var/www/templates/change_password.html
index e5f13a84..7722d86e 100644
--- a/var/www/templates/change_password.html
+++ b/var/www/templates/change_password.html
@@ -65,10 +65,15 @@
 		      <img class="mb-4" src="{{ url_for('static', filename='image/logo-small.png')}}" width="300">
 		      <h1 class="h3 mb-3 text-secondary">Change Password</h1>
 		      <label for="inputPassword1" class="sr-only">Password</label>
-		      <input type="password" id="inputPassword1" name="password1" class="form-control" placeholder="Password" autocomplete="new-password" required autofocus>
+		      <input type="password" id="inputPassword1" name="password1" class="form-control {% if error %}is-invalid{% endif %}" placeholder="Password" autocomplete="new-password" required autofocus>
 		      <label for="inputPassword2" class="sr-only">Confirm Password</label>
-		      <input type="password" id="inputPassword2" name="password2" class="form-control" placeholder="Confirm Password"  value="" autocomplete="new-password" required>
-		      <button class="btn btn-lg btn-primary btn-block" type="submit">Submit</button>
+		      <input type="password" id="inputPassword2" name="password2" class="form-control {% if error %}is-invalid{% endif %}" placeholder="Confirm Password"  value="" autocomplete="new-password" required>
+					{% if error %}
+						<div class="invalid-feedback">
+							{{error}}
+						</div>
+					{% endif %}
+					<button class="btn btn-lg btn-primary btn-block" type="submit">Submit</button>
 
 					<br>
 					<br>
diff --git a/var/www/templates/error/404.html b/var/www/templates/error/404.html
index 9b25da48..6943c674 100644
--- a/var/www/templates/error/404.html
+++ b/var/www/templates/error/404.html
@@ -2,7 +2,7 @@
 <html>
 
 <head>
-  <title>403 - AIL</title>
+  <title>404 - AIL</title>
   <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
   <!-- Core CSS -->

From 821cf3cbea36beb6d4f2cda4850aa0c28508b861 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Thu, 20 Jun 2019 11:15:56 +0200
Subject: [PATCH 12/22] chg: [UI user_management] user_role acl: hide admin
 panel

---
 var/www/modules/settings/Flask_settings.py   | 21 ++++++----
 var/www/templates/settings/menu_sidebar.html | 40 ++++++++++----------
 2 files changed, 35 insertions(+), 26 deletions(-)

diff --git a/var/www/modules/settings/Flask_settings.py b/var/www/modules/settings/Flask_settings.py
index 2914213d..2080c637 100644
--- a/var/www/modules/settings/Flask_settings.py
+++ b/var/www/modules/settings/Flask_settings.py
@@ -119,7 +119,10 @@ def settings_page():
     current_version = r_serv_db.get('ail:version')
     update_metadata = get_update_metadata()
 
+    admin_level = current_user.is_in_role('admin')
+
     return render_template("settings_index.html", git_metadata=git_metadata,
+                            admin_level=admin_level,
                             current_version=current_version)
 
 @settings.route("/settings/edit_profile", methods=['GET'])
@@ -127,7 +130,9 @@ def settings_page():
 @login_analyst
 def edit_profile():
     user_metadata = get_user_metadata(current_user.get_id())
-    return render_template("edit_profile.html", user_metadata=user_metadata)
+    admin_level = current_user.is_in_role('admin')
+    return render_template("edit_profile.html", user_metadata=user_metadata,
+                            admin_level=admin_level)
 
 @settings.route("/settings/new_token", methods=['GET'])
 @login_required
@@ -158,7 +163,9 @@ def create_user():
     else:
         user_id = None
     all_roles = get_all_roles()
-    return render_template("create_user.html", all_roles=all_roles, user_id=user_id, user_role=role, error=error, error_mail=error_mail)
+    return render_template("create_user.html", all_roles=all_roles, user_id=user_id, user_role=role,
+                                        error=error, error_mail=error_mail,
+                                        admin_level=True)
 
 @settings.route("/settings/create_user_post", methods=['POST'])
 @login_required
@@ -179,9 +186,9 @@ def create_user_post():
                     if check_password_strength(password1):
                         password = password1
                     else:
-                        return render_template("create_user.html", all_roles=all_roles, error="Incorrect Password")
+                        return render_template("create_user.html", all_roles=all_roles, error="Incorrect Password", admin_level=True)
                 else:
-                    return render_template("create_user.html", all_roles=all_roles, error="Passwords don't match")
+                    return render_template("create_user.html", all_roles=all_roles, error="Passwords don't match", admin_level=True)
             # generate password
             else:
                 password = secrets.token_urlsafe()
@@ -201,9 +208,9 @@ def create_user_post():
                     return redirect(url_for('settings.users_list', new_user=email, new_user_password=password, new_user_edited=False))
 
         else:
-            return render_template("create_user.html", all_roles=all_roles)
+            return render_template("create_user.html", all_roles=all_roles, admin_level=True)
     else:
-        return render_template("create_user.html", all_roles=all_roles, error_mail=True)
+        return render_template("create_user.html", all_roles=all_roles, error_mail=True, admin_level=True)
 
 @settings.route("/settings/users_list", methods=['GET'])
 @login_required
@@ -216,7 +223,7 @@ def users_list():
         new_user_dict['email'] = new_user
         new_user_dict['edited'] = request.args.get('new_user_edited')
         new_user_dict['password'] = request.args.get('new_user_password')
-    return render_template("users_list.html", all_users=all_users, new_user=new_user_dict)
+    return render_template("users_list.html", all_users=all_users, new_user=new_user_dict, admin_level=True)
 
 @settings.route("/settings/edit_user", methods=['GET'])
 @login_required
diff --git a/var/www/templates/settings/menu_sidebar.html b/var/www/templates/settings/menu_sidebar.html
index f1af27d1..3fe196e4 100644
--- a/var/www/templates/settings/menu_sidebar.html
+++ b/var/www/templates/settings/menu_sidebar.html
@@ -37,23 +37,25 @@
       	    </li>
           </ul>
         </nav>
-        <nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_users">
-      		<h5 class="d-flex text-muted w-100" id="nav_user_management">
-      		  <span>User Management</span>
-      		</h5>
-          <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
-      	    <li class="nav-item">
-      	      <a class="nav-link" href="{{url_for('settings.create_user')}}" id="nav_create_user">
-      	        <i class="fas fa-user-plus"></i>
-      	        <span>Create User</span>
-      	      </a>
-      	    </li>
-            <li class="nav-item">
-      	      <a class="nav-link" href="{{url_for('settings.users_list')}}" id="nav_users_list">
-      	        <i class="fas fa-users"></i>
-      	        <span>Users List</span>
-      	      </a>
-      	    </li>
-          </ul>
-      	</nav>
+        {% if admin_level %}
+          <nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_users">
+        		<h5 class="d-flex text-muted w-100" id="nav_user_management">
+        		  <span>User Management</span>
+        		</h5>
+            <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
+        	    <li class="nav-item">
+        	      <a class="nav-link" href="{{url_for('settings.create_user')}}" id="nav_create_user">
+        	        <i class="fas fa-user-plus"></i>
+        	        <span>Create User</span>
+        	      </a>
+        	    </li>
+              <li class="nav-item">
+        	      <a class="nav-link" href="{{url_for('settings.users_list')}}" id="nav_users_list">
+        	        <i class="fas fa-users"></i>
+        	        <span>Users List</span>
+        	      </a>
+        	    </li>
+            </ul>
+        	</nav>
+        {% endif %}
       </div>

From a9837c6e2731f2b2f6f5ba30d9f9c6a084cd1372 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Thu, 20 Jun 2019 14:47:59 +0200
Subject: [PATCH 13/22] chg: [user_management 2.0] add update scripts + fix
 create_default_user

---
 .gitignore                                    |   2 +
 OVERVIEW.md                                   |   6 +-
 installing_deps.sh                            |   1 +
 installing_deps_archlinux.sh                  | 108 ----------
 update/v2.0/Update.py                         |  41 ++++
 update/v2.0/Update.sh                         |  62 ++++++
 var/www/Flask_server.py                       |  12 --
 var/www/create_default_user.py                |  34 +++-
 var/www/modules/restApi/Flask_restApi.py      |   8 +-
 .../restApi/templates/api_default.html        | 187 ------------------
 .../modules/restApi/templates/header_api.html |   1 -
 var/www/templates/error/403.html              |   1 +
 var/www/templates/error/404.html              |   1 +
 13 files changed, 145 insertions(+), 319 deletions(-)
 delete mode 100755 installing_deps_archlinux.sh
 create mode 100755 update/v2.0/Update.py
 create mode 100755 update/v2.0/Update.sh
 delete mode 100644 var/www/modules/restApi/templates/header_api.html

diff --git a/.gitignore b/.gitignore
index 785419e5..28ed8063 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,6 +20,8 @@ indexdir/
 logs/
 old/
 
+DEFAULT_PASSWORD
+
 # Webstuff
 var/www/static/
 !var/www/static/css/dygraph_gallery.css
diff --git a/OVERVIEW.md b/OVERVIEW.md
index 97991b57..74674bd4 100644
--- a/OVERVIEW.md
+++ b/OVERVIEW.md
@@ -42,8 +42,6 @@ Redis and ARDB overview
 | | **background update name** |
 | | **...** |
 | | |
-| ail:update_date_v1.5 | **update date** |
-| | |
 | ail:update_error | **update message error** |
 | | |
 | ail:update_in_progress | **update version in progress** |
@@ -52,6 +50,10 @@ Redis and ARDB overview
 | ail:current_background_script | **name of the background script currently executed** |
 | ail:current_background_script_stat | **progress in % of the background script** |
 
+| Hset Key | Field | Value |
+| ------ | ------ | ------ |
+| ail:update_date | **update tag** | **update date**  |
+
 ##### User Management:
 | Hset Key | Field | Value |
 | ------ | ------ | ------ |
diff --git a/installing_deps.sh b/installing_deps.sh
index e8de3fd6..bf72cf4b 100755
--- a/installing_deps.sh
+++ b/installing_deps.sh
@@ -101,6 +101,7 @@ fi
 
 pushd var/www/
 ./update_thirdparty.sh
+python3 create_default_user.py
 popd
 
 mkdir -p $AIL_HOME/PASTES
diff --git a/installing_deps_archlinux.sh b/installing_deps_archlinux.sh
deleted file mode 100755
index 38ac4ca7..00000000
--- a/installing_deps_archlinux.sh
+++ /dev/null
@@ -1,108 +0,0 @@
-#!/bin/bash
-
-
-echo "Currently unmaintained, continue at your own risk of not having a working AIL at the end :( Will be merged into main install deps later on."
-exit 1
-
-set -e
-set -x
-
-sudo pacman -Syu
-
-sudo pacman -S python2-pip screen gcc unzip freetype2 python2 git --needed
-sudo yaourt -S snappy --needed
-sudo pip2 install virtualenv
-
-#Needed for bloom filters
-sudo pacman -S openssl python2-numpy --needed
-
-# DNS deps
-sudo pacman -S adns --needed
-
-#Needed for redis-lvlDB
-sudo pacman -S libev gmp --needed
-
-#needed for mathplotlib
-test ! -L /usr/include/ft2build.h && sudo ln -s freetype2/ft2build.h /usr/include/
-sudo easy_install-2.7 -U distribute
-
-# REDIS #
-test ! -d redis/ && git clone https://github.com/antirez/redis.git
-pushd redis/
-git checkout 2.8
-make
-popd
-
-# REDIS LEVEL DB #
-test ! -d redis-leveldb/ && git clone https://github.com/KDr2/redis-leveldb.git
-pushd redis-leveldb/
-git submodule init
-git submodule update
-make
-popd
-
-# Faup
-test ! -d faup/ && git clone https://github.com/stricaud/faup.git
-pushd faup/
-test ! -d build && mkdir build
-cd build
-cmake .. && make
-sudo make install
-echo '/usr/local/lib' | sudo tee -a /etc/ld.so.conf.d/faup.conf
-sudo ldconfig
-popd
-
-# tlsh
-test ! -d tlsh && git clone git://github.com/trendmicro/tlsh.git
-pushd tlsh/
-./make.sh
-pushd build/release/
-sudo make install
-sudo ldconfig
-popd
-popd
-
-
-
-if [ ! -f bin/packages/config.cfg ]; then
-    cp bin/packages/config.cfg.sample bin/packages/config.cfg
-fi
-
-pushd var/www/
-./update_thirdparty.sh
-popd
-
-virtualenv AILENV
-
-echo export AIL_HOME=$(pwd) >> ./AILENV/bin/activate
-echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate
-echo export AIL_FLASK=$(pwd)/var/www/ >> ./AILENV/bin/activate
-echo export AIL_REDIS=$(pwd)/redis/src/ >> ./AILENV/bin/activate
-echo export AIL_LEVELDB=$(pwd)/redis-leveldb/ >> ./AILENV/bin/activate
-
-. ./AILENV/bin/activate
-
-mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}
-mkdir -p $AIL_HOME/LEVEL_DB_DATA/2017
-mkdir -p $AIL_HOME/LEVEL_DB_DATA/3017
-
-pip install -U pip
-pip install -U -r pip_packages_requirement.txt
-
-# Pyfaup
-pushd faup/src/lib/bindings/python/
-python setup.py install
-popd
-
-# Py tlsh
-pushd tlsh/py_ext
-python setup.py build
-python setup.py install
-
-# Download the necessary NLTK corpora and sentiment vader
-HOME=$(pwd) python -m textblob.download_corpora
-python -m nltk.downloader vader_lexicon
-python -m nltk.downloader punkt
-
-#Create the file all_module and update the graph in doc
-$AIL_HOME/doc/generate_modules_data_flow_graph.sh
diff --git a/update/v2.0/Update.py b/update/v2.0/Update.py
new file mode 100755
index 00000000..4e026e1f
--- /dev/null
+++ b/update/v2.0/Update.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import time
+import redis
+import datetime
+import configparser
+
+if __name__ == '__main__':
+
+    start_deb = time.time()
+
+    configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+    if not os.path.exists(configfile):
+        raise Exception('Unable to find the configuration file. \
+                        Did you set environment variables? \
+                        Or activate the virtualenv.')
+    cfg = configparser.ConfigParser()
+    cfg.read(configfile)
+
+    r_serv = redis.StrictRedis(
+        host=cfg.get("ARDB_DB", "host"),
+        port=cfg.getint("ARDB_DB", "port"),
+        db=cfg.getint("ARDB_DB", "db"),
+        decode_responses=True)
+
+    #Set current ail version
+    r_serv.set('ail:version', 'v2.0')
+
+    # use new update_date format
+    date_tag_to_replace = ['v1.5', 'v1.7']
+    for tag in date_tag_to_replace:
+        if r_serv.exists('ail:update_date_{}'.format(tag)):
+            date_tag = r_serv.get('ail:update_date_{}'.format(tag))
+            r_serv.hset('ail:update_date', tag, date_tag)
+            r_serv.delete('ail:update_date_{}'.format(tag))
+
+    #Set current ail version
+    r_serv.hset('ail:update_date', 'v2.0', datetime.datetime.now().strftime("%Y%m%d"))
diff --git a/update/v2.0/Update.sh b/update/v2.0/Update.sh
new file mode 100755
index 00000000..bda6a127
--- /dev/null
+++ b/update/v2.0/Update.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
+
+export PATH=$AIL_HOME:$PATH
+export PATH=$AIL_REDIS:$PATH
+export PATH=$AIL_ARDB:$PATH
+export PATH=$AIL_BIN:$PATH
+export PATH=$AIL_FLASK:$PATH
+
+GREEN="\\033[1;32m"
+DEFAULT="\\033[0;39m"
+
+echo -e $GREEN"Shutting down AIL ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -k
+wait
+
+echo ""
+echo -e $GREEN"Update requirement"$DEFAULT
+echo ""
+pip3 install flask-login
+wait
+echo ""
+pip3 install bcrypt
+wait
+echo ""
+echo ""
+
+bash ${AIL_BIN}LAUNCH.sh -lav &
+wait
+echo ""
+
+echo ""
+echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
+echo ""
+python ${AIL_HOME}/update/v2.0/Update.py
+wait
+echo ""
+echo ""
+
+echo ""
+echo -e $GREEN"Update thirdparty ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -t
+wait
+echo ""
+
+echo ""
+echo -e $GREEN"Create Default User"$DEFAULT
+echo ""
+python3 ${AIL_FLASK}create_default_user.py
+
+
+echo ""
+echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -k
+wait
+
+exit 0
diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py
index 71827852..3dbab06a 100755
--- a/var/www/Flask_server.py
+++ b/var/www/Flask_server.py
@@ -36,18 +36,6 @@ import Flask_config
 from Role_Manager import create_user_db, check_password_strength
 from Role_Manager import login_admin, login_analyst
 
-def flask_init():
-    # # TODO: move this to update
-    # role init
-    if not r_serv_db.exists('ail:all_role'):
-        r_serv_db.zadd('ail:all_role', 1, 'admin')
-        r_serv_db.zadd('ail:all_role', 2, 'analyst')
-
-    # check if an account exists
-    if not r_serv_db.exists('user:all'):
-        password = secrets.token_urlsafe()
-        create_user_db('admin@admin.test', password, role='admin',default=True)
-
 # CONFIG #
 cfg = Flask_config.cfg
 baseUrl = cfg.get("Flask", "baseurl")
diff --git a/var/www/create_default_user.py b/var/www/create_default_user.py
index 9391af56..c0735611 100755
--- a/var/www/create_default_user.py
+++ b/var/www/create_default_user.py
@@ -3,20 +3,44 @@
 
 import os
 import sys
-
+import redis
 import secrets
+import configparser
 
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
-sys.path.append('./modules/')
+sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules'))
 
-from Role_Manager import create_user_db, get_default_admin_token
+from Role_Manager import create_user_db, edit_user_db, get_default_admin_token
+
+
+
+configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+if not os.path.exists(configfile):
+    raise Exception('Unable to find the configuration file. \
+                    Did you set environment variables? \
+                    Or activate the virtualenv.')
+cfg = configparser.ConfigParser()
+cfg.read(configfile)
+
+r_serv = redis.StrictRedis(
+    host=cfg.get("ARDB_DB", "host"),
+    port=cfg.getint("ARDB_DB", "port"),
+    db=cfg.getint("ARDB_DB", "db"),
+    decode_responses=True)
 
 
 if __name__ == "__main__":
 
+    # create role_list
+    if not r_serv.exists('ail:all_role'):
+        r_serv.zadd('ail:all_role', 1, 'admin')
+        r_serv.zadd('ail:all_role', 2, 'analyst')
+
     username = 'admin@admin.test'
     password = secrets.token_urlsafe()
-    create_user_db(username, password, role='admin', default=True)
+    if r_serv.exists('user_metadata:admin@admin.test'):
+        edit_user_db(username, password=password, role='admin')
+    else:
+        create_user_db(username, password, role='admin', default=True)
     token = get_default_admin_token()
 
     default_passwd_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD')
diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py
index 82c5ba45..4535da28 100644
--- a/var/www/modules/restApi/Flask_restApi.py
+++ b/var/www/modules/restApi/Flask_restApi.py
@@ -93,10 +93,10 @@ def one():
 
 # ============= ROUTES ==============
 
-@restApi.route("/api", methods=['GET'])
-@login_required
-def api():
-    return 'api doc'
+# @restApi.route("/api", methods=['GET'])
+# @login_required
+# def api():
+#     return 'api doc'
 
 @restApi.route("api/items", methods=['POST'])
 @token_required
diff --git a/var/www/modules/restApi/templates/api_default.html b/var/www/modules/restApi/templates/api_default.html
index 91d300af..cb2cc24f 100644
--- a/var/www/modules/restApi/templates/api_default.html
+++ b/var/www/modules/restApi/templates/api_default.html
@@ -26,82 +26,6 @@
 
 			<div class="col-12 col-lg-10" id="core_content">
 
-				<div class="row">
-					<div class="col-xl-6">
-
-						<div class="card mt-1 mb-1">
-						  <div class="card-header text-white bg-dark">
-								<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=onion"><i class="fas fa-user-secret"></i> Onions Crawlers</a></h5>
-								<div class="row">
-									<div class="col-6">
-										<span class="badge badge-success" id="stat_onion_domain_up">{{ statDomains_onion['domains_up'] }}</span> UP
-										<span class="badge badge-danger ml-md-3" id="stat_onion_domain_down">{{ statDomains_onion['domains_down'] }}</span> DOWN
-									</div>
-									<div class="col-6">
-										<span class="badge badge-success" id="stat_onion_total">{{ statDomains_onion['total'] }}</span> Crawled
-										<span class="badge badge-warning ml-md-3" id="stat_onion_queue">{{ statDomains_onion['domains_queue'] }}</span> Queue
-									</div>
-								</div>
-							</div>
-						  <div class="card-body px-0 py-0 ">
-								<table class="table">
-									<tbody id="tbody_crawler_onion_info">
-										{% for crawler in crawler_metadata_onion %}
-											<tr>
-						      			<td>
-													<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
-												</td>
-												<td>
-													{{crawler['crawling_domain']}}
-												</td>
-												<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
-													{{crawler['status_info']}}
-												</td>
-											</tr>
-										{% endfor %}
-									</tbody>
-								</table>
-							</div>
-						</div>
-
-					</div>
-					<div class="col-xl-6">
-						<div class="card mt-1 mb-1">
-							<div class="card-header text-white bg-dark">
-								<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=regular"><i class="fab fa-html5"></i> Regular Crawlers</a></h5>
-								<div class="row">
-									<div class="col-6">
-										<span class="badge badge-success" id="stat_regular_domain_up">{{ statDomains_regular['domains_up'] }}</span> UP
-										<span class="badge badge-danger ml-md-3" id="stat_regular_domain_down">{{ statDomains_regular['domains_down'] }}</span> DOWN
-									</div>
-									<div class="col-6">
-										<span class="badge badge-success" id="stat_regular_total">{{ statDomains_regular['total'] }}</span> Crawled
-										<span class="badge badge-warning ml-md-3" id="stat_regular_queue">{{ statDomains_regular['domains_queue'] }}</span> Queue
-									</div>
-								</div>
-							</div>
-							<div class="card-body px-0 py-0 ">
-								<table class="table">
-									<tbody id="tbody_crawler_regular_info">
-										{% for crawler in crawler_metadata_regular %}
-											<tr>
-												<td>
-													<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
-												</td>
-												<td>
-													{{crawler['crawling_domain']}}
-												</td>
-												<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
-													{{crawler['status_info']}}
-												</td>
-											</tr>
-										{% endfor %}
-									</tbody>
-								</table>
-							</div>
-						</div>
-					</div>
-				</div>
 
 			</div>
 		</div>
@@ -109,114 +33,3 @@
 
 
 </body>
-
-<script>
-var to_refresh = false
-$(document).ready(function(){
-	$("#page-Crawler").addClass("active");
-	$("#nav_dashboard").addClass("active");
-	$( window ).focus(function() {
-			to_refresh = true
-			refresh_crawler_status();
-	});
-	$( window ).blur(function() {
-			to_refresh = false
-	});
-
-	to_refresh = true
-	refresh_crawler_status();
-});
-
-function toggle_sidebar(){
-	if($('#nav_menu').is(':visible')){
-		$('#nav_menu').hide();
-		$('#side_menu').removeClass('border-right')
-		$('#side_menu').removeClass('col-lg-2')
-		$('#core_content').removeClass('col-lg-10')
-	}else{
-		$('#nav_menu').show();
-		$('#side_menu').addClass('border-right')
-		$('#side_menu').addClass('col-lg-2')
-		$('#core_content').addClass('col-lg-10')
-	}
-}
-
-function refresh_crawler_status(){
-
-	$.getJSON("{{ url_for('hiddenServices.crawler_dashboard_json') }}",
-		function(data) {
-
-			$('#stat_onion_domain_up').text(data.statDomains_onion['domains_up']);
-			$('#stat_onion_domain_down').text(data.statDomains_onion['domains_down']);
-			$('#stat_onion_total').text(data.statDomains_onion['total']);
-			$('#stat_onion_queue').text(data.statDomains_onion['domains_queue']);
-
-			$('#stat_regular_domain_up').text(data.statDomains_regular['domains_up']);
-			$('#stat_regular_domain_down').text(data.statDomains_regular['domains_down']);
-			$('#stat_regular_total').text(data.statDomains_regular['total']);
-			$('#stat_regular_queue').text(data.statDomains_regular['domains_queue']);
-
-			if(data.crawler_metadata_onion.length!=0){
-				$("#tbody_crawler_onion_info").empty();
-				var tableRef = document.getElementById('tbody_crawler_onion_info');
-				for (var i = 0; i < data.crawler_metadata_onion.length; i++) {
-						var crawler = data.crawler_metadata_onion[i];
-						var newRow = tableRef.insertRow(tableRef.rows.length);
-						var text_color;
-						var icon;
-						if(crawler['status']){
-							text_color = 'Green';
-							icon = 'check';
-						} else {
-							text_color = 'Red';
-							icon = 'times';
-						}
-
-				    var newCell  = newRow.insertCell(0);
-						newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
-
-						newCell  = newRow.insertCell(1);
-						newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
-
-						newCell  = newRow.insertCell(2);
-						newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
-
-						//$("#panel_crawler").show();
-				}
-			}
-			if(data.crawler_metadata_regular.length!=0){
-				$("#tbody_crawler_regular_info").empty();
-				var tableRef = document.getElementById('tbody_crawler_regular_info');
-				for (var i = 0; i < data.crawler_metadata_regular.length; i++) {
-						var crawler = data.crawler_metadata_regular[i];
-						var newRow = tableRef.insertRow(tableRef.rows.length);
-						var text_color;
-						var icon;
-						if(crawler['status']){
-							text_color = 'Green';
-							icon = 'check';
-						} else {
-							text_color = 'Red';
-							icon = 'times';
-						}
-
-				    var newCell  = newRow.insertCell(0);
-						newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
-
-						newCell  = newRow.insertCell(1);
-						newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
-
-						newCell  = newRow.insertCell(2);
-						newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
-
-						//$("#panel_crawler").show();
-				}
-			}
-		}
-	);
-
-	if (to_refresh) {
-    setTimeout("refresh_crawler_status()", 10000);
-	}
-}
-</script>
diff --git a/var/www/modules/restApi/templates/header_api.html b/var/www/modules/restApi/templates/header_api.html
deleted file mode 100644
index 34e20319..00000000
--- a/var/www/modules/restApi/templates/header_api.html
+++ /dev/null
@@ -1 +0,0 @@
-<li id='page-hiddenServices'><a href="{{ url_for('hiddenServices.dashboard') }}"><i class="fa fa-user-secret"></i> hidden Services </a></li>
diff --git a/var/www/templates/error/403.html b/var/www/templates/error/403.html
index e90433ba..3193da28 100644
--- a/var/www/templates/error/403.html
+++ b/var/www/templates/error/403.html
@@ -7,6 +7,7 @@
 
   <!-- Core CSS -->
 	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
 </head>
 
 <body>
diff --git a/var/www/templates/error/404.html b/var/www/templates/error/404.html
index 6943c674..f911a550 100644
--- a/var/www/templates/error/404.html
+++ b/var/www/templates/error/404.html
@@ -7,6 +7,7 @@
 
   <!-- Core CSS -->
 	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
 </head>
 
 <body>

From bb65179e5074a444e71913f0e48723151db80b94 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Thu, 20 Jun 2019 15:49:40 +0200
Subject: [PATCH 14/22] fix: [user_management] fix tokens duplicate + check
 user_acl_integrity + add login errors messages

---
 OVERVIEW.md                                |  4 +--
 bin/packages/User.py                       |  9 ++++++
 var/www/Flask_server.py                    | 13 +++++---
 var/www/modules/Role_Manager.py            | 36 ++++++++++++++++++++--
 var/www/modules/settings/Flask_settings.py | 10 +-----
 var/www/templates/login.html               |  9 ++++--
 6 files changed, 60 insertions(+), 21 deletions(-)

diff --git a/OVERVIEW.md b/OVERVIEW.md
index 74674bd4..c349ae4e 100644
--- a/OVERVIEW.md
+++ b/OVERVIEW.md
@@ -63,12 +63,10 @@ Redis and ARDB overview
 | | | |
 | user_metadata:**user id** | token | **token**  |
 |                           | change_passwd  | **boolean** |
+|                           | role  | **role** |
 
 | Set Key | Value |
 | ------ | ------ |
-| user:request_password_change | **user id** |
-| user:admin | **user id** |
-| | |
 | user_role:**role** | **user id** |
 
 
diff --git a/bin/packages/User.py b/bin/packages/User.py
index cac5c688..829e4205 100755
--- a/bin/packages/User.py
+++ b/bin/packages/User.py
@@ -42,7 +42,16 @@ class User(UserMixin):
     def get(self_class, id):
         return self_class(id)
 
+    def user_is_anonymous(self):
+        if self.id == "__anonymous__":
+            return True
+        else:
+            return False
+
     def check_password(self, password):
+        if self.user_is_anonymous():
+            return False
+
         password = password.encode()
         hashed_password = self.r_serv_db.hget('user:all', self.id).encode()
         if bcrypt.checkpw(password, hashed_password):
diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py
index 3dbab06a..fa8ec95f 100755
--- a/var/www/Flask_server.py
+++ b/var/www/Flask_server.py
@@ -33,7 +33,7 @@ from pytaxonomies import Taxonomies
 import Flask_config
 
 # Import Role_Manager
-from Role_Manager import create_user_db, check_password_strength
+from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
 from Role_Manager import login_admin, login_analyst
 
 # CONFIG #
@@ -162,19 +162,24 @@ def login():
         if username is not None:
             user = User.get(username)
             if user and user.check_password(password):
+                if not check_user_role_integrity(user.get_id()):
+                    error = 'Incorrect User ACL, Please contact your administrator'
+                    return render_template("login.html", error=error)
                 login_user(user) ## TODO: use remember me ?
                 if user.request_password_change():
                     return redirect(url_for('change_password'))
                 else:
                     return redirect(url_for('dashboard.index'))
             else:
-                return 'incorrect password'
+                error = 'Password Incorrect'
+                return render_template("login.html", error=error)
 
-        return 'none'
+        return 'please provide a valid username'
 
     else:
         #next_page = request.args.get('next')
-        return render_template("login.html")
+        error = request.args.get('error')
+        return render_template("login.html" , error=error)
 
 @app.route('/change_password', methods=['POST', 'GET'])
 @login_required
diff --git a/var/www/modules/Role_Manager.py b/var/www/modules/Role_Manager.py
index aa524265..bb48898a 100644
--- a/var/www/modules/Role_Manager.py
+++ b/var/www/modules/Role_Manager.py
@@ -67,6 +67,14 @@ def login_analyst(func):
 ###############################################################
 ###############################################################
 
+def generate_new_token(user_id):
+    # create user token
+    current_token = r_serv_db.hget('user_metadata:{}'.format(user_id), 'token')
+    r_serv_db.hdel('user:tokens', current_token)
+    token = secrets.token_urlsafe(41)
+    r_serv_db.hset('user:tokens', token, user_id)
+    r_serv_db.hset('user_metadata:{}'.format(user_id), 'token', token)
+
 def get_default_admin_token():
     if r_serv_db.exists('user_metadata:admin@admin.test'):
         return r_serv_db.hget('user_metadata:admin@admin.test', 'token')
@@ -78,9 +86,7 @@ def create_user_db(username_id , password, default=False, role=None, update=Fals
     password_hash = hashing_password(password)
 
     # create user token
-    token = secrets.token_urlsafe(41)
-    r_serv_db.hset('user:tokens', token, username_id)
-    r_serv_db.hset('user_metadata:{}'.format(username_id), 'token', token)
+    generate_new_token(username_id)
 
     if update:
         r_serv_db.hdel('user_metadata:{}'.format(username_id), 'change_passwd')
@@ -150,5 +156,29 @@ def get_all_user_role(user_role):
     current_role_val = get_role_level(user_role)
     return r_serv_db.zrange('ail:all_role', current_role_val -1, -1)
 
+def get_all_user_upper_role(user_role):
+    current_role_val = get_role_level(user_role)
+    # remove one rank
+    if current_role_val > 1:
+        return r_serv_db.zrange('ail:all_role', 0, current_role_val -2)
+    else:
+        return []
+
 def get_user_role_by_range(inf, sup):
     return r_serv_db.zrange('ail:all_role', inf, sup)
+
+def get_user_role(user_id):
+    return r_serv_db.hget('user_metadata:{}'.format(user_id), 'role')
+
+def check_user_role_integrity(user_id):
+    user_role = get_user_role(user_id)
+    all_user_role = get_all_user_role(user_role)
+    res = True
+    for role in all_user_role:
+        if not r_serv_db.sismember('user_role:{}'.format(role), user_id):
+            res = False
+    upper_role = get_all_user_upper_role(user_role)
+    for role in upper_role:
+        if r_serv_db.sismember('user_role:{}'.format(role), user_id):
+            res = False
+    return res
diff --git a/var/www/modules/settings/Flask_settings.py b/var/www/modules/settings/Flask_settings.py
index 2080c637..b1d89554 100644
--- a/var/www/modules/settings/Flask_settings.py
+++ b/var/www/modules/settings/Flask_settings.py
@@ -8,7 +8,7 @@ from flask import Flask, render_template, jsonify, request, Blueprint, redirect,
 from flask_login import login_required, current_user
 
 from Role_Manager import login_admin, login_analyst
-from Role_Manager import create_user_db, edit_user_db, delete_user_db, check_password_strength
+from Role_Manager import create_user_db, edit_user_db, delete_user_db, check_password_strength, generate_new_token
 
 import json
 import secrets
@@ -44,14 +44,6 @@ def check_email(email):
     else:
         return False
 
-def generate_new_token(user_id):
-    # create user token
-    current_token = r_serv_db.hget('user_metadata:{}'.format(user_id), 'token')
-    r_serv_db.hdel('user:tokens', current_token)
-    token = secrets.token_urlsafe(41)
-    r_serv_db.hset('user:tokens', token, user_id)
-    r_serv_db.hset('user_metadata:{}'.format(user_id), 'token', token)
-
 def get_git_metadata():
     dict_git = {}
     dict_git['current_branch'] = git_status.get_current_branch()
diff --git a/var/www/templates/login.html b/var/www/templates/login.html
index b846aaa9..d7c1b60b 100644
--- a/var/www/templates/login.html
+++ b/var/www/templates/login.html
@@ -72,8 +72,13 @@
 		      <label for="inputEmail" class="sr-only">Email address</label>
 		      <input type="email" id="inputEmail" name="username" class="form-control" placeholder="Email address" required autofocus>
 		      <label for="inputPassword" class="sr-only">Password</label>
-		      <input type="password" id="inputPassword" name="password" class="form-control" placeholder="Password" required>
-		      <button class="btn btn-lg btn-primary btn-block" type="submit">Sign in</button>
+		      <input type="password" id="inputPassword" name="password" class="form-control {% if error %}is-invalid{% endif %}" placeholder="Password" required>
+					{% if error %}
+						<div class="invalid-feedback">
+							{{error}}
+						</div>
+					{% endif %}
+					<button class="btn btn-lg btn-primary btn-block" type="submit">Sign in</button>
 		    </form>
 
 

From 516d09c6b60a2ec7e765b7b4f32fe4fa24a66760 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thirion=20Aur=C3=A9lien?= <Terrtia@users.noreply.github.com>
Date: Thu, 20 Jun 2019 16:06:29 +0200
Subject: [PATCH 15/22] Update README.md

Update install instructions
---
 README.md | 26 ++++++++------------------
 1 file changed, 8 insertions(+), 18 deletions(-)

diff --git a/README.md b/README.md
index 9f9d8d4d..e894b163 100644
--- a/README.md
+++ b/README.md
@@ -78,14 +78,13 @@ Type these command lines for a fully automated installation and start AIL framew
 git clone https://github.com/CIRCL/AIL-framework.git
 cd AIL-framework
 ./installing_deps.sh
+
 cd ~/AIL-framework/
-. ./AILENV/bin/activate
 cd bin/
-./LAUNCH.sh
+./LAUNCH.sh -l
 ```
 
-The default [installing_deps.sh](./installing_deps.sh) is for Debian and Ubuntu based distributions. For Arch
-linux based distributions, you can replace it with [installing_deps_archlinux.sh](./installing_deps_archlinux.sh).
+The default [installing_deps.sh](./installing_deps.sh) is for Debian and Ubuntu based distributions.
 
 There is also a [Travis file](.travis.yml) used for automating the installation that can be used to build and install AIL on other systems.
 
@@ -136,23 +135,12 @@ Install using Ansible
 
 Please check the [Ansible readme](ansible/README.md).
 
-Starting AIL web interface
+Starting AIL
 --------------------------
 
-To start the web interface, you first need to fetch the required JavaScript/CSS files:
-
 ```bash
-cd $AILENV
-cd var/www/
-bash update_thirdparty.sh
-```
-
-and then you can start the web interface python script:
-
-```bash
-cd $AILENV
-cd var/www/
-./Flask_server.py
+cd bin/
+./LAUNCH -l
 ```
 
 Eventually you can browse the status of the AIL framework website at the following URL:
@@ -161,6 +149,8 @@ Eventually you can browse the status of the AIL framework website at the followi
 http://localhost:7000/
 ```
 
+The default credentials for the web interface are located in ``DEFAULT_PASSWORD``. This file is removed when you change your password.
+
 Training
 --------
 

From be251289a7873b8efb04a5a6b483ee30db8fa038 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Mon, 24 Jun 2019 13:43:16 +0200
Subject: [PATCH 16/22] chg: [Flask server] https support + create self signed
 certificate

---
 update/v2.0/Update.sh           | 15 ++++++++++++++-
 var/www/Flask_server.py         | 17 +++++++++++------
 var/www/modules/Flask_config.py |  2 +-
 3 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/update/v2.0/Update.sh b/update/v2.0/Update.sh
index bda6a127..6eccb0e3 100755
--- a/update/v2.0/Update.sh
+++ b/update/v2.0/Update.sh
@@ -19,6 +19,19 @@ echo -e $GREEN"Shutting down AIL ..."$DEFAULT
 bash ${AIL_BIN}/LAUNCH.sh -k
 wait
 
+echo ""
+echo -e $GREEN"Create Self-Signed Certificate"$DEFAULT
+echo ""
+pushd ${AIL_BIN}/helper/gen_cert
+bash gen_root.sh
+wait
+bash gen_cert.sh
+wait
+popd
+
+cp ${AIL_BIN}/helper/gen_cert/server.crt ${AIL_FLASK}/server.crt
+cp ${AIL_BIN}/helper/gen_cert/server.key ${AIL_FLASK}/server.key
+
 echo ""
 echo -e $GREEN"Update requirement"$DEFAULT
 echo ""
@@ -30,7 +43,7 @@ wait
 echo ""
 echo ""
 
-bash ${AIL_BIN}LAUNCH.sh -lav &
+bash ${AIL_BIN}/LAUNCH.sh -lav &
 wait
 echo ""
 
diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py
index fa8ec95f..9455a628 100755
--- a/var/www/Flask_server.py
+++ b/var/www/Flask_server.py
@@ -4,14 +4,13 @@
 import os
 import re
 import sys
+import ssl
+import time
 
 import redis
-import configparser
 import random
-import json
-import datetime
-import time
-import calendar
+import configparser
+
 from flask import Flask, render_template, jsonify, request, Request, session, redirect, url_for
 from flask_login import LoginManager, current_user, login_user, logout_user, login_required
 
@@ -57,6 +56,12 @@ r_serv_tags = redis.StrictRedis(
 
 # =========       =========#
 
+# =========  TLS  =========#
+ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
+ssl_context.load_cert_chain(certfile='server.crt', keyfile='server.key')
+#print(ssl_context.get_ciphers())
+# =========       =========#
+
 Flask_config.app = Flask(__name__, static_url_path=baseUrl+'/static/')
 app = Flask_config.app
 app.config['MAX_CONTENT_LENGTH'] = 900 * 1024 * 1024
@@ -258,4 +263,4 @@ r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"')
 # ============ MAIN ============
 
 if __name__ == "__main__":
-    app.run(host='0.0.0.0', port=7000, threaded=True)
+    app.run(host='0.0.0.0', port=7000, threaded=True, ssl_context=ssl_context)
diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py
index 29ae9eae..ff5ba02a 100644
--- a/var/www/modules/Flask_config.py
+++ b/var/www/modules/Flask_config.py
@@ -176,7 +176,7 @@ max_dashboard_logs = int(cfg.get("Flask", "max_dashboard_logs"))
 
 crawler_enabled = cfg.getboolean("Crawler", "activate_crawler")
 
-email_regex = r'[^@]+@[^@]+\.[^@]+'
+email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
 email_regex = re.compile(email_regex)
 
 # VT

From e090b664ff6e93b1e52dc472f20de20474f27572 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Mon, 24 Jun 2019 13:57:08 +0200
Subject: [PATCH 17/22] fix: [install_dep] create default user

---
 installing_deps.sh | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/installing_deps.sh b/installing_deps.sh
index bf72cf4b..a996ff7b 100755
--- a/installing_deps.sh
+++ b/installing_deps.sh
@@ -99,9 +99,18 @@ if [ -z "$VIRTUAL_ENV" ]; then
 
 fi
 
+pushd ${AIL_BIN}/helper/gen_cert
+./gen_root.sh
+wait
+./gen_cert.sh
+wait
+popd
+
+cp ${AIL_BIN}/helper/gen_cert/server.crt ${AIL_FLASK}/server.crt
+cp ${AIL_BIN}/helper/gen_cert/server.key ${AIL_FLASK}/server.key
+
 pushd var/www/
 ./update_thirdparty.sh
-python3 create_default_user.py
 popd
 
 mkdir -p $AIL_HOME/PASTES
@@ -124,6 +133,23 @@ python3 setup.py install
 HOME=$(pwd) python3 -m textblob.download_corpora
 python3 -m nltk.downloader vader_lexicon
 python3 -m nltk.downloader punkt
+popd
 
 #Create the file all_module and update the graph in doc
 $AIL_HOME/doc/generate_modules_data_flow_graph.sh
+
+#### DB SETUP ####
+
+# LAUNCH ARDB
+bash ${AIL_BIN}/LAUNCH.sh -lav &
+wait
+echo ""
+
+# create default user
+pushd ${AIL_FLASK}
+python3 create_default_user.py
+popd
+
+bash ${AIL_BIN}/LAUNCH.sh -k &
+wait
+echo ""

From c759142ca24c0145af8df6c417014b0f1b000943 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Mon, 24 Jun 2019 14:37:52 +0200
Subject: [PATCH 18/22] fix: [UI settings] fix toggle_sidebar

---
 installing_deps.sh                               | 12 ++++++------
 .../modules/settings/templates/create_user.html  | 16 +++++++++++++++-
 .../modules/settings/templates/edit_profile.html | 16 +++++++++++++++-
 .../settings/templates/settings_index.html       |  1 +
 .../modules/settings/templates/users_list.html   | 16 +++++++++++++++-
 var/www/templates/settings/menu_sidebar.html     | 15 ++++++---------
 6 files changed, 58 insertions(+), 18 deletions(-)

diff --git a/installing_deps.sh b/installing_deps.sh
index a996ff7b..db7cfa18 100755
--- a/installing_deps.sh
+++ b/installing_deps.sh
@@ -99,17 +99,17 @@ if [ -z "$VIRTUAL_ENV" ]; then
 
 fi
 
-pushd ${AIL_BIN}/helper/gen_cert
+pushd ${AIL_BIN}helper/gen_cert
 ./gen_root.sh
 wait
 ./gen_cert.sh
 wait
 popd
 
-cp ${AIL_BIN}/helper/gen_cert/server.crt ${AIL_FLASK}/server.crt
-cp ${AIL_BIN}/helper/gen_cert/server.key ${AIL_FLASK}/server.key
+cp ${AIL_BIN}helper/gen_cert/server.crt ${AIL_FLASK}server.crt
+cp ${AIL_BIN}helper/gen_cert/server.key ${AIL_FLASK}server.key
 
-pushd var/www/
+pushd ${AIL_FLASK}
 ./update_thirdparty.sh
 popd
 
@@ -141,7 +141,7 @@ $AIL_HOME/doc/generate_modules_data_flow_graph.sh
 #### DB SETUP ####
 
 # LAUNCH ARDB
-bash ${AIL_BIN}/LAUNCH.sh -lav &
+bash ${AIL_BIN}LAUNCH.sh -lav &
 wait
 echo ""
 
@@ -150,6 +150,6 @@ pushd ${AIL_FLASK}
 python3 create_default_user.py
 popd
 
-bash ${AIL_BIN}/LAUNCH.sh -k &
+bash ${AIL_BIN}LAUNCH.sh -k &
 wait
 echo ""
diff --git a/var/www/modules/settings/templates/create_user.html b/var/www/modules/settings/templates/create_user.html
index 139a2a90..5ab809ed 100644
--- a/var/www/modules/settings/templates/create_user.html
+++ b/var/www/modules/settings/templates/create_user.html
@@ -27,7 +27,7 @@
 
 			{% include 'settings/menu_sidebar.html' %}
 
-      <div class="col-12 col-xl-10" id="core_content">
+      <div class="col-12 col-lg-10" id="core_content">
 
         <form class="form-signin" action="{{ url_for('settings.create_user_post')}}" autocomplete="off" method="post">
 
@@ -120,6 +120,20 @@ $(document).ready(function(){
   {% endif %}
 } );
 
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+
 function toggle_password_fields() {
   var password_div = $("#password-section");
   if(password_div.is(":visible")){
diff --git a/var/www/modules/settings/templates/edit_profile.html b/var/www/modules/settings/templates/edit_profile.html
index 8dfe4cf5..ffa65e83 100644
--- a/var/www/modules/settings/templates/edit_profile.html
+++ b/var/www/modules/settings/templates/edit_profile.html
@@ -27,7 +27,7 @@
 
 			{% include 'settings/menu_sidebar.html' %}
 
-			<div class="col-12 col-xl-10" id="core_content">
+			<div class="col-12 col-lg-10" id="core_content">
 
 				<div class="card mb-3 mt-1">
 					<div class="card-header text-white bg-dark pb-1">
@@ -77,6 +77,20 @@ $(document).ready(function(){
   $("#nav_edit_profile").addClass("active");
 	$("#nav_my_profile").removeClass("text-muted");
 } );
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
 </script>
 
 </html>
diff --git a/var/www/modules/settings/templates/settings_index.html b/var/www/modules/settings/templates/settings_index.html
index 5fc21dc5..a43afc64 100644
--- a/var/www/modules/settings/templates/settings_index.html
+++ b/var/www/modules/settings/templates/settings_index.html
@@ -192,6 +192,7 @@ update_progress();
 var progress_interval = setInterval(function(){
 	update_progress()
 }, 4000);
+
 </script>
 
 </html>
diff --git a/var/www/modules/settings/templates/users_list.html b/var/www/modules/settings/templates/users_list.html
index ccf4003b..0c58ab2e 100644
--- a/var/www/modules/settings/templates/users_list.html
+++ b/var/www/modules/settings/templates/users_list.html
@@ -38,7 +38,7 @@
 
 			{% include 'settings/menu_sidebar.html' %}
 
-			<div class="col-12 col-xl-10" id="core_content">
+			<div class="col-12 col-lg-10" id="core_content">
 
         {% if new_user %}
         <div class="text-center my-3 ">
@@ -103,6 +103,20 @@ $(document).ready(function(){
   $("#nav_users_list").addClass("active");
 	$("#nav_user_management").removeClass("text-muted");
 } );
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
 </script>
 
 </html>
diff --git a/var/www/templates/settings/menu_sidebar.html b/var/www/templates/settings/menu_sidebar.html
index 3fe196e4..7732bee5 100644
--- a/var/www/templates/settings/menu_sidebar.html
+++ b/var/www/templates/settings/menu_sidebar.html
@@ -1,11 +1,11 @@
       <div class="col-12 col-lg-2 p-0 bg-light border-right" id="side_menu">
 
-      	<button type="button" class="btn btn-outline-secondary mt-1 ml-3" onclick="toggle_sidebar()">
+      	<button type="button" class="btn btn-outline-secondary mt-1 ml-3" onclick="toggle_sidebar();">
           <i class="fas fa-align-left"></i>
       	  <span>Toggle Sidebar</span>
       	</button>
 
-      	<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2">
+      	<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
       		<h5 class="d-flex text-muted w-100" id="nav_server">
       		  <span>Diagnostic</span>
       		</h5>
@@ -17,9 +17,7 @@
       	      </a>
       	    </li>
           </ul>
-      	</nav>
-        <nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_profile">
-          <h5 class="d-flex text-muted w-100" id="nav_my_profile">
+          <h5 class="d-flex text-muted w-100 py-2" id="nav_my_profile">
             <span>My Profile</span>
           </h5>
           <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
@@ -36,10 +34,8 @@
       	      </a>
       	    </li>
           </ul>
-        </nav>
         {% if admin_level %}
-          <nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_users">
-        		<h5 class="d-flex text-muted w-100" id="nav_user_management">
+        		<h5 class="d-flex text-muted w-100 py-2" id="nav_user_management">
         		  <span>User Management</span>
         		</h5>
             <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
@@ -56,6 +52,7 @@
         	      </a>
         	    </li>
             </ul>
-        	</nav>
         {% endif %}
+
+        </nav>
       </div>

From 03ed3c63a1ee89e5b798370b2f3c61fe40cea5ec Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Mon, 24 Jun 2019 14:41:54 +0200
Subject: [PATCH 19/22] chg: [helper] generate self signed certificates

---
 bin/helper/gen_cert/README.md   | 14 ++++++++++++++
 bin/helper/gen_cert/ext3.cnf    |  5 +++++
 bin/helper/gen_cert/gen_cert.sh |  9 +++++++++
 bin/helper/gen_cert/gen_root.sh |  5 +++++
 bin/helper/gen_cert/san.cnf     | 11 +++++++++++
 5 files changed, 44 insertions(+)
 create mode 100644 bin/helper/gen_cert/README.md
 create mode 100644 bin/helper/gen_cert/ext3.cnf
 create mode 100755 bin/helper/gen_cert/gen_cert.sh
 create mode 100755 bin/helper/gen_cert/gen_root.sh
 create mode 100644 bin/helper/gen_cert/san.cnf

diff --git a/bin/helper/gen_cert/README.md b/bin/helper/gen_cert/README.md
new file mode 100644
index 00000000..d007add5
--- /dev/null
+++ b/bin/helper/gen_cert/README.md
@@ -0,0 +1,14 @@
+Usage
+=====
+These scripts are useful to generate the server's self-signed certificate.
+## Root Certification Authority ##
+`gen_root.sh`
+This will create a rootCA.crt to verify the server's cert.
+## Server certificate ##
+### Configuration Files ###
+Look into these files if you plan to verify the certificate:
+  * san.cnf holds basic information about the certificate.
+  * ext.3 holds in particular the 'subjectAltNames** option that is for the verification to succeed.
+
+ ### Generation ###
+`gen_root.sh`
diff --git a/bin/helper/gen_cert/ext3.cnf b/bin/helper/gen_cert/ext3.cnf
new file mode 100644
index 00000000..8f36435b
--- /dev/null
+++ b/bin/helper/gen_cert/ext3.cnf
@@ -0,0 +1,5 @@
+keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
+subjectAltName = @alt_names
+
+[alt_names]
+IP.1 = 127.0.0.1
diff --git a/bin/helper/gen_cert/gen_cert.sh b/bin/helper/gen_cert/gen_cert.sh
new file mode 100755
index 00000000..f22de1cb
--- /dev/null
+++ b/bin/helper/gen_cert/gen_cert.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Create Server key
+openssl genrsa -out server.key 4096
+# Create the Server Signing Request - non interactive, config in san.cnf
+openssl req -sha256 -new -key server.key -out server.csr -config san.cnf
+# Create the server certificate by rootCA, with ext3 subjectAltName in ext3.cnf
+openssl x509 -req -in server.csr -CA rootCA.crt -CAkey rootCA.key -CAcreateserial -out server.crt -days 500 -sha256 -extfile ext3.cnf
+# Concat in pem
+cat server.crt server.key > server.pem
diff --git a/bin/helper/gen_cert/gen_root.sh b/bin/helper/gen_cert/gen_root.sh
new file mode 100755
index 00000000..1f6e582b
--- /dev/null
+++ b/bin/helper/gen_cert/gen_root.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash 
+# Create Root key
+openssl genrsa -out rootCA.key 4096
+# Create and Sign the Root CA Certificate
+openssl req -x509 -new -nodes -key rootCA.key -sha256 -days 1024 -out rootCA.crt -config san.cnf
diff --git a/bin/helper/gen_cert/san.cnf b/bin/helper/gen_cert/san.cnf
new file mode 100644
index 00000000..61d8a11a
--- /dev/null
+++ b/bin/helper/gen_cert/san.cnf
@@ -0,0 +1,11 @@
+[req]
+default_bits = 4096
+prompt = no
+default_md = sha256
+distinguished_name = dn
+
+[dn]
+C=LU
+L=Luxembourg
+O=AIL-Default
+CN = localhost

From 7c328800e613fc14b89a5e5dc169189c0ce47471 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Mon, 24 Jun 2019 15:09:31 +0200
Subject: [PATCH 20/22] fix: [install_dep] create update current_version

---
 installing_deps.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/installing_deps.sh b/installing_deps.sh
index db7cfa18..99325f82 100755
--- a/installing_deps.sh
+++ b/installing_deps.sh
@@ -140,6 +140,9 @@ $AIL_HOME/doc/generate_modules_data_flow_graph.sh
 
 #### DB SETUP ####
 
+# init update version
+git describe --abbrev=0 --tags | tr -d '\n' > ${AIL_HOME}/update/current_version
+
 # LAUNCH ARDB
 bash ${AIL_BIN}LAUNCH.sh -lav &
 wait

From 27ae671b3d4b7c696f27d7e96518631d9ab37599 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Mon, 24 Jun 2019 17:14:16 +0200
Subject: [PATCH 21/22] fix: [UI login/change_password] add missing ail-logo

---
 var/www/templates/change_password.html | 2 +-
 var/www/templates/login.html           | 2 +-
 var/www/update_thirdparty.sh           | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/var/www/templates/change_password.html b/var/www/templates/change_password.html
index 7722d86e..7560d8df 100644
--- a/var/www/templates/change_password.html
+++ b/var/www/templates/change_password.html
@@ -62,7 +62,7 @@
 
 
 				<form class="form-signin" action="{{ url_for('change_password')}}" autocomplete="off" method="post">
-		      <img class="mb-4" src="{{ url_for('static', filename='image/logo-small.png')}}" width="300">
+		      <img class="mb-4" src="{{ url_for('static', filename='image/AIL-logo.png')}}" width="300">
 		      <h1 class="h3 mb-3 text-secondary">Change Password</h1>
 		      <label for="inputPassword1" class="sr-only">Password</label>
 		      <input type="password" id="inputPassword1" name="password1" class="form-control {% if error %}is-invalid{% endif %}" placeholder="Password" autocomplete="new-password" required autofocus>
diff --git a/var/www/templates/login.html b/var/www/templates/login.html
index d7c1b60b..43a3ee36 100644
--- a/var/www/templates/login.html
+++ b/var/www/templates/login.html
@@ -67,7 +67,7 @@
 
 
 				<form class="form-signin" action="{{ url_for('login')}}"  method="post">
-		      <img class="mb-4" src="{{ url_for('static', filename='image/logo-small.png')}}" width="300">
+		      <img class="mb-4" src="{{ url_for('static', filename='image/AIL-logo.png')}}" width="300">
 		      <h1 class="h3 mb-3 text-secondary">Please sign in</h1>
 		      <label for="inputEmail" class="sr-only">Email address</label>
 		      <input type="email" id="inputEmail" name="username" class="form-control" placeholder="Email address" required autofocus>
diff --git a/var/www/update_thirdparty.sh b/var/www/update_thirdparty.sh
index 50adf2a0..e5b66451 100755
--- a/var/www/update_thirdparty.sh
+++ b/var/www/update_thirdparty.sh
@@ -102,6 +102,7 @@ rm -rf temp
 mkdir -p ./static/image
 pushd static/image
 wget -q https://www.circl.lu/assets/images/logos/AIL.png -O AIL.png
+wget -q https://www.circl.lu/assets/images/logos/AIL-logo.png -O AIL-logo.png
 popd
 
 if ! [[ -n "$AIL_HOME" ]]

From 5f157c52d8cd71cd748b3680db763302f04303b1 Mon Sep 17 00:00:00 2001
From: Terrtia <or1994@hotmail.fr>
Date: Wed, 26 Jun 2019 16:36:40 +0200
Subject: [PATCH 22/22] chg: [Flask login] add brute force protection + log
 login errors

---
 OVERVIEW.md             |  8 ++++++
 var/www/Flask_server.py | 57 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)

diff --git a/OVERVIEW.md b/OVERVIEW.md
index c349ae4e..38ac7e7f 100644
--- a/OVERVIEW.md
+++ b/OVERVIEW.md
@@ -30,6 +30,14 @@ Redis and ARDB overview
 
 # Database Map:
 
+### Redis cache
+
+##### Brute force protection:
+| Set Key | Value |
+| ------ | ------ |
+| failed_login_ip:**ip**           | **nb login failed** | TTL
+| failed_login_user_id:**user_id** | **nb login failed** | TTL
+
 ## DB0 - Core:
 
 ##### Update keys:
diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py
index 9455a628..8ba4526e 100755
--- a/var/www/Flask_server.py
+++ b/var/www/Flask_server.py
@@ -9,6 +9,8 @@ import time
 
 import redis
 import random
+import logging
+import logging.handlers
 import configparser
 
 from flask import Flask, render_template, jsonify, request, Request, session, redirect, url_for
@@ -54,6 +56,27 @@ r_serv_tags = redis.StrictRedis(
     db=cfg.getint("ARDB_Tags", "db"),
     decode_responses=True)
 
+r_cache = redis.StrictRedis(
+    host=cfg.get("Redis_Cache", "host"),
+    port=cfg.getint("Redis_Cache", "port"),
+    db=cfg.getint("Redis_Cache", "db"),
+    decode_responses=True)
+
+# logs
+log_dir = os.path.join(os.environ['AIL_HOME'], 'logs')
+if not os.path.isdir(log_dir):
+    os.makedirs(logs_dir)
+
+log_filename = os.path.join(log_dir, 'flask_server.logs')
+logger = logging.getLogger()
+formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+handler_log = logging.handlers.TimedRotatingFileHandler(log_filename, when="midnight", interval=1)
+handler_log.suffix = '%Y-%m-%d.log'
+handler_log.setFormatter(formatter)
+handler_log.setLevel(30)
+logger.addHandler(handler_log)
+logger.setLevel(30)
+
 # =========       =========#
 
 # =========  TLS  =========#
@@ -156,9 +179,26 @@ def add_header(response):
     response.headers['Cache-Control'] = 'public, max-age=0'
     return response
 
+# @app.route('/test', methods=['GET'])
+# def test():
+#     for rule in app.url_map.iter_rules():
+#         print(rule)
+#     return 'o'
+
 # ========== ROUTES ============
 @app.route('/login', methods=['POST', 'GET'])
 def login():
+
+    current_ip = request.remote_addr
+    login_failed_ip = r_cache.get('failed_login_ip:{}'.format(current_ip))
+
+    # brute force by ip
+    if login_failed_ip:
+        login_failed_ip = int(login_failed_ip)
+        if login_failed_ip >= 5:
+            error = 'Max Connection Attempts reached, Please wait {}s'.format(r_cache.ttl('failed_login_ip:{}'.format(current_ip)))
+            return render_template("login.html", error=error)
+
     if request.method == 'POST':
         username = request.form.get('username')
         password = request.form.get('password')
@@ -166,6 +206,14 @@ def login():
 
         if username is not None:
             user = User.get(username)
+            login_failed_user_id = r_cache.get('failed_login_user_id:{}'.format(username))
+            # brute force by user_id
+            if login_failed_user_id:
+                login_failed_user_id = int(login_failed_user_id)
+                if login_failed_user_id >= 5:
+                    error = 'Max Connection Attempts reached, Please wait {}s'.format(r_cache.ttl('failed_login_user_id:{}'.format(username)))
+                    return render_template("login.html", error=error)
+
             if user and user.check_password(password):
                 if not check_user_role_integrity(user.get_id()):
                     error = 'Incorrect User ACL, Please contact your administrator'
@@ -175,7 +223,16 @@ def login():
                     return redirect(url_for('change_password'))
                 else:
                     return redirect(url_for('dashboard.index'))
+            # login failed
             else:
+                # set brute force protection
+                logger.warning("Login failed, ip={}, username={}".format(current_ip, username))
+                r_cache.incr('failed_login_ip:{}'.format(current_ip))
+                r_cache.expire('failed_login_ip:{}'.format(current_ip), 300)
+                r_cache.incr('failed_login_user_id:{}'.format(username))
+                r_cache.expire('failed_login_user_id:{}'.format(username), 300)
+                #
+
                 error = 'Password Incorrect'
                 return render_template("login.html", error=error)