merge master

This commit is contained in:
Terrtia 2018-11-14 15:55:42 +01:00
commit d9b776d58a
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
18 changed files with 526 additions and 92 deletions

View file

@ -114,6 +114,9 @@ There are two types of installation. You can install a *local* or a *remote* Spl
2. *(Splash host)* To install and setup your tor proxy: 2. *(Splash host)* To install and setup your tor proxy:
- Install the tor proxy: ``sudo apt-get install tor -y`` - Install the tor proxy: ``sudo apt-get install tor -y``
(Not required if ``Splah host == AIL host`` - The tor proxy is installed by default in AIL) (Not required if ``Splah host == AIL host`` - The tor proxy is installed by default in AIL)
- Allow Tor to bind to any interface or to the docker interface (by default binds to 127.0.0.1 only) in ``/etc/tor/torrc``
``SOCKSPort 0.0.0.0:9050`` or
``SOCKSPort 172.17.0.1:9050``
- Add the following line ``SOCKSPolicy accept 172.17.0.0/16`` in ``/etc/tor/torrc`` - Add the following line ``SOCKSPolicy accept 172.17.0.0/16`` in ``/etc/tor/torrc``
(for a linux docker, the localhost IP is *172.17.0.1*; Should be adapted for other platform) (for a linux docker, the localhost IP is *172.17.0.1*; Should be adapted for other platform)
- Restart the tor proxy: ``sudo service tor restart`` - Restart the tor proxy: ``sudo service tor restart``

View file

@ -26,6 +26,24 @@ ARDB overview
ARDB_DB ARDB_DB
* DB 1 - Curve * DB 1 - Curve
* DB 2 - TermFreq * DB 2 - TermFreq
----------------------------------------- TERM ----------------------------------------
SET - 'TrackedRegexSet' term
HSET - 'TrackedRegexDate' tracked_regex today_timestamp
SET - 'TrackedSetSet' set_to_add
HSET - 'TrackedSetDate' set_to_add today_timestamp
SET - 'TrackedSetTermSet' term
HSET - 'TrackedTermDate' tracked_regex today_timestamp
SET - 'TrackedNotificationEmails_'+term/set email
SET - 'TrackedNotifications' term/set
* DB 3 - Trending * DB 3 - Trending
* DB 4 - Sentiment * DB 4 - Sentiment
* DB 5 - TermCred * DB 5 - TermCred

View file

@ -48,6 +48,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
top_termFreq_setName_month = ["TopTermFreq_set_month", 31] top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
# create direct link in mail # create direct link in mail
full_paste_url = "/showsavedpaste/?paste=" full_paste_url = "/showsavedpaste/?paste="
@ -71,6 +73,11 @@ def check_if_tracked_term(term, path):
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term): for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term):
sendEmailNotification(email, 'Term', mail_body) sendEmailNotification(email, 'Term', mail_body)
# tag paste
for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + term):
msg = '{};{}'.format(tag, path)
p.populate_set_out(msg, 'Tags')
def getValueOverRange(word, startDate, num_day): def getValueOverRange(word, startDate, num_day):
to_return = 0 to_return = 0

View file

@ -54,7 +54,7 @@ from thehive4py.models import Case, CaseTask, CustomFieldHelper
def create_the_hive_alert(source, path, content, tag): def create_the_hive_alert(source, path, tag):
tags = list(r_serv_metadata.smembers('tag:'+path)) tags = list(r_serv_metadata.smembers('tag:'+path))
artifacts = [ artifacts = [
@ -63,7 +63,6 @@ def create_the_hive_alert(source, path, content, tag):
] ]
l_tags = tag.split(',') l_tags = tag.split(',')
print(tag)
# Prepare the sample Alert # Prepare the sample Alert
sourceRef = str(uuid.uuid4())[0:6] sourceRef = str(uuid.uuid4())[0:6]
@ -91,6 +90,41 @@ def create_the_hive_alert(source, path, content, tag):
except: except:
print('hive connection error') print('hive connection error')
def feeder(message, count=0):
if flag_the_hive or flag_misp:
tag, path = message.split(';')
## FIXME: remove it
if PASTES_FOLDER not in path:
path = os.path.join(PASTES_FOLDER, path)
try:
paste = Paste.Paste(path)
except FileNotFoundError:
if count < 10:
r_serv_db.zincrby('mess_not_saved_export', message, 1)
return 0
else:
r_serv_db.zrem('mess_not_saved_export', message)
print('Error: {} do not exist, tag= {}'.format(path, tag))
return 0
source = '/'.join(paste.p_path.split('/')[-6:])
if HiveApi != False:
if int(r_serv_db.get('hive:auto-alerts')) == 1:
whitelist_hive = r_serv_db.scard('whitelist_hive')
if r_serv_db.sismember('whitelist_hive', tag):
create_the_hive_alert(source, path, tag)
else:
print('hive, auto alerts creation disable')
if flag_misp:
if int(r_serv_db.get('misp:auto-events')) == 1:
if r_serv_db.sismember('whitelist_misp', tag):
misp_wrapper.pushToMISP(uuid_ail, path, tag)
else:
print('misp, auto events creation disable')
if __name__ == "__main__": if __name__ == "__main__":
publisher.port = 6380 publisher.port = 6380
@ -119,10 +153,18 @@ if __name__ == "__main__":
db=cfg.getint("ARDB_Metadata", "db"), db=cfg.getint("ARDB_Metadata", "db"),
decode_responses=True) decode_responses=True)
# set sensor uuid
uuid_ail = r_serv_db.get('ail:uuid') uuid_ail = r_serv_db.get('ail:uuid')
if uuid_ail is None: if uuid_ail is None:
uuid_ail = r_serv_db.set('ail:uuid', uuid.uuid4() ) uuid_ail = r_serv_db.set('ail:uuid', uuid.uuid4() )
# set default
if r_serv_db.get('hive:auto-alerts') is None:
r_serv_db.set('hive:auto-alerts', 0)
if r_serv_db.get('misp:auto-events') is None:
r_serv_db.set('misp:auto-events', 0)
p = Process(config_section) p = Process(config_section)
# create MISP connection # create MISP connection
if flag_misp: if flag_misp:
@ -167,36 +209,30 @@ if __name__ == "__main__":
r_serv_db.set('ail:thehive', False) r_serv_db.set('ail:thehive', False)
print('Not connected to The HIVE') print('Not connected to The HIVE')
refresh_time = 3
## FIXME: remove it
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes"))
time_1 = time.time()
while True: while True:
# Get one message from the input queue # Get one message from the input queue
message = p.get_from_set() message = p.get_from_set()
if message is None: if message is None:
# handle not saved pastes
if int(time.time() - time_1) > refresh_time:
num_queu = r_serv_db.zcard('mess_not_saved_export')
list_queu = r_serv_db.zrange('mess_not_saved_export', 0, -1, withscores=True)
if num_queu and list_queu:
for i in range(0, num_queu):
feeder(list_queu[i][0],list_queu[i][1])
time_1 = time.time()
else:
publisher.debug("{} queue is empty, waiting 1s".format(config_section)) publisher.debug("{} queue is empty, waiting 1s".format(config_section))
time.sleep(1) time.sleep(1)
continue
else: else:
feeder(message)
if flag_the_hive or flag_misp:
tag, path = message.split(';')
paste = Paste.Paste(path)
source = '/'.join(paste.p_rel_path.split('/')[-6:])
full_path = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "pastes"), path)
if HiveApi != False:
if int(r_serv_db.get('hive:auto-alerts')) == 1:
whitelist_hive = r_serv_db.scard('whitelist_hive')
if r_serv_db.sismember('whitelist_hive', tag):
create_the_hive_alert(source, path, full_path, tag)
else:
print('hive, auto alerts creation disable')
if flag_misp:
if int(r_serv_db.get('misp:auto-events')) == 1:
if r_serv_db.sismember('whitelist_misp', tag):
misp_wrapper.pushToMISP(uuid_ail, path, tag)
else:
print('misp, auto events creation disable')

View file

@ -80,6 +80,7 @@ if __name__ == '__main__':
# OTHER CONFIG # # OTHER CONFIG #
operation_mode = cfg.getint("Module_Mixer", "operation_mode") operation_mode = cfg.getint("Module_Mixer", "operation_mode")
ttl_key = cfg.getint("Module_Mixer", "ttl_duplicate") ttl_key = cfg.getint("Module_Mixer", "ttl_duplicate")
default_unnamed_feed_name = cfg.get("Module_Mixer", "default_unnamed_feed_name")
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes")) PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes"))
@ -107,10 +108,11 @@ if __name__ == '__main__':
feeder_name = feeder_name.split('/')[1] feeder_name = feeder_name.split('/')[1]
except ValueError as e: except ValueError as e:
feeder_name = "unnamed_feeder" feeder_name = default_unnamed_feed_name
paste_name = complete_paste
# remove absolute path # remove absolute path
paste_name = complete_paste.replace(PASTES_FOLDER, '', 1) paste_name = paste_name.replace(PASTES_FOLDER, '', 1)
# Processed paste # Processed paste
processed_paste += 1 processed_paste += 1

View file

@ -3,8 +3,10 @@
import argparse import argparse
import configparser import configparser
import traceback
import os import os
import smtplib import smtplib
from pubsublogger import publisher
from email.mime.multipart import MIMEMultipart from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText from email.mime.text import MIMEText
@ -15,6 +17,9 @@ This module allows the global configuration and management of notification setti
# CONFIG # # CONFIG #
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
publisher.port = 6380
publisher.channel = "Script"
# notifications enabled/disabled # notifications enabled/disabled
TrackedTermsNotificationEnabled_Name = "TrackedNotifications" TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
@ -22,7 +27,6 @@ TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
# Keys will be e.g. TrackedNotificationEmails<TERMNAME> # Keys will be e.g. TrackedNotificationEmails<TERMNAME>
TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_" TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
def sendEmailNotification(recipient, alert_name, content): def sendEmailNotification(recipient, alert_name, content):
if not os.path.exists(configfile): if not os.path.exists(configfile):
@ -33,22 +37,12 @@ def sendEmailNotification(recipient, alert_name, content):
cfg = configparser.ConfigParser() cfg = configparser.ConfigParser()
cfg.read(configfile) cfg.read(configfile)
sender = cfg.get("Notifications", "sender"), sender = cfg.get("Notifications", "sender")
sender_host = cfg.get("Notifications", "sender_host"), sender_host = cfg.get("Notifications", "sender_host")
sender_port = cfg.getint("Notifications", "sender_port"), sender_port = cfg.getint("Notifications", "sender_port")
sender_pw = cfg.get("Notifications", "sender_pw"), sender_pw = cfg.get("Notifications", "sender_pw")
if sender_pw == 'None':
if isinstance(sender, tuple): sender_pw = None
sender = sender[0]
if isinstance(sender_host, tuple):
sender_host = sender_host[0]
if isinstance(sender_port, tuple):
sender_port = sender_port[0]
if isinstance(sender_pw, tuple):
sender_pw = sender_pw[0]
# raise an exception if any of these is None # raise an exception if any of these is None
if (sender is None or if (sender is None or
@ -83,9 +77,9 @@ def sendEmailNotification(recipient, alert_name, content):
smtp_server.quit() smtp_server.quit()
print('Send notification ' + alert_name + ' to '+recipient) print('Send notification ' + alert_name + ' to '+recipient)
except Exception as e: except Exception as err:
print(str(e)) traceback.print_tb(err.__traceback__)
# raise e publisher.warning(err)
if __name__ == '__main__': if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Test notification sender.') parser = argparse.ArgumentParser(description='Test notification sender.')

View file

@ -42,6 +42,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
top_termFreq_setName_month = ["TopTermFreq_set_month", 31] top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
top_termFreq_set_array = [top_termFreq_setName_day, top_termFreq_setName_week, top_termFreq_setName_month] top_termFreq_set_array = [top_termFreq_setName_day, top_termFreq_setName_week, top_termFreq_setName_month]
TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
# create direct link in mail # create direct link in mail
full_paste_url = "/showsavedpaste/?paste=" full_paste_url = "/showsavedpaste/?paste="
@ -129,6 +131,11 @@ if __name__ == "__main__":
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + regex_str_complete): for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + regex_str_complete):
sendEmailNotification(email, 'Term', mail_body) sendEmailNotification(email, 'Term', mail_body)
# tag paste
for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + regex_str_complete):
msg = '{};{}'.format(tag, filename)
p.populate_set_out(msg, 'Tags')
set_name = 'regex_' + dico_regexname_to_redis[regex_str] set_name = 'regex_' + dico_regexname_to_redis[regex_str]
new_to_the_set = server_term.sadd(set_name, filename) new_to_the_set = server_term.sadd(set_name, filename)
new_to_the_set = True if new_to_the_set == 1 else False new_to_the_set = True if new_to_the_set == 1 else False

View file

@ -34,6 +34,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
top_termFreq_setName_month = ["TopTermFreq_set_month", 31] top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
# create direct link in mail # create direct link in mail
full_paste_url = "/showsavedpaste/?paste=" full_paste_url = "/showsavedpaste/?paste="
@ -121,6 +123,11 @@ if __name__ == "__main__":
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + dico_setname_to_redis[str(the_set)]): for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + dico_setname_to_redis[str(the_set)]):
sendEmailNotification(email, 'Term', mail_body) sendEmailNotification(email, 'Term', mail_body)
# tag paste
for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + dico_setname_to_redis[str(the_set)]):
msg = '{};{}'.format(tag, filename)
p.populate_set_out(msg, 'Tags')
print(the_set, "matched in", filename) print(the_set, "matched in", filename)
set_name = 'set_' + dico_setname_to_redis[the_set] set_name = 'set_' + dico_setname_to_redis[the_set]
new_to_the_set = server_term.sadd(set_name, filename) new_to_the_set = server_term.sadd(set_name, filename)

View file

@ -27,6 +27,7 @@ ail_domain = http://localhost:7000
sender = sender@example.com sender = sender@example.com
sender_host = smtp.example.com sender_host = smtp.example.com
sender_port = 1337 sender_port = 1337
sender_pw = None
# optional for using with authenticated SMTP over SSL # optional for using with authenticated SMTP over SSL
# sender_pw = securepassword # sender_pw = securepassword
@ -98,6 +99,7 @@ threshold_stucked_module=600
operation_mode = 3 operation_mode = 3
#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400) #Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
ttl_duplicate = 86400 ttl_duplicate = 86400
default_unnamed_feed_name = unnamed_feeder
[RegexForTermsFrequency] [RegexForTermsFrequency]
max_execution_time = 60 max_execution_time = 60

View file

@ -32,13 +32,15 @@ publish = Redis_Words
[Curve] [Curve]
subscribe = Redis_Words subscribe = Redis_Words
publish = Redis_CurveManageTopSets publish = Redis_CurveManageTopSets,Redis_Tags
[RegexForTermsFrequency] [RegexForTermsFrequency]
subscribe = Redis_Global subscribe = Redis_Global
publish = Redis_Tags
[SetForTermsFrequency] [SetForTermsFrequency]
subscribe = Redis_Global subscribe = Redis_Global
publish = Redis_Tags
[CurveManageTopSets] [CurveManageTopSets]
subscribe = Redis_CurveManageTopSets subscribe = Redis_CurveManageTopSets

View file

@ -102,7 +102,6 @@ r_serv_onion = redis.StrictRedis(
db=cfg.getint("ARDB_Onion", "db"), db=cfg.getint("ARDB_Onion", "db"),
decode_responses=True) decode_responses=True)
sys.path.append('../../configs/keys') sys.path.append('../../configs/keys')
# MISP # # MISP #
try: try:

View file

@ -506,6 +506,8 @@ def edit_tag_export():
status_misp = [] status_misp = []
status_hive = [] status_hive = []
infoleak_tags = Taxonomies().get('infoleak').machinetags()
is_infoleak_tag = []
for tag in list_export_tags: for tag in list_export_tags:
if r_serv_db.sismember('whitelist_misp', tag): if r_serv_db.sismember('whitelist_misp', tag):
@ -519,6 +521,11 @@ def edit_tag_export():
else: else:
status_hive.append(False) status_hive.append(False)
if tag in infoleak_tags:
is_infoleak_tag.append(True)
else:
is_infoleak_tag.append(False)
if misp_auto_events is not None: if misp_auto_events is not None:
if int(misp_auto_events) == 1: if int(misp_auto_events) == 1:
misp_active = True misp_active = True
@ -543,6 +550,7 @@ def edit_tag_export():
misp_active=misp_active, misp_active=misp_active,
hive_active=hive_active, hive_active=hive_active,
list_export_tags=list_export_tags, list_export_tags=list_export_tags,
is_infoleak_tag=is_infoleak_tag,
status_misp=status_misp, status_misp=status_misp,
status_hive=status_hive, status_hive=status_hive,
nb_tags_whitelist_misp=nb_tags_whitelist_misp, nb_tags_whitelist_misp=nb_tags_whitelist_misp,
@ -594,5 +602,37 @@ def disable_hive_auto_alert():
r_serv_db.set('hive:auto-alerts', 0) r_serv_db.set('hive:auto-alerts', 0)
return edit_tag_export() return edit_tag_export()
@PasteSubmit.route("/PasteSubmit/add_push_tag")
def add_push_tag():
tag = request.args.get('tag')
if tag is not None:
#limit tag length
if len(tag) > 49:
tag = tag[0:48]
r_serv_db.sadd('list_export_tags', tag)
to_return = {}
to_return["tag"] = tag
return jsonify(to_return)
else:
return 'None args', 400
@PasteSubmit.route("/PasteSubmit/delete_push_tag")
def delete_push_tag():
tag = request.args.get('tag')
infoleak_tags = Taxonomies().get('infoleak').machinetags()
if tag not in infoleak_tags and r_serv_db.sismember('list_export_tags', tag):
r_serv_db.srem('list_export_tags', tag)
r_serv_db.srem('whitelist_misp', tag)
r_serv_db.srem('whitelist_hive', tag)
to_return = {}
to_return["tag"] = tag
return jsonify(to_return)
else:
return 'this tag can\'t be removed', 400
# ========= REGISTRATION ========= # ========= REGISTRATION =========
app.register_blueprint(PasteSubmit, url_prefix=baseUrl) app.register_blueprint(PasteSubmit, url_prefix=baseUrl)

View file

@ -37,6 +37,9 @@
background: #d91f2d; background: #d91f2d;
color: #fff; color: #fff;
} }
.mouse_pointer{
cursor: pointer;
}
</style> </style>
</head> </head>
@ -169,7 +172,14 @@
<input type="checkbox" value="{{ tag }}" name="tag_enabled_misp" > <input type="checkbox" value="{{ tag }}" name="tag_enabled_misp" >
{% endif %} {% endif %}
</td> </td>
<td>{{ tag }}</td> <td>
{{ tag }}
{% if not is_infoleak_tag[loop.index0] %}
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" title="Delete this tag" onclick="delete_push_tag('{{ tag }}')">
<span class="glyphicon glyphicon-trash"></span>
</div>
{% endif %}
</td>
</tr> </tr>
{% endfor %} {% endfor %}
@ -209,7 +219,14 @@
<input type="checkbox" value="{{ tag }}" name="tag_enabled_hive" > <input type="checkbox" value="{{ tag }}" name="tag_enabled_hive" >
{% endif %} {% endif %}
</td> </td>
<td>{{ tag }}</td> <td>
{{ tag }}
{% if not is_infoleak_tag[loop.index0] %}
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" title="Delete this tag" onclick="delete_push_tag('{{ tag }}')">
<span class="glyphicon glyphicon-trash"></span>
</div>
{% endif %}
</td>
</tr> </tr>
{% endfor %} {% endfor %}
@ -232,6 +249,42 @@
</form> </form>
<div>
<div id="add_custom_tag_modal" class="modal fade" role="dialog">
<div class="modal-dialog">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
<h2 class="text-center">Add Custom Tag</h2>
</div>
<div class="modal-body">
<div class="form-group input-group" style="margin-bottom: 0px;">
<span class="input-group-addon"><i class="fa fa-tag fa"></i></span>
<input id="new_custom_tag" class="form-control" placeholder="Add a new custom tag to the tag export" type="text">
</div>
</div>
<div class="modal-footer">
<button class="btn btn-primary btn-tags" onclick="add_custom_tag()">
<span class="glyphicon glyphicon-plus"></span>
<span class="label-icon">Add Custom Tag</span>
</button>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
<button type="button" class="btn btn-primary pull-right" data-toggle="modal" data-target="#add_custom_tag_modal" data-url="{{ url_for('Tags.taxonomies') }}">
<span class="glyphicon glyphicon-plus "></span> Add Custom Tag
</button>
</div>
</div> </div>
<!-- /#page-wrapper --> <!-- /#page-wrapper -->
@ -277,6 +330,25 @@ $(document).ready(function(){
} }
); );
} }
function delete_push_tag(tag){
//var row_tr = $(this).closest("tr");
$.get("{{ url_for('PasteSubmit.delete_push_tag') }}", { tag: tag }, function(data, status){
if(status == "success") {
//row_tr.remove();
window.location.reload(false);
}
});
}
function add_custom_tag(){
$.get("{{ url_for('PasteSubmit.add_push_tag') }}", { tag: document.getElementById('new_custom_tag').value }, function(data, status){
if(status == "success") {
//row_tr.remove();
window.location.reload(false);
}
});
}
</script> </script>
</html> </html>

View file

@ -16,6 +16,8 @@ from whoosh import index
from whoosh.fields import Schema, TEXT, ID from whoosh.fields import Schema, TEXT, ID
from whoosh.qparser import QueryParser from whoosh.qparser import QueryParser
import time
# ============ VARIABLES ============ # ============ VARIABLES ============
import Flask_config import Flask_config
@ -55,8 +57,8 @@ def get_index_list(selected_index=""):
if os.path.isdir(os.path.join(baseindexpath, dirs)): if os.path.isdir(os.path.join(baseindexpath, dirs)):
value = dirs value = dirs
name = to_iso_date(dirs) + " - " + \ name = to_iso_date(dirs) + " - " + \
str(get_dir_size(dirs) / (1000*1000)) + " Mb " + \ str(get_dir_size(dirs) / (1000*1000)) + " Mb " #+ \
"(" + str(get_item_count(dirs)) + " Items" + ")" #"(" + str(get_item_count(dirs))''' + " Items" + ")"
flag = dirs==selected_index.split('/')[-1] flag = dirs==selected_index.split('/')[-1]
if dirs == "old_index": if dirs == "old_index":
temp = [value, name, flag] temp = [value, name, flag]
@ -66,6 +68,7 @@ def get_index_list(selected_index=""):
index_list.sort(reverse=True, key=lambda x: x[0]) index_list.sort(reverse=True, key=lambda x: x[0])
if len(temp) != 0: if len(temp) != 0:
index_list.append(temp) index_list.append(temp)
return index_list return index_list
def get_dir_size(directory): def get_dir_size(directory):
@ -108,6 +111,7 @@ def search():
else: else:
selected_index = os.path.join(baseindexpath, index_name) selected_index = os.path.join(baseindexpath, index_name)
''' temporary disabled
# Search filename # Search filename
for path in r_serv_pasteName.smembers(q[0]): for path in r_serv_pasteName.smembers(q[0]):
r.append(path) r.append(path)
@ -119,13 +123,14 @@ def search():
curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
paste_date.append(curr_date) paste_date.append(curr_date)
paste_size.append(paste._get_p_size()) paste_size.append(paste._get_p_size())
'''
# Search full line # Search full line
schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
ix = index.open_dir(selected_index) ix = index.open_dir(selected_index)
with ix.searcher() as searcher: with ix.searcher() as searcher:
query = QueryParser("content", ix.schema).parse(" ".join(q)) query = QueryParser("content", ix.schema).parse("".join(q))
results = searcher.search_page(query, 1, pagelen=num_elem_to_get) results = searcher.search_page(query, 1, pagelen=num_elem_to_get)
for x in results: for x in results:
r.append(x.items()[0][1]) r.append(x.items()[0][1])
@ -159,15 +164,18 @@ def search():
results = searcher.search(query) results = searcher.search(query)
num_res = len(results) num_res = len(results)
index_list = get_index_list()
index_min = 1 index_min = 1
index_max = len(get_index_list()) index_max = len(index_list)
return render_template("search.html", r=r, c=c, return render_template("search.html", r=r, c=c,
query=request.form['query'], paste_date=paste_date, query=request.form['query'], paste_date=paste_date,
paste_size=paste_size, char_to_display=max_preview_modal, paste_size=paste_size, char_to_display=max_preview_modal,
num_res=num_res, index_min=index_min, index_max=index_max, num_res=num_res, index_min=index_min, index_max=index_max,
bootstrap_label=bootstrap_label, bootstrap_label=bootstrap_label,
paste_tags=paste_tags, paste_tags=paste_tags,
index_list=get_index_list(selected_index) index_list=index_list
) )
@ -212,6 +220,7 @@ def get_more_search_result():
p_tags = r_serv_metadata.smembers('tag:'+path) p_tags = r_serv_metadata.smembers('tag:'+path)
l_tags = [] l_tags = []
for tag in p_tags: for tag in p_tags:
complete_tag = tag
tag = tag.split('=') tag = tag.split('=')
if len(tag) > 1: if len(tag) > 1:
if tag[1] != '': if tag[1] != '':
@ -223,7 +232,7 @@ def get_more_search_result():
else: else:
tag = tag[0] tag = tag[0]
l_tags.append(tag) l_tags.append( (tag, complete_tag) )
list_tags.append(l_tags) list_tags.append(l_tags)
to_return = {} to_return = {}

View file

@ -201,7 +201,9 @@
var curr_preview = data.preview_array[i].replace(/\"/g, "\'"); var curr_preview = data.preview_array[i].replace(/\"/g, "\'");
var tag = "" var tag = ""
for(j=0; j<data.list_tags[i].length; j++) { for(j=0; j<data.list_tags[i].length; j++) {
tag = tag + "<span class=\"label label-" + data.bootstrap_label[j % 5] + " pull-left\">" + data.list_tags[j] + "</span>" tag = tag + "<a href=\"{{ url_for('Tags.get_tagged_paste') }}?ltags=" + data.list_tags[i][j][1] + ">"
+ "<span class=\"label label-" + data.bootstrap_label[j % 5] + " pull-left\">" + data.list_tags[i][j][0]
+ "</span>" + "</a>"
} }
search_table.row.add( [ search_table.row.add( [
init_num_of_elements_in_table+((offset))+i+1, init_num_of_elements_in_table+((offset))+i+1,

View file

@ -10,7 +10,7 @@ import redis
import datetime import datetime
import calendar import calendar
import flask import flask
from flask import Flask, render_template, jsonify, request, Blueprint from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect
import re import re
import Paste import Paste
from pprint import pprint from pprint import pprint
@ -24,6 +24,8 @@ cfg = Flask_config.cfg
baseUrl = Flask_config.baseUrl baseUrl = Flask_config.baseUrl
r_serv_term = Flask_config.r_serv_term r_serv_term = Flask_config.r_serv_term
r_serv_cred = Flask_config.r_serv_cred r_serv_cred = Flask_config.r_serv_cred
r_serv_db = Flask_config.r_serv_db
bootstrap_label = Flask_config.bootstrap_label
terms = Blueprint('terms', __name__, template_folder='templates') terms = Blueprint('terms', __name__, template_folder='templates')
@ -51,6 +53,7 @@ TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
# same value as in `bin/NotificationHelper.py` # same value as in `bin/NotificationHelper.py`
# Keys will be e.g. TrackedNotificationEmails_<TERMNAME> # Keys will be e.g. TrackedNotificationEmails_<TERMNAME>
TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_" TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
'''CRED''' '''CRED'''
REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+' REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+'
@ -130,6 +133,12 @@ def mixUserName(supplied, extensive=False):
filtered_usernames.append(usr) filtered_usernames.append(usr)
return filtered_usernames return filtered_usernames
def save_tag_to_auto_push(list_tag):
for tag in set(list_tag):
#limit tag length
if len(tag) > 49:
tag = tag[0:48]
r_serv_db.sadd('list_export_tags', tag)
# ============ ROUTES ============ # ============ ROUTES ============
@ -152,6 +161,7 @@ def terms_management():
# Maps a specific term to the associated email addresses # Maps a specific term to the associated email addresses
notificationEMailTermMapping = {} notificationEMailTermMapping = {}
notificationTagsTermMapping = {}
#Regex #Regex
trackReg_list = [] trackReg_list = []
@ -159,7 +169,8 @@ def terms_management():
trackReg_list_num_of_paste = [] trackReg_list_num_of_paste = []
for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name): for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name):
notificationEMailTermMapping[tracked_regex] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)) ) notificationEMailTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)
notificationTagsTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_regex)
if tracked_regex not in notificationEnabledDict: if tracked_regex not in notificationEnabledDict:
notificationEnabledDict[tracked_regex] = False notificationEnabledDict[tracked_regex] = False
@ -185,8 +196,8 @@ def terms_management():
for tracked_set in r_serv_term.smembers(TrackedSetSet_Name): for tracked_set in r_serv_term.smembers(TrackedSetSet_Name):
tracked_set = tracked_set tracked_set = tracked_set
notificationEMailTermMapping[tracked_set] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)) ) notificationEMailTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)
notificationTagsTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_set)
if tracked_set not in notificationEnabledDict: if tracked_set not in notificationEnabledDict:
notificationEnabledDict[tracked_set] = False notificationEnabledDict[tracked_set] = False
@ -211,7 +222,8 @@ def terms_management():
track_list_num_of_paste = [] track_list_num_of_paste = []
for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name): for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name):
notificationEMailTermMapping[tracked_term] = "\n".join( r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term)) notificationEMailTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term)
notificationTagsTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_term)
if tracked_term not in notificationEnabledDict: if tracked_term not in notificationEnabledDict:
notificationEnabledDict[tracked_term] = False notificationEnabledDict[tracked_term] = False
@ -244,7 +256,8 @@ def terms_management():
track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste, track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste,
trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste, trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste,
trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste, trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste,
per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, notificationEMailTermMapping=notificationEMailTermMapping) per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, bootstrap_label=bootstrap_label,
notificationEMailTermMapping=notificationEMailTermMapping, notificationTagsTermMapping=notificationTagsTermMapping)
@terms.route("/terms_management_query_paste/") @terms.route("/terms_management_query_paste/")
@ -313,6 +326,7 @@ def terms_management_action():
action = request.args.get('action') action = request.args.get('action')
term = request.args.get('term') term = request.args.get('term')
notificationEmailsParam = request.args.get('emailAddresses') notificationEmailsParam = request.args.get('emailAddresses')
input_tags = request.args.get('tags')
if action is None or term is None or notificationEmailsParam is None: if action is None or term is None or notificationEmailsParam is None:
return "None" return "None"
@ -320,11 +334,8 @@ def terms_management_action():
if section == "followTerm": if section == "followTerm":
if action == "add": if action == "add":
# Strip all whitespace
notificationEmailsParam = "".join(notificationEmailsParam.split())
# Make a list of all passed email addresses # Make a list of all passed email addresses
notificationEmails = notificationEmailsParam.split(",") notificationEmails = notificationEmailsParam.split()
validNotificationEmails = [] validNotificationEmails = []
# check for valid email addresses # check for valid email addresses
@ -334,6 +345,8 @@ def terms_management_action():
if re.match(r"[^@]+@[^@]+\.[^@]+", email): if re.match(r"[^@]+@[^@]+\.[^@]+", email):
validNotificationEmails.append(email) validNotificationEmails.append(email)
# create tags list
list_tags = input_tags.split()
# check if regex/set or simple term # check if regex/set or simple term
#regex #regex
@ -345,6 +358,10 @@ def terms_management_action():
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email) r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email)
# enable notifications by default # enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term) r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term)
# add tags list
for tag in list_tags:
r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term, tag)
save_tag_to_auto_push(list_tags)
#set #set
elif term.startswith('\\') and term.endswith('\\'): elif term.startswith('\\') and term.endswith('\\'):
@ -363,6 +380,10 @@ def terms_management_action():
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + set_to_add, email) r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + set_to_add, email)
# enable notifications by default # enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, set_to_add) r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, set_to_add)
# add tags list
for tag in list_tags:
r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + set_to_add, tag)
save_tag_to_auto_push(list_tags)
#simple term #simple term
else: else:
@ -373,6 +394,10 @@ def terms_management_action():
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email) r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email)
# enable notifications by default # enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower()) r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower())
# add tags list
for tag in list_tags:
r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term.lower(), tag)
save_tag_to_auto_push(list_tags)
elif action == "toggleEMailNotification": elif action == "toggleEMailNotification":
# get the current state # get the current state
@ -397,6 +422,8 @@ def terms_management_action():
# delete the associated notification emails too # delete the associated notification emails too
r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term) r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term)
# delete the associated tags set
r_serv_term.delete(TrackedTermsNotificationTagsPrefix_Name + term)
elif section == "blacklistTerm": elif section == "blacklistTerm":
if action == "add": if action == "add":
@ -413,6 +440,28 @@ def terms_management_action():
to_return["term"] = term to_return["term"] = term
return jsonify(to_return) return jsonify(to_return)
@terms.route("/terms_management/delete_terms_tags", methods=['POST'])
def delete_terms_tags():
term = request.form.get('term')
tags_to_delete = request.form.getlist('tags_to_delete')
if term is not None and tags_to_delete is not None:
for tag in tags_to_delete:
r_serv_term.srem(TrackedTermsNotificationTagsPrefix_Name + term, tag)
return redirect(url_for('terms.terms_management'))
else:
return 'None args', 400
@terms.route("/terms_management/delete_terms_email", methods=['GET'])
def delete_terms_email():
term = request.args.get('term')
email = request.args.get('email')
if term is not None and email is not None:
r_serv_term.srem(TrackedTermsNotificationEmailsPrefix_Name + term, email)
return redirect(url_for('terms.terms_management'))
else:
return 'None args', 400
@terms.route("/terms_plot_tool/") @terms.route("/terms_plot_tool/")

View file

@ -36,6 +36,12 @@
white-space:pre-wrap; white-space:pre-wrap;
word-wrap:break-word; word-wrap:break-word;
} }
.mouse_pointer{
cursor: pointer;
}
.lb-md {
font-size: 16px;
}
</style> </style>
</head> </head>
<body> <body>
@ -74,6 +80,7 @@
<div class="row"> <div class="row">
<div class="col-lg-12"> <div class="col-lg-12">
<div class="row"> <div class="row">
{% set uniq_id = namespace(modal_id=0)%}
<div class="col-lg-12"> <div class="col-lg-12">
<label class="switch"> <label class="switch">
<input id="per_paste" class="switch-input" value="per_paste" type="checkbox" onclick="reload_per_paste()"> <input id="per_paste" class="switch-input" value="per_paste" type="checkbox" onclick="reload_per_paste()">
@ -98,7 +105,8 @@
<div class="form-group input-group" style="margin-bottom: 30px;"> <div class="form-group input-group" style="margin-bottom: 30px;">
<span class="input-group-addon"><span class="fa fa-eye"></span></span> <span class="input-group-addon"><span class="fa fa-eye"></span></span>
<input id="followTermInput" class="form-control" placeholder="Term to track." type="text" style="max-width: 400px;"> <input id="followTermInput" class="form-control" placeholder="Term to track." type="text" style="max-width: 400px;">
<input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (comma separated)" type="text" style="max-width: 400px;"> <input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (optional, space separated)" type="text" style="max-width: 400px;">
<input id="followTermTag" class="form-control" placeholder="Tags (optional, space separated)" type="text" style="max-width: 400px;">
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button> <button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button>
</div> </div>
@ -119,7 +127,57 @@
<!-- SET --> <!-- SET -->
{% for set in trackSet_list %} {% for set in trackSet_list %}
<tr style="background-color: #cdffca;"> <tr style="background-color: #cdffca;">
<td>{{ set }}</td> <td>
<span class="term_name">{{ set }}</span>
<div>
{% for tag in notificationTagsTermMapping[set] %}
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
</a>
{% endfor %}
{% if notificationTagsTermMapping[set] %}
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
<div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
<div class="modal-dialog">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
<h2 class="text-center">Remove Custom Tag</h2>
</div>
<div class="modal-body">
<form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
{% for tag in notificationTagsTermMapping[set] %}
<div class="form-check">
<input type="hidden" class="form-control" name="term" value="{{ set }}">
<input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
<label class="form-check-label">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
</label>
<br>
</div>
{% endfor %}
</form>
</div>
<div class="modal-footer">
<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
<span class="glyphicon glyphicon-trash"></span>
<span class="label-icon">Remove Tags</span>
</button>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
{% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
{% endif %}
</div>
</td>
<td>{{ trackSet_list_values[loop.index0][3] }}</td> <td>{{ trackSet_list_values[loop.index0][3] }}</td>
<td>{{ trackSet_list_values[loop.index0][0] }}</td> <td>{{ trackSet_list_values[loop.index0][0] }}</td>
<td>{{ trackSet_list_values[loop.index0][1] }}</td> <td>{{ trackSet_list_values[loop.index0][1] }}</td>
@ -130,13 +188,73 @@
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ set }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button> <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ set }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ set }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[set] %} checked {% endif %}> &nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ set }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[set] %} checked {% endif %}>
</p></td> </p></td>
<td style="white-space:pre">{{ notificationEMailTermMapping[set] }}</td> <td>
{% for email in notificationEMailTermMapping[set] %}
<a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{set}}">
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
<span class="glyphicon glyphicon-trash" style="color:Red;" ></span>
</div>
</a>
{{ email }}
<br>
{% endfor %}
</td>
</tr> </tr>
{% endfor %} {% endfor %}
<!-- REGEX --> <!-- REGEX -->
{% for regex in trackReg_list %} {% for regex in trackReg_list %}
<tr style="background-color: #fffdca;"> <tr style="background-color: #fffdca;">
<td>{{ regex }}</td> <td>
<span class="term_name">{{ regex }}</span>
<div>
{% for tag in notificationTagsTermMapping[regex] %}
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
</a>
{% endfor %}
{% if notificationTagsTermMapping[regex] %}
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
<div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
<div class="modal-dialog">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
<h2 class="text-center">Remove Custom Tag</h2>
</div>
<div class="modal-body">
<form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
{% for tag in notificationTagsTermMapping[regex] %}
<div class="form-check">
<input type="hidden" class="form-control" name="term" value="{{ regex }}">
<input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
<label class="form-check-label">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
</label>
<br>
</div>
{% endfor %}
</form>
</div>
<div class="modal-footer">
<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
<span class="glyphicon glyphicon-trash"></span>
<span class="label-icon">Remove Tags</span>
</button>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
{% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
{% endif %}
</div>
</td>
<td>{{ trackReg_list_values[loop.index0][3] }}</td> <td>{{ trackReg_list_values[loop.index0][3] }}</td>
<td>{{ trackReg_list_values[loop.index0][0] }}</td> <td>{{ trackReg_list_values[loop.index0][0] }}</td>
<td>{{ trackReg_list_values[loop.index0][1] }}</td> <td>{{ trackReg_list_values[loop.index0][1] }}</td>
@ -147,13 +265,73 @@
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ regex }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button> <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ regex }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ regex }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[regex] %} checked {% endif %}> &nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ regex }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[regex] %} checked {% endif %}>
</p></td> </p></td>
<td style="white-space:pre">{{ notificationEMailTermMapping[regex] }}</td> <td>
{% for email in notificationEMailTermMapping[regex] %}
<a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{regex}}">
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
<span class="glyphicon glyphicon-trash" style="color:Red;"></span>
</div>
</a>
{{ email }}
<br>
{% endfor %}
</td>
</tr> </tr>
{% endfor %} {% endfor %}
<!-- Normal term --> <!-- Normal term -->
{% for term in track_list %} {% for term in track_list %}
<tr> <tr>
<td>{{ term }}</td> <td>
<span class="term_name">{{ term }}</span>
<div>
{% for tag in notificationTagsTermMapping[term] %}
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
</a>
{% endfor %}
{% if notificationTagsTermMapping[term] %}
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
<div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
<div class="modal-dialog">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
<h2 class="text-center">Remove Custom Tag</h2>
</div>
<div class="modal-body">
<form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
{% for tag in notificationTagsTermMapping[term] %}
<div class="form-check">
<input type="hidden" class="form-control" name="term" value="{{ term }}">
<input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
<label class="form-check-label">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
</label>
<br>
</div>
{% endfor %}
</form>
</div>
<div class="modal-footer">
<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
<span class="glyphicon glyphicon-trash"></span>
<span class="label-icon">Remove Tags</span>
</button>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
{% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
{% endif %}
</div>
</td>
<td>{{ track_list_values[loop.index0][3] }}</td> <td>{{ track_list_values[loop.index0][3] }}</td>
<td>{{ track_list_values[loop.index0][0] }}</td> <td>{{ track_list_values[loop.index0][0] }}</td>
<td>{{ track_list_values[loop.index0][1] }}</td> <td>{{ track_list_values[loop.index0][1] }}</td>
@ -164,7 +342,17 @@
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ term }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button> <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ term }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ term }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[term] %} checked {% endif %}> &nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ term }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[term] %} checked {% endif %}>
</p></td> </p></td>
<td style="white-space:pre">{{ notificationEMailTermMapping[term] }}</td> <td>
{% for email in notificationEMailTermMapping[term] %}
<a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{term}}">
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
<span class="glyphicon glyphicon-trash" style="color:Red;"></span>
</div>
</a>
{{ email }}
<br>
{% endfor %}
</td>
</tr> </tr>
{% endfor %} {% endfor %}
</tbody> </tbody>
@ -351,17 +539,19 @@ function perform_binding() {
function perform_operation(){ function perform_operation(){
var curr_section = $(this).attr('data-section'); var curr_section = $(this).attr('data-section');
var curr_action = $(this).attr('data-action'); var curr_action = $(this).attr('data-action');
var row_tr = $(this).closest("tr");
if (curr_action == "add") { if (curr_action == "add") {
var curr_term = $('#'+curr_section+'Input').val(); var curr_term = $('#'+curr_section+'Input').val();
var email_addresses = $('#followTermEMailNotificationReceiversInput').val(); var email_addresses = $('#followTermEMailNotificationReceiversInput').val();
var tags = $('#followTermTag').val();
} else { } else {
var curr_term = $(this).attr('data-content'); var curr_term = $(this).attr('data-content');
var email_addresses = ""; var email_addresses = "";
} }
var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses}; var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses, tags: tags};
if (curr_term != "") { if (curr_term != "") {
console.log(data_to_send); //console.log(data_to_send);
$.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){ $.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){
if(status == "success") { if(status == "success") {
var json = data; var json = data;
@ -373,12 +563,7 @@ function perform_operation(){
reload_per_paste(); reload_per_paste();
}); });
} else if (json.action == "delete") { } else if (json.action == "delete") {
// Find indexes of row which have the term in the first column row_tr.remove()
var index = table_track.rows().eq( 0 ).filter( function (rowIdx) {
console.log(table_track.cell( rowIdx, 0 ).data())
return table_track.cell( rowIdx, 0 ).data() === json.term;
} );
table_track.rows(index).remove().draw( false );
} }
} else if(json.section == "blacklistTerm"){ } else if(json.section == "blacklistTerm"){
if(json.action == "add") { if(json.action == "add") {

View file

@ -86,11 +86,11 @@ popd
#active virtualenv #active virtualenv
source ./../../AILENV/bin/activate source ./../../AILENV/bin/activate
#Update MISP Taxonomies and Galaxies #Update MISP Taxonomies and Galaxies
python3 -m pip install git+https://github.com/MISP/PyTaxonomies python3 -m pip install git+https://github.com/MISP/PyTaxonomies --upgrade
python3 -m pip install git+https://github.com/MISP/PyMISPGalaxies python3 -m pip install git+https://github.com/MISP/PyMISPGalaxies --upgrade
#Update PyMISP #Update PyMISP
python3 -m pip install git+https://github.com/MISP/PyMISP python3 -m pip install git+https://github.com/MISP/PyMISP --upgrade
#Update the Hive #Update the Hive
python3 -m pip install git+https://github.com/TheHive-Project/TheHive4py python3 -m pip install git+https://github.com/TheHive-Project/TheHive4py --upgrade