diff --git a/.gitignore b/.gitignore index d62fbd8b..4a266743 100644 --- a/.gitignore +++ b/.gitignore @@ -7,10 +7,15 @@ AILENV redis-leveldb redis +ardb +faup +tlsh Blooms LEVEL_DB_DATA PASTES -bin/indexdir/ +BASE64 +DATA_ARDB +indexdir/ logs/ # Webstuff @@ -20,7 +25,7 @@ var/www/static/ !var/www/static/js/moduleTrending.js !var/www/static/js/plot-graph.js !var/www/static/js/trendingchart.js -var/www/templates/header.html # auto-generated +var/www/templates/header.html # Local config bin/packages/config.cfg @@ -28,4 +33,6 @@ bin/packages/config.cfg # installed files nltk_data/ doc/all_modules.txt -doc/module-data-flow.png # auto-generated +# auto generated +doc/module-data-flow.png +doc/data-flow.png diff --git a/README.md b/README.md index b977c185..74426e4d 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ Features * Detect Amazon AWS and Google API keys * Detect Bitcoin address and Bitcoin private keys * Detect private keys and certificate +* Tagging system with [MISP Galaxy](https://github.com/MISP/misp-galaxy) and [MISP Taxonomies](https://github.com/MISP/misp-taxonomies) tags Installation ------------ @@ -144,6 +145,11 @@ Browsing ![Browse-Pastes](./doc/screenshots/browse-important.png?raw=true "AIL framework browseImportantPastes") +Tagging system +-------- + +![Tags](./doc/screenshots/tags.png?raw=true "AIL framework tags") + Sentiment analysis ------------------ diff --git a/bin/ApiKey.py b/bin/ApiKey.py index 8ce7e2b4..e7ded9b2 100755 --- a/bin/ApiKey.py +++ b/bin/ApiKey.py @@ -41,6 +41,8 @@ def search_api_key(message): print(to_print) publisher.warning('{}Checked {} found Google API Key;{}'.format( to_print, len(google_api_key), paste.p_path)) + msg = 'infoleak:automatic-detection="google-api-key";{}'.format(filename) + p.populate_set_out(msg, 'Tags') if(len(aws_access_key) > 0 or len(aws_secret_key) > 0): print('found AWS key') @@ -48,8 +50,13 @@ def search_api_key(message): total = len(aws_access_key) + len(aws_secret_key) publisher.warning('{}Checked {} found AWS Key;{}'.format( to_print, total, paste.p_path)) + msg = 'infoleak:automatic-detection="aws-key";{}'.format(filename) + p.populate_set_out(msg, 'Tags') + msg = 'infoleak:automatic-detection="api-key";{}'.format(filename) + p.populate_set_out(msg, 'Tags') + msg = 'apikey;{}'.format(filename) p.populate_set_out(msg, 'alertHandler') #Send to duplicate diff --git a/bin/Base64.py b/bin/Base64.py index c7700994..960ca6de 100755 --- a/bin/Base64.py +++ b/bin/Base64.py @@ -65,6 +65,9 @@ def search_base64(content, message): msg = ('base64;{}'.format(message)) p.populate_set_out( msg, 'alertHandler') + msg = 'infoleak:automatic-detection="base64";{}'.format(message) + p.populate_set_out(msg, 'Tags') + def save_base64_as_file(decode, type, hash, json_data): filename_b64 = os.path.join(os.environ['AIL_HOME'], diff --git a/bin/Bitcoin.py b/bin/Bitcoin.py index 42468759..5ec2199f 100755 --- a/bin/Bitcoin.py +++ b/bin/Bitcoin.py @@ -63,7 +63,14 @@ def search_key(content, message, paste): publisher.warning(to_print) msg = ('bitcoin;{}'.format(message)) p.populate_set_out( msg, 'alertHandler') + + msg = 'infoleak:automatic-detection="bitcoin-address";{}'.format(message) + p.populate_set_out(msg, 'Tags') + if(key): + msg = 'infoleak:automatic-detection="bitcoin-private-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') + to_print = 'Bitcoin;{};{};{};'.format(paste.p_source, paste.p_date, paste.p_name) publisher.warning('{}Detected {} Bitcoin private key;{}'.format( diff --git a/bin/Credential.py b/bin/Credential.py index fde80d12..5112f534 100755 --- a/bin/Credential.py +++ b/bin/Credential.py @@ -105,6 +105,9 @@ if __name__ == "__main__": msg = 'credential;{}'.format(filepath) p.populate_set_out(msg, 'alertHandler') + msg = 'infoleak:automatic-detection="credential";{}'.format(filepath) + p.populate_set_out(msg, 'Tags') + #Put in form, count occurences, then send to moduleStats creds_sites = {} site_occurence = re.findall(regex_site_for_stats, content) diff --git a/bin/CreditCards.py b/bin/CreditCards.py index a7441807..260d1345 100755 --- a/bin/CreditCards.py +++ b/bin/CreditCards.py @@ -85,6 +85,9 @@ if __name__ == "__main__": #send to Browse_warning_paste msg = 'creditcard;{}'.format(filename) p.populate_set_out(msg, 'alertHandler') + + msg = 'infoleak:automatic-detection="credit-card";{}'.format(filename) + p.populate_set_out(msg, 'Tags') else: publisher.info('{}CreditCard related;{}'.format(to_print, paste.p_path)) else: diff --git a/bin/Cve.py b/bin/Cve.py index 9ac4efc8..bd240260 100755 --- a/bin/Cve.py +++ b/bin/Cve.py @@ -34,6 +34,9 @@ def search_cve(message): #send to Browse_warning_paste msg = 'cve;{}'.format(filepath) p.populate_set_out(msg, 'alertHandler') + + msg = 'infoleak:automatic-detection="cve";{}'.format(filepath) + p.populate_set_out(msg, 'Tags') #Send to duplicate p.populate_set_out(filepath, 'Duplicate') diff --git a/bin/Keys.py b/bin/Keys.py index 9f39cf50..7b1ec7dc 100755 --- a/bin/Keys.py +++ b/bin/Keys.py @@ -28,47 +28,76 @@ def search_key(paste): if '-----BEGIN PGP MESSAGE-----' in content: publisher.warning('{} has a PGP enc message'.format(paste.p_name)) + msg = 'infoleak:automatic-detection="pgp-message";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN CERTIFICATE-----' in content: publisher.warning('{} has a certificate message'.format(paste.p_name)) + + msg = 'infoleak:automatic-detection="certificate";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN RSA PRIVATE KEY-----' in content: publisher.warning('{} has a RSA private key message'.format(paste.p_name)) print('rsa private key message found') + + msg = 'infoleak:automatic-detection="rsa-private-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN PRIVATE KEY-----' in content: publisher.warning('{} has a private key message'.format(paste.p_name)) print('private key message found') + + msg = 'infoleak:automatic-detection="private-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN ENCRYPTED PRIVATE KEY-----' in content: publisher.warning('{} has an encrypted private key message'.format(paste.p_name)) print('encrypted private key message found') + + msg = 'infoleak:automatic-detection="encrypted-private-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN OPENSSH PRIVATE KEY-----' in content: publisher.warning('{} has an openssh private key message'.format(paste.p_name)) print('openssh private key message found') + + msg = 'infoleak:automatic-detection="private-ssh-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN OpenVPN Static key V1-----' in content: publisher.warning('{} has an openssh private key message'.format(paste.p_name)) print('OpenVPN Static key message found') + + msg = 'infoleak:automatic-detection="vpn-static-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN DSA PRIVATE KEY-----' in content: publisher.warning('{} has a dsa private key message'.format(paste.p_name)) + + msg = 'infoleak:automatic-detection="dsa-private-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN EC PRIVATE KEY-----' in content: publisher.warning('{} has an ec private key message'.format(paste.p_name)) + + msg = 'infoleak:automatic-detection="ec-private-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN PGP PRIVATE KEY BLOCK-----' in content: publisher.warning('{} has a pgp private key block message'.format(paste.p_name)) + + msg = 'infoleak:automatic-detection="pgp-private-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') find = True if find : diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index aca72e8e..ed05f676 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -160,6 +160,8 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "alertHandler" bash -c './alertHandler.py; read x' sleep 0.1 + screen -S "Script_AIL" -X screen -t "Tags" bash -c './Tags.py; read x' + sleep 0.1 screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x' } @@ -228,7 +230,7 @@ islogged=`screen -ls | egrep '[0-9]+.Logging_AIL' | cut -d. -f1` isqueued=`screen -ls | egrep '[0-9]+.Queue_AIL' | cut -d. -f1` isscripted=`screen -ls | egrep '[0-9]+.Script_AIL' | cut -d. -f1` -options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Killall" "Shutdown" "Update-config") +options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Killall" "Shutdown" "Update-config" "Update-thirdparty") menu() { echo "What do you want to Launch?:" @@ -328,6 +330,17 @@ for i in ${!options[@]}; do echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT fi ;; + Update-thirdparty) + echo -e "\t* Updating thirdparty..." + bash -c "(cd ../var/www && ./update_thirdparty.sh)" + exitStatus=$? + if [ $exitStatus -ge 1 ]; then + echo -e $RED"\t* Configuration not up-to-date"$DEFAULT + exit + else + echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT + fi + ;; esac fi done diff --git a/bin/Mail.py b/bin/Mail.py index abc112a6..c1d8cf70 100755 --- a/bin/Mail.py +++ b/bin/Mail.py @@ -76,6 +76,9 @@ if __name__ == "__main__": p.populate_set_out(filename, 'Duplicate') p.populate_set_out('mail;{}'.format(filename), 'alertHandler') + msg = 'infoleak:automatic-detection="mail";{}'.format(filename) + p.populate_set_out(msg, 'Tags') + else: publisher.info(to_print) #Send to ModuleStats diff --git a/bin/Onion.py b/bin/Onion.py index 77ed75fe..277f1c71 100755 --- a/bin/Onion.py +++ b/bin/Onion.py @@ -152,6 +152,9 @@ if __name__ == "__main__": for url in fetch(p, r_cache, urls, domains_list, path): publisher.info('{}Checked {};{}'.format(to_print, url, PST.p_path)) p.populate_set_out('onion;{}'.format(PST.p_path), 'alertHandler') + + msg = 'infoleak:automatic-detection="onion";{}'.format(PST.p_path) + p.populate_set_out(msg, 'Tags') else: publisher.info('{}Onion related;{}'.format(to_print, PST.p_path)) diff --git a/bin/Phone.py b/bin/Phone.py index e3f0f908..213db2b3 100755 --- a/bin/Phone.py +++ b/bin/Phone.py @@ -36,6 +36,10 @@ def search_phone(message): msg = 'phone;{}'.format(message) p.populate_set_out(msg, 'alertHandler') #Send to duplicate + + msg = 'infoleak:automatic-detection="phone-number";{}'.format(message) + p.populate_set_out(msg, 'Tags') + p.populate_set_out(message, 'Duplicate') stats = {} for phone_number in results: diff --git a/bin/SQLInjectionDetection.py b/bin/SQLInjectionDetection.py index 9e28de72..117f3dc0 100755 --- a/bin/SQLInjectionDetection.py +++ b/bin/SQLInjectionDetection.py @@ -82,6 +82,9 @@ def analyse(url, path): p.populate_set_out(path, 'Duplicate') #send to Browse_warning_paste p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler') + + msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path) + p.populate_set_out(msg, 'Tags') else: print("Potential SQL injection:") print(urllib.request.unquote(url)) diff --git a/bin/Tags.py b/bin/Tags.py new file mode 100755 index 00000000..f4939ec3 --- /dev/null +++ b/bin/Tags.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +""" +The Tags Module +================================ + +This module create tags. + +""" +import redis + +import time + +from pubsublogger import publisher +from Helper import Process +from packages import Paste + +if __name__ == '__main__': + + # Port of the redis instance used by pubsublogger + publisher.port = 6380 + # Script is the default channel used for the modules. + publisher.channel = 'Script' + + # Section name in bin/packages/modules.cfg + config_section = 'Tags' + + # Setup the I/O queues + p = Process(config_section) + + server = redis.StrictRedis( + host=p.config.get("ARDB_Tags", "host"), + port=p.config.get("ARDB_Tags", "port"), + db=p.config.get("ARDB_Tags", "db"), + decode_responses=True) + + server_metadata = redis.StrictRedis( + host=p.config.get("ARDB_Metadata", "host"), + port=p.config.get("ARDB_Metadata", "port"), + db=p.config.get("ARDB_Metadata", "db"), + decode_responses=True) + + # Sent to the logging a description of the module + publisher.info("Tags module started") + + # Endless loop getting messages from the input queue + while True: + # Get one message from the input queue + message = p.get_from_set() + + if message is None: + publisher.debug("{} queue is empty, waiting 10s".format(config_section)) + time.sleep(10) + continue + + else: + tag, path = message.split(';') + # add the tag to the tags word_list + res = server.sadd('list_tags', tag) + if res == 1: + print("new tags added : {}".format(tag)) + # add the path to the tag set + res = server.sadd(tag, path) + if res == 1: + print("new paste: {}".format(path)) + print(" tagged: {}".format(tag)) + server_metadata.sadd('tag:'+path, tag) diff --git a/bin/packages/Paste.py b/bin/packages/Paste.py index ccaf3400..317743f4 100755 --- a/bin/packages/Paste.py +++ b/bin/packages/Paste.py @@ -340,7 +340,7 @@ class Paste(object): Save a new duplicate on others pastes """ for hash_type, path, percent, date in list_value: - to_add = [hash_type, self.p_path, percent, date] + to_add = (hash_type, self.p_path, percent, date) self.store_duplicate.sadd('dup:'+path,to_add) def _get_from_redis(self, r_serv): diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index b9e29506..975b7b2c 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -49,15 +49,15 @@ publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Re [CreditCards] subscribe = Redis_CreditCards -publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler +publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler,Redis_Tags [Mail] subscribe = Redis_Mail -publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler +publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler,Redis_Tags [Onion] subscribe = Redis_Onion -publish = Redis_ValidOnion,ZMQ_FetchedOnion,Redis_alertHandler +publish = Redis_ValidOnion,ZMQ_FetchedOnion,Redis_alertHandler,Redis_Tags #publish = Redis_Global,Redis_ValidOnion,ZMQ_FetchedOnion,Redis_alertHandler [DumpValidOnion] @@ -72,7 +72,7 @@ subscribe = Redis_Url [SQLInjectionDetection] subscribe = Redis_Url -publish = Redis_alertHandler,Redis_Duplicate +publish = Redis_alertHandler,Redis_Duplicate,Redis_Tags [ModuleStats] subscribe = Redis_ModuleStats @@ -80,9 +80,12 @@ subscribe = Redis_ModuleStats [alertHandler] subscribe = Redis_alertHandler +[Tags] +subscribe = Redis_Tags + #[send_to_queue] #subscribe = Redis_Cve -#publish = Redis_alertHandler +#publish = Redis_alertHandler,Redis_Tags [SentimentAnalysis] subscribe = Redis_Global @@ -92,28 +95,28 @@ subscribe = Redis_Global [Credential] subscribe = Redis_Credential -publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler +publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler,Redis_Tags [Cve] subscribe = Redis_Cve -publish = Redis_alertHandler,Redis_Duplicate +publish = Redis_alertHandler,Redis_Duplicate,Redis_Tags [Phone] subscribe = Redis_Global -publish = Redis_Duplicate,Redis_alertHandler +publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags [Keys] subscribe = Redis_Global -publish = Redis_Duplicate,Redis_alertHandler +publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags [ApiKey] subscribe = Redis_ApiKey -publish = Redis_Duplicate,Redis_alertHandler +publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags [Base64] subscribe = Redis_Global -publish = Redis_Duplicate,Redis_alertHandler +publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags [Bitcoin] subscribe = Redis_Global -publish = Redis_Duplicate,Redis_alertHandler +publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags diff --git a/doc/screenshots/galaxies_list.png b/doc/screenshots/galaxies_list.png new file mode 100644 index 00000000..31ae76d8 Binary files /dev/null and b/doc/screenshots/galaxies_list.png differ diff --git a/doc/screenshots/galaxy_tag_edit.png b/doc/screenshots/galaxy_tag_edit.png new file mode 100644 index 00000000..93806873 Binary files /dev/null and b/doc/screenshots/galaxy_tag_edit.png differ diff --git a/doc/screenshots/paste_tags_edit.png b/doc/screenshots/paste_tags_edit.png new file mode 100644 index 00000000..7127e43a Binary files /dev/null and b/doc/screenshots/paste_tags_edit.png differ diff --git a/doc/screenshots/tag_delete_confirm.png b/doc/screenshots/tag_delete_confirm.png new file mode 100644 index 00000000..88ad5197 Binary files /dev/null and b/doc/screenshots/tag_delete_confirm.png differ diff --git a/doc/screenshots/tags.png b/doc/screenshots/tags.png new file mode 100644 index 00000000..11ce1812 Binary files /dev/null and b/doc/screenshots/tags.png differ diff --git a/doc/screenshots/tags2.png b/doc/screenshots/tags2.png new file mode 100644 index 00000000..81ce4853 Binary files /dev/null and b/doc/screenshots/tags2.png differ diff --git a/doc/screenshots/tags_search.png b/doc/screenshots/tags_search.png new file mode 100644 index 00000000..892e15da Binary files /dev/null and b/doc/screenshots/tags_search.png differ diff --git a/installing_deps.sh b/installing_deps.sh index 0cc9b2f2..6bb73f6b 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -26,10 +26,11 @@ sudo apt-get install libev-dev libgmp-dev -y #Need for generate-data-flow graph sudo apt-get install graphviz -y -#needed for mathplotlib -sudo easy_install -U distribute +# install nosetests +sudo pip install nose -y + # ssdeep -sudo apt-get install libfuzzy-dev +sudo apt-get install libfuzzy-dev -y sudo apt-get install build-essential libffi-dev automake autoconf libtool -y # REDIS # @@ -70,10 +71,6 @@ if [ ! -f bin/packages/config.cfg ]; then cp bin/packages/config.cfg.sample bin/packages/config.cfg fi -pushd var/www/ -sudo ./update_thirdparty.sh -popd - if [ -z "$VIRTUAL_ENV" ]; then virtualenv -p python3 AILENV @@ -88,6 +85,10 @@ if [ -z "$VIRTUAL_ENV" ]; then fi +pushd var/www/ +./update_thirdparty.sh +popd + year1=20`date +%y` year2=20`date --date='-1 year' +%y` mkdir -p $AIL_HOME/{PASTES,Blooms,dumps} @@ -102,8 +103,6 @@ popd # Py tlsh pushd tlsh/py_ext -#python setup.py build -#python setup.py install python3 setup.py build python3 setup.py install @@ -112,8 +111,5 @@ HOME=$(pwd) python3 -m textblob.download_corpora python3 -m nltk.downloader vader_lexicon python3 -m nltk.downloader punkt -# install nosetests -sudo pip install nose - #Create the file all_module and update the graph in doc $AIL_HOME/doc/generate_modules_data_flow_graph.sh diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 0be6854a..a03999ab 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -18,6 +18,8 @@ sys.path.append('./modules/') import Paste from Date import Date +from pytaxonomies import Taxonomies + # Import config import Flask_config @@ -113,6 +115,25 @@ def searchbox(): return render_template("searchbox.html") +# ========== INITIAL taxonomies ============ +r_serv_tags = redis.StrictRedis( + host=cfg.get("ARDB_Tags", "host"), + port=cfg.getint("ARDB_Tags", "port"), + db=cfg.getint("ARDB_Tags", "db"), + decode_responses=True) +# add default ail taxonomies +r_serv_tags.sadd('active_taxonomies', 'infoleak') +r_serv_tags.sadd('active_taxonomies', 'gdpr') +r_serv_tags.sadd('active_taxonomies', 'fpf') +# add default tags +taxonomies = Taxonomies() +for tag in taxonomies.get('infoleak').machinetags(): + r_serv_tags.sadd('active_tag_infoleak', tag) +for tag in taxonomies.get('gdpr').machinetags(): + r_serv_tags.sadd('active_tag_infoleak', tag) +for tag in taxonomies.get('fpf').machinetags(): + r_serv_tags.sadd('active_tag_infoleak', tag) + # ============ MAIN ============ if __name__ == "__main__": diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index 41745f21..26edccfa 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -82,3 +82,5 @@ max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal DiffMaxLineLength = int(cfg.get("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value + +bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] diff --git a/var/www/modules/Tags/Flask_Tags.py b/var/www/modules/Tags/Flask_Tags.py new file mode 100644 index 00000000..a8dd4c8b --- /dev/null +++ b/var/www/modules/Tags/Flask_Tags.py @@ -0,0 +1,845 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Flask functions and routes for the trending modules page +''' +import redis +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for + +import json +from datetime import datetime + +import Paste + +from pytaxonomies import Taxonomies +from pymispgalaxies import Galaxies, Clusters + +# ============ VARIABLES ============ +import Flask_config + +app = Flask_config.app +cfg = Flask_config.cfg +r_serv_tags = Flask_config.r_serv_tags +r_serv_metadata = Flask_config.r_serv_metadata +max_preview_char = Flask_config.max_preview_char +max_preview_modal = Flask_config.max_preview_modal +bootstrap_label = Flask_config.bootstrap_label + +Tags = Blueprint('Tags', __name__, template_folder='templates') + +galaxies = Galaxies() +clusters = Clusters(skip_duplicates=True) + +list_all_tags = {} +for name, c in clusters.items(): #galaxy name + tags + list_all_tags[name] = c + +list_galaxies = [] +for g in galaxies.values(): + list_galaxies.append(g.to_json()) + +list_clusters = [] +for c in clusters.values(): + list_clusters.append(c.to_json()) + +# tags numbers in galaxies +total_tags = {} +for name, tags in clusters.items(): #galaxie name + tags + total_tags[name] = len(tags) + +# ============ FUNCTIONS ============ +def one(): + return 1 + +def get_tags_with_synonyms(tag): + str_synonyms = ' - synonyms: ' + synonyms = r_serv_tags.smembers('synonym_tag_' + tag) + # synonyms to display + for synonym in synonyms: + str_synonyms = str_synonyms + synonym + ', ' + # add real tag + if str_synonyms != ' - synonyms: ': + return {'name':tag + str_synonyms,'id':tag} + else: + return {'name':tag,'id':tag} + +# ============= ROUTES ============== + +@Tags.route("/Tags/", methods=['GET']) +def Tags_page(): + return render_template("Tags.html") + +@Tags.route("/Tags/get_all_tags") +def get_all_tags(): + + all_tags = r_serv_tags.smembers('list_tags') + + list_tags = [] + for tag in all_tags: + t = tag.split(':')[0] + # add synonym + str_synonyms = ' - synonyms: ' + if t == 'misp-galaxy': + synonyms = r_serv_tags.smembers('synonym_tag_' + tag) + for synonym in synonyms: + str_synonyms = str_synonyms + synonym + ', ' + # add real tag + if str_synonyms != ' - synonyms: ': + list_tags.append({'name':tag + str_synonyms,'id':tag}) + else: + list_tags.append({'name':tag,'id':tag}) + + return jsonify(list_tags) + +@Tags.route("/Tags/get_all_tags_taxonomies") +def get_all_tags_taxonomies(): + + taxonomies = Taxonomies() + list_taxonomies = list(taxonomies.keys()) + + active_taxonomie = r_serv_tags.smembers('active_taxonomies') + + list_tags = [] + for taxonomie in active_taxonomie: + #l_tags = taxonomies.get(taxonomie).machinetags() + l_tags = r_serv_tags.smembers('active_tag_' + taxonomie) + for tag in l_tags: + list_tags.append( tag ) + + return jsonify(list_tags) + +@Tags.route("/Tags/get_all_tags_galaxies") +def get_all_tags_galaxy(): + + active_galaxies = r_serv_tags.smembers('active_galaxies') + + list_tags = [] + for galaxy in active_galaxies: + l_tags = r_serv_tags.smembers('active_tag_galaxies_' + galaxy) + for tag in l_tags: + list_tags.append(get_tags_with_synonyms(tag)) + + return jsonify(list_tags) + +@Tags.route("/Tags/get_tags_taxonomie") +def get_tags_taxonomie(): + + taxonomie = request.args.get('taxonomie') + + taxonomies = Taxonomies() + list_taxonomies = list(taxonomies.keys()) + + active_taxonomie = r_serv_tags.smembers('active_taxonomies') + + #verify input + if taxonomie in list_taxonomies: + if taxonomie in active_taxonomie: + + list_tags = [] + l_tags = r_serv_tags.smembers('active_tag_' + taxonomie) + for tag in l_tags: + list_tags.append( tag ) + + return jsonify(list_tags) + + else: + return 'this taxonomie is disable' + else: + return 'INCORRECT INPUT' + +@Tags.route("/Tags/get_tags_galaxy") +def get_tags_galaxy(): + + galaxy = request.args.get('galaxy') + + active_galaxies = r_serv_tags.smembers('active_galaxies') + + #verify input + if galaxy in active_galaxies: + + list_tags = [] + l_tags = r_serv_tags.smembers('active_tag_galaxies_' + galaxy) + for tag in l_tags: + list_tags.append(get_tags_with_synonyms(tag)) + + return jsonify(list_tags) + + else: + return 'this galaxy is disable' + + +@Tags.route("/Tags/get_tagged_paste") +def get_tagged_paste(): + + tags = request.args.get('ltags') + + list_tags = tags.split(',') + list_tag = [] + for tag in list_tags: + list_tag.append(tag.replace('"','\"')) + + # TODO verify input + + if(type(list_tags) is list): + # no tag + if list_tags is False: + print('empty') + # 1 tag + elif len(list_tags) < 2: + tagged_pastes = r_serv_tags.smembers(list_tags[0]) + + # 2 tags or more + else: + tagged_pastes = r_serv_tags.sinter(list_tags[0], *list_tags[1:]) + + else : + return 'INCORRECT INPUT' + + #TODO FIXME + currentSelectYear = int(datetime.now().year) + + all_content = [] + paste_date = [] + paste_linenum = [] + all_path = [] + allPastes = list(tagged_pastes) + paste_tags = [] + + for path in allPastes[0:50]: ######################moduleName + all_path.append(path) + paste = Paste.Paste(path) + content = paste.get_p_content() + content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 + all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " ")) + curr_date = str(paste._get_p_date()) + curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] + paste_date.append(curr_date) + paste_linenum.append(paste.get_lines_info()[0]) + p_tags = r_serv_metadata.smembers('tag:'+path) + complete_tags = [] + l_tags = [] + for tag in p_tags: + complete_tag = tag + + tag = tag.split('=') + if len(tag) > 1: + if tag[1] != '': + tag = tag[1][1:-1] + # no value + else: + tag = tag[0][1:-1] + # use for custom tags + else: + tag = tag[0] + + l_tags.append( (tag,complete_tag) ) + + paste_tags.append(l_tags) + + if len(allPastes) > 10: + finished = False + else: + finished = True + + return render_template("tagged.html", + year=currentSelectYear, + all_path=all_path, + tags=tags, + list_tag = list_tag, + paste_tags=paste_tags, + bootstrap_label=bootstrap_label, + content=all_content, + paste_date=paste_date, + paste_linenum=paste_linenum, + char_to_display=max_preview_modal, + finished=finished) + + +@Tags.route("/Tags/remove_tag") +def remove_tag(): + + #TODO verify input + path = request.args.get('paste') + tag = request.args.get('tag') + + #remove tag + r_serv_metadata.srem('tag:'+path, tag) + r_serv_tags.srem(tag, path) + + if r_serv_tags.scard(tag) == 0: + r_serv_tags.srem('list_tags', tag) + + return redirect(url_for('showsavedpastes.showsavedpaste', paste=path)) + +@Tags.route("/Tags/confirm_tag") +def confirm_tag(): + + #TODO verify input + path = request.args.get('paste') + tag = request.args.get('tag') + + if(tag[9:28] == 'automatic-detection'): + + #remove automatic tag + r_serv_metadata.srem('tag:'+path, tag) + r_serv_tags.srem(tag, path) + + tag = tag.replace('automatic-detection','analyst-detection', 1) + #add analyst tag + r_serv_metadata.sadd('tag:'+path, tag) + r_serv_tags.sadd(tag, path) + #add new tag in list of all used tags + r_serv_tags.sadd('list_tags', tag) + + return redirect(url_for('showsavedpastes.showsavedpaste', paste=path)) + + return 'incompatible tag' + +@Tags.route("/Tags/addTags") +def addTags(): + + tags = request.args.get('tags') + tagsgalaxies = request.args.get('tagsgalaxies') + path = request.args.get('path') + + list_tag = tags.split(',') + list_tag_galaxies = tagsgalaxies.split(',') + + taxonomies = Taxonomies() + active_taxonomies = r_serv_tags.smembers('active_taxonomies') + + active_galaxies = r_serv_tags.smembers('active_galaxies') + + if not path: + return 'INCORRECT INPUT0' + + if list_tag != ['']: + for tag in list_tag: + # verify input + tax = tag.split(':')[0] + if tax in active_taxonomies: + if tag in r_serv_tags.smembers('active_tag_' + tax): + + #add tag + r_serv_metadata.sadd('tag:'+path, tag) + r_serv_tags.sadd(tag, path) + #add new tag in list of all used tags + r_serv_tags.sadd('list_tags', tag) + + else: + return 'INCORRECT INPUT1' + else: + return 'INCORRECT INPUT2' + + if list_tag_galaxies != ['']: + for tag in list_tag_galaxies: + # verify input + gal = tag.split(':')[1] + gal = gal.split('=')[0] + + if gal in active_galaxies: + if tag in r_serv_tags.smembers('active_tag_galaxies_' + gal): + + #add tag + r_serv_metadata.sadd('tag:'+path, tag) + r_serv_tags.sadd(tag, path) + #add new tag in list of all used tags + r_serv_tags.sadd('list_tags', tag) + + else: + return 'INCORRECT INPUT3' + else: + return 'INCORRECT INPUT4' + + return redirect(url_for('showsavedpastes.showsavedpaste', paste=path)) + + +@Tags.route("/Tags/taxonomies") +def taxonomies(): + + active_taxonomies = r_serv_tags.smembers('active_taxonomies') + + taxonomies = Taxonomies() + list_taxonomies = list(taxonomies.keys()) + + id = [] + name = [] + description = [] + version = [] + enabled = [] + n_tags = [] + + for taxonomie in list_taxonomies: + id.append(taxonomie) + name.append(taxonomies.get(taxonomie).name) + description.append(taxonomies.get(taxonomie).description) + version.append(taxonomies.get(taxonomie).version) + if taxonomie in active_taxonomies: + enabled.append(True) + else: + enabled.append(False) + + n = str(r_serv_tags.scard('active_tag_' + taxonomie)) + n_tags.append(n + '/' + str(len(taxonomies.get(taxonomie).machinetags())) ) + + return render_template("taxonomies.html", + id=id, + all_name = name, + description = description, + version = version, + enabled = enabled, + n_tags=n_tags) + +@Tags.route("/Tags/edit_taxonomie") +def edit_taxonomie(): + + taxonomies = Taxonomies() + list_taxonomies = list(taxonomies.keys()) + + id = request.args.get('taxonomie') + + #verify input + if id in list(taxonomies.keys()): + active_tag = r_serv_tags.smembers('active_tag_' + id) + list_tag = taxonomies.get(id).machinetags() + list_tag_desc = taxonomies.get(id).machinetags_expanded() + + active_taxonomies = r_serv_tags.smembers('active_taxonomies') + if id in active_taxonomies: + active = True + else: + active = False + + n = str(r_serv_tags.scard('active_tag_' + id)) + badge = n + '/' + str(len(taxonomies.get(id).machinetags())) + + name = taxonomies.get(id).name + description = taxonomies.get(id).description + version = taxonomies.get(id).version + + status = [] + for tag in list_tag: + if tag in active_tag: + status.append(True) + else: + status.append(False) + + return render_template("edit_taxonomie.html", + id=id, + name=name, + badge = badge, + description = description, + version = version, + active=active, + all_tags = list_tag, + list_tag_desc=list_tag_desc, + status = status) + + else: + return 'INVALID TAXONOMIE' + +@Tags.route("/Tags/disable_taxonomie") +def disable_taxonomie(): + + taxonomies = Taxonomies() + list_taxonomies = list(taxonomies.keys()) + + id = request.args.get('taxonomie') + + if id in list_taxonomies: + r_serv_tags.srem('active_taxonomies', id) + for tag in taxonomies.get(id).machinetags(): + r_serv_tags.srem('active_tag_' + id, tag) + + return redirect(url_for('Tags.taxonomies')) + + else: + return "INCORRECT INPUT" + + + +@Tags.route("/Tags/active_taxonomie") +def active_taxonomie(): + + taxonomies = Taxonomies() + list_taxonomies = list(taxonomies.keys()) + + id = request.args.get('taxonomie') + + # verify input + if id in list_taxonomies: + r_serv_tags.sadd('active_taxonomies', id) + for tag in taxonomies.get(id).machinetags(): + r_serv_tags.sadd('active_tag_' + id, tag) + + return redirect(url_for('Tags.taxonomies')) + + else: + return "INCORRECT INPUT" + +@Tags.route("/Tags/edit_taxonomie_tag") +def edit_taxonomie_tag(): + + taxonomies = Taxonomies() + list_taxonomies = list(taxonomies.keys()) + + arg1 = request.args.getlist('tag_enabled') + arg2 = request.args.getlist('tag_disabled') + + id = request.args.get('taxonomie') + + #verify input + if id in list_taxonomies: + list_tag = taxonomies.get(id).machinetags() + + #check tags validity + if ( all(elem in list_tag for elem in arg1) or (len(arg1) == 0) ) and ( all(elem in list_tag for elem in arg2) or (len(arg2) == 0) ): + + active_tag = r_serv_tags.smembers('active_tag_' + id) + + diff = list(set(arg1) ^ set(list_tag)) + + #remove tags + for tag in diff: + r_serv_tags.srem('active_tag_' + id, tag) + + #all tags unchecked + if len(arg1) == 0 and len(arg2) == 0: + r_serv_tags.srem('active_taxonomies', id) + + #add new tags + for tag in arg2: + r_serv_tags.sadd('active_taxonomies', id) + r_serv_tags.sadd('active_tag_' + id, tag) + + return redirect(url_for('Tags.taxonomies')) + else: + return "INCORRECT INPUT" + + else: + return "INCORRECT INPUT" + +@Tags.route("/Tags/galaxies") +def galaxies(): + + active_galaxies = r_serv_tags.smembers('active_galaxies') + + name = [] + icon = [] + version = [] + all_type = [] + namespace = [] + description = [] + enabled = [] + n_tags = [] + + for galaxie_json in list_galaxies: + + galaxie = json.loads(galaxie_json) + + name.append(galaxie['name']) + icon.append(galaxie['icon']) + version.append(galaxie['version']) + type = galaxie['type'] + all_type.append(type) + namespace.append(galaxie['namespace']) + description.append(galaxie['description']) + + + if type in active_galaxies: + enabled.append(True) + else: + enabled.append(False) + + n = str(r_serv_tags.scard('active_tag_galaxies_' + type)) + n_tags.append(n + '/' + str(total_tags[type]) ) + + return render_template("galaxies.html", + name=name, + icon = icon, + version = version, + description = description, + namespace = namespace, + all_type = all_type, + enabled = enabled, + n_tags=n_tags) + + +@Tags.route("/Tags/edit_galaxy") +def edit_galaxy(): + + id = request.args.get('galaxy') + + for clusters_json in list_clusters: + + #get clusters + cluster = json.loads(clusters_json) + + if cluster['type'] == id: + + type = id + active_tag = r_serv_tags.smembers('active_tag_galaxies_' + type) + + n = str(r_serv_tags.scard('active_tag_galaxies_' + type)) + badge = n + '/' + str(total_tags[type]) + + name = cluster['name'] + description = cluster['description'] + version = cluster['version'] + source = cluster['source'] + + val = cluster['values'] + + tags = [] + for data in val: + try: + meta = data['meta'] + except KeyError: + meta = [] + tag_name = data['value'] + tag_name = 'misp-galaxy:{}="{}"'.format(type, tag_name) + try: + tag_description = data['description'] + except KeyError: + tag_description = '' + + tags.append( (tag_name, tag_description, meta) ) + + status = [] + for tag in tags: + if tag[0] in active_tag: + status.append(True) + else: + status.append(False) + + active_galaxies = r_serv_tags.smembers('active_galaxies') + if id in active_galaxies: + active = True + else: + active = False + + return render_template("edit_galaxy.html", + id = type, + name = name, + badge = badge, + description = description, + version = version, + active = active, + tags = tags, + status = status) + + + return 'INVALID GALAXY' + + +@Tags.route("/Tags/active_galaxy") +def active_galaxy(): + + id = request.args.get('galaxy') + + # verify input + try: + l_tags = list_all_tags[id] + except KeyError: + return "INCORRECT INPUT" + + r_serv_tags.sadd('active_galaxies', id) + for tag in l_tags: + r_serv_tags.sadd('active_tag_galaxies_' + id, 'misp-galaxy:{}="{}"'.format(id, tag)) + + #save synonyms + for clusters_json in list_clusters: + + #get clusters + cluster = json.loads(clusters_json) + + if cluster['type'] == id: + + val = cluster['values'] + + tags = [] + for data in val: + try: + meta = data['meta'] + synonyms = meta['synonyms'] + tag_name = data['value'] + tag_name = 'misp-galaxy:{}="{}"'.format(id, tag_name) + #save synonyms + for synonym in synonyms: + r_serv_tags.sadd('synonym_tag_' + tag_name, synonym) + + except KeyError: + pass + + break + + return redirect(url_for('Tags.galaxies')) + + +@Tags.route("/Tags/disable_galaxy") +def disable_galaxy(): + + id = request.args.get('galaxy') + + # verify input + try: + l_tags = list_all_tags[id] + except KeyError: + return "INCORRECT INPUT" + + r_serv_tags.srem('active_galaxies', id) + for tag in l_tags: + tag_name = 'misp-galaxy:{}="{}"'.format(id, tag) + r_serv_tags.srem('active_tag_galaxies_' + id, tag_name) + r_serv_tags.delete('synonym_tag_' + tag_name) + + return redirect(url_for('Tags.galaxies')) + + +@Tags.route("/Tags/edit_galaxy_tag") +def edit_galaxy_tag(): + + arg1 = request.args.getlist('tag_enabled') + arg2 = request.args.getlist('tag_disabled') + + id = request.args.get('galaxy') + + #verify input + try: + l_tags = list_all_tags[id] + except KeyError: + return "INCORRECT INPUT" + + #get full tags + list_tag = [] + for tag in l_tags: + list_tag.append('misp-galaxy:{}="{}"'.format(id, tag)) + + + #check tags validity + if ( all(elem in list_tag for elem in arg1) or (len(arg1) == 0) ) and ( all(elem in list_tag for elem in arg2) or (len(arg2) == 0) ): + + active_tag = r_serv_tags.smembers('active_tag_galaxies_' + id) + + diff = list(set(arg1) ^ set(list_tag)) + + #remove tags + for tag in diff: + r_serv_tags.srem('active_tag_galaxies_' + id, tag) + r_serv_tags.delete('synonym_tag_' + tag) + + #all tags unchecked + if len(arg1) == 0 and len(arg2) == 0: + r_serv_tags.srem('active_galaxies', id) + + #add new tags + for tag in arg2: + r_serv_tags.sadd('active_galaxies', id) + r_serv_tags.sadd('active_tag_galaxies_' + id, tag) + + #get tags synonyms + for clusters_json in list_clusters: + + #get clusters + cluster = json.loads(clusters_json) + + if cluster['type'] == id: + + val = cluster['values'] + + tags = [] + for data in val: + try: + meta = data['meta'] + synonyms = meta['synonyms'] + tag_name = data['value'] + tag_name = 'misp-galaxy:{}="{}"'.format(id, tag_name) + if tag_name in arg2: + #save synonyms + for synonym in synonyms: + r_serv_tags.sadd('synonym_tag_' + tag_name, synonym) + + except KeyError: + pass + break + + return redirect(url_for('Tags.galaxies')) + + else: + return "INCORRECT INPUT" + +@Tags.route("/Tags/tag_galaxy_info") +def tag_galaxy_info(): + + galaxy = request.args.get('galaxy') + tag = request.args.get('tag') + + full_tag = tag + title = tag.split(':')[1] + tag = tag.split('=')[1] + tag = tag[1:-1] + + #get clusters + for clusters_json in list_clusters: + cluster = json.loads(clusters_json) + + if cluster['type'] == galaxy: + val = cluster['values'] + source = cluster['source'] + + for data in val: + if tag == data['value']: + try: + description = data['description'] + except KeyError: + description = '' + if r_serv_tags.sismember('active_tag_galaxies_' + galaxy, full_tag): + active = True + else: + active = False + + synonyms = [] + metadata = [] + list_metadata = [] + try: + meta = data['meta'] + for key in meta: + if key != 'synonyms': + if type(meta[key]) is list: + for item in meta[key]: + list_metadata.append(key + ' : ' + item) + else: + list_metadata.append(key + ' : ' + meta[key]) + try: + synonyms = meta['synonyms'] + bool_synonyms = True + except KeyError: + synonyms = [] + bool_synonyms = False + except KeyError: + pass + + if synonyms: + bool_synonyms = True + else: + bool_synonyms = False + if list_metadata: + metadata = True + else: + metadata = False + + return render_template("tag_galaxy_info.html", + title = title, + description = description, + source = source, + active = active, + synonyms = synonyms, + bool_synonyms = bool_synonyms, + metadata = metadata, + list_metadata = list_metadata) + + return 'INVALID INPUT' + +# ========= REGISTRATION ========= +app.register_blueprint(Tags) diff --git a/var/www/modules/Tags/templates/Tags.html b/var/www/modules/Tags/templates/Tags.html new file mode 100644 index 00000000..f3cf1a44 --- /dev/null +++ b/var/www/modules/Tags/templates/Tags.html @@ -0,0 +1,103 @@ + + + + + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + +
+
+
+

Tags

+
+ +
+ +
+ + +
+ +
+ +
+ + + +
+

+ + +

+ Edit Taxonomies List +
+ + +

+ Edit Galaxies List +
+
+ +
+ + + + + + + + + diff --git a/var/www/modules/Tags/templates/edit_galaxy.html b/var/www/modules/Tags/templates/edit_galaxy.html new file mode 100644 index 00000000..c6e10f6c --- /dev/null +++ b/var/www/modules/Tags/templates/edit_galaxy.html @@ -0,0 +1,174 @@ + + + + + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + +
+
+ + +
+ + +
+
{{ name }} + {% if active %} + Enabled +     + {{ badge }} + {% endif %} + {% if not active %} + Disabled + {% endif %} +
+
+ {{ description }} +

+ Version: {{ version }} + {% if active %} + + Disable Galaxy + + {% endif %} + {% if not active %} + + Enable Galaxy + + {% endif %} +
+
+ +
+ + + + + + + + + + + + + + + {% for tag in tags %} + + + + + + + {% endfor %} + + +
TagDescriptionEnabled
+ {% if status[loop.index0] %} +
Enabled
+ + {% endif %} + {% if not status[loop.index0] %} +
Disabled
+ + {% endif %} +
+ {{ tag[0] }} + {{ tag[1] }} + {% if status[loop.index0] %} +
Enabled
+
+ {% endif %} + {% if not status[loop.index0] %} +
Disabled
+
+ {% endif %} +
+ + +
+ +
+ +
+ +
+ + + + + + + + + + diff --git a/var/www/modules/Tags/templates/edit_taxonomie.html b/var/www/modules/Tags/templates/edit_taxonomie.html new file mode 100644 index 00000000..74ea5b9c --- /dev/null +++ b/var/www/modules/Tags/templates/edit_taxonomie.html @@ -0,0 +1,172 @@ + + + + + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + +
+
+ + +
+ + +
+
{{ name }} + {% if active %} + Enabled +     + {{ badge }} + {% endif %} + {% if not active %} + Disabled + {% endif %} +
+
+ {{ description }} +

+ Version: {{ version }} + {% if active %} + + Disable Taxonomie + + {% endif %} + {% if not active %} + + Enable Taxonomie + + {% endif %} +
+
+ +
+ + + + + + + + + + + + + + + {% for tag in all_tags %} + + + + + + + {% endfor %} + + +
TagDescriptionEnabled
+ {% if status[loop.index0] %} +
Enabled
+ + {% endif %} + {% if not status[loop.index0] %} +
Disabled
+ + {% endif %} +
{{ tag }}{{ list_tag_desc[loop.index0] }} + {% if status[loop.index0] %} +
Enabled
+
+ {% endif %} + {% if not status[loop.index0] %} +
Disabled
+
+ {% endif %} +
+ + +
+ +
+ +
+ +
+ + + + + + + + + + diff --git a/var/www/modules/Tags/templates/galaxies.html b/var/www/modules/Tags/templates/galaxies.html new file mode 100644 index 00000000..5013c356 --- /dev/null +++ b/var/www/modules/Tags/templates/galaxies.html @@ -0,0 +1,120 @@ + + + + + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + +
+
+
+

Galaxies

+
+ +
+ + + + + + + + + + + + + + + + {% for type in all_type %} + + + + + + + + + {% endfor %} + + +
NameDescriptionNamespaceVersionEnabledActive Tags
+ {{ name[loop.index0] }} +
+ +
+
{{ description[loop.index0] }}{{ namespace[loop.index0] }}{{ version[loop.index0] }} + {% if enabled[loop.index0] %} +
Enabled
+
+ {% endif %} + {% if not enabled[loop.index0] %} +
Disabled
+
+ {% endif %} +
+
{{ n_tags[loop.index0] }}
+ Active Tags {{ n_tags[loop.index0] }} +
+ +
+ + + + + + + + diff --git a/var/www/modules/Tags/templates/header_Tags.html b/var/www/modules/Tags/templates/header_Tags.html new file mode 100644 index 00000000..624adb2f --- /dev/null +++ b/var/www/modules/Tags/templates/header_Tags.html @@ -0,0 +1 @@ +
  • Tags
  • diff --git a/var/www/modules/Tags/templates/tag_galaxy_info.html b/var/www/modules/Tags/templates/tag_galaxy_info.html new file mode 100644 index 00000000..a1544fcd --- /dev/null +++ b/var/www/modules/Tags/templates/tag_galaxy_info.html @@ -0,0 +1,113 @@ + + + + + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + +
    +
    + + +
    + + +
    +
    {{ title }} + {% if active %} + Enabled + {% endif %} + {% if not active %} + Disabled + {% endif %} +
    +
    + {{ description }} +

    + Source: {{ source }} + {% if metadata %} +
    +
    Metadata : +
      + {% for meta in list_metadata %} +
    • {{ meta }}
    • + {% endfor %} +
    +
    +
    + {% endif %} + {% if bool_synonyms %} +

    +
      +
    • synonyms :
    • + {% for synonym in synonyms %} +
    • {{ synonym }}
    • + {% endfor %} +
    + {% endif %} +
    +
    + + +
    + + + + + + + + diff --git a/var/www/modules/Tags/templates/tagged.html b/var/www/modules/Tags/templates/tagged.html new file mode 100644 index 00000000..284323fd --- /dev/null +++ b/var/www/modules/Tags/templates/tagged.html @@ -0,0 +1,307 @@ + + + + + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + + + + +
    +
    +
    +

    Tags

    +
    + +
    + +
    + + +
    + +
    + +
    + + + + + + + + + + + + + + + + {% for path in all_path %} + + + + + + + + {% endfor %} + + +
    #PathDate# of linesAction
    {{ loop.index0 }}{{ path }} +
    + {% for tag in paste_tags[loop.index0] %} + + {{ tag[0] }} + + {% endfor %} +
    +
    {{ paste_date[loop.index0] }}{{ paste_linenum[loop.index0] }}

    + +
    +
    + +
    +
    + + +
    +
    + +
    +

    + + +

    + Edit Taxonomies List +
    + + +

    + Edit Galaxies List +
    +
    + +
    + + + + + + + + + diff --git a/var/www/modules/Tags/templates/taxonomies.html b/var/www/modules/Tags/templates/taxonomies.html new file mode 100644 index 00000000..74feda6f --- /dev/null +++ b/var/www/modules/Tags/templates/taxonomies.html @@ -0,0 +1,111 @@ + + + + + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + +
    +
    +
    +

    Taxonomies

    +
    + +
    + + + + + + + + + + + + + + + {% for name in all_name %} + + + + + + + + {% endfor %} + + +
    NameDescriptionVersionEnabledActive Tags
    {{ name }}{{ description[loop.index0] }}{{ version[loop.index0] }} + {% if enabled[loop.index0] %} +
    Enabled
    +
    + {% endif %} + {% if not enabled[loop.index0] %} +
    Disabled
    +
    + {% endif %} +
    +
    {{ n_tags[loop.index0] }}
    + Active Tags {{ n_tags[loop.index0] }} +
    + +
    + + + + + + + + diff --git a/var/www/modules/browsepastes/Flask_browsepastes.py b/var/www/modules/browsepastes/Flask_browsepastes.py index 67923fbd..2a73d9b8 100644 --- a/var/www/modules/browsepastes/Flask_browsepastes.py +++ b/var/www/modules/browsepastes/Flask_browsepastes.py @@ -20,6 +20,8 @@ app = Flask_config.app cfg = Flask_config.cfg max_preview_char = Flask_config.max_preview_char max_preview_modal = Flask_config.max_preview_modal +r_serv_metadata = Flask_config.r_serv_metadata +bootstrap_label = Flask_config.bootstrap_label #init all lvlDB servers curYear = datetime.now().year @@ -56,6 +58,7 @@ def getPastebyType(server, module_name): def event_stream_getImportantPasteByModule(module_name, year): index = 0 all_pastes_list = getPastebyType(r_serv_db[year], module_name) + paste_tags = [] for path in all_pastes_list: index += 1 @@ -64,6 +67,23 @@ def event_stream_getImportantPasteByModule(module_name, year): content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 curr_date = str(paste._get_p_date()) curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] + p_tags = r_serv_metadata.smembers('tag:'+path) + l_tags = [] + for tag in p_tags: + complete_tag = tag.replace('"', '"') + tag = tag.split('=') + if len(tag) > 1: + if tag[1] != '': + tag = tag[1][1:-1] + # no value + else: + tag = tag[0][1:-1] + # use for custom tags + else: + tag = tag[0] + + l_tags.append( (tag, complete_tag) ) + data = {} data["module"] = module_name data["index"] = index @@ -71,6 +91,8 @@ def event_stream_getImportantPasteByModule(module_name, year): data["content"] = content[0:content_range] data["linenum"] = paste.get_lines_info()[0] data["date"] = curr_date + data["l_tags"] = l_tags + data["bootstrap_label"] = bootstrap_label data["char_to_display"] = max_preview_modal data["finished"] = True if index == len(all_pastes_list) else False yield 'retry: 100000\ndata: %s\n\n' % json.dumps(data) #retry to avoid reconnection of the browser @@ -98,6 +120,7 @@ def importantPasteByModule(): paste_date = [] paste_linenum = [] all_path = [] + paste_tags = [] allPastes = getPastebyType(r_serv_db[currentSelectYear], module_name) for path in allPastes[0:10]: @@ -110,6 +133,24 @@ def importantPasteByModule(): curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] paste_date.append(curr_date) paste_linenum.append(paste.get_lines_info()[0]) + p_tags = r_serv_metadata.smembers('tag:'+path) + l_tags = [] + for tag in p_tags: + complete_tag = tag + tag = tag.split('=') + if len(tag) > 1: + if tag[1] != '': + tag = tag[1][1:-1] + # no value + else: + tag = tag[0][1:-1] + # use for custom tags + else: + tag = tag[0] + + l_tags.append( (tag, complete_tag) ) + + paste_tags.append(l_tags) if len(allPastes) > 10: finished = False @@ -124,6 +165,8 @@ def importantPasteByModule(): paste_date=paste_date, paste_linenum=paste_linenum, char_to_display=max_preview_modal, + paste_tags=paste_tags, + bootstrap_label=bootstrap_label, finished=finished) @browsepastes.route("/_getImportantPasteByModule", methods=['GET']) diff --git a/var/www/modules/browsepastes/templates/important_paste_by_module.html b/var/www/modules/browsepastes/templates/important_paste_by_module.html index a76fb870..2c86531c 100644 --- a/var/www/modules/browsepastes/templates/important_paste_by_module.html +++ b/var/www/modules/browsepastes/templates/important_paste_by_module.html @@ -12,7 +12,15 @@ {% for path in all_path %} {{ loop.index0 }} - {{ path }} + {{ path }} +
    + {% for tag in paste_tags[loop.index0] %} + + {{ tag[0] }} + + {% endfor %} +
    + {{ paste_date[loop.index0] }} {{ paste_linenum[loop.index0] }}

    @@ -89,14 +97,21 @@ function add_entries_X(to_add) { } else { var feed = json_array.shift(); elem_added++; - search_table.row.add( [ - feed.index, - " "+ feed.path +"", - feed.date, - feed.linenum, - "

    " - ] ).draw( false ); - $("#myTable_"+moduleName).attr('data-numElem', curr_numElem+1); + var tag = "" + for(j=0; j" + + "" + feed.l_tags[j][0] + "" + ""; + } + search_table.row.add( [ + feed.index, + " "+ feed.path +"" + + "
    " + tag + "
    " , + feed.date, + feed.linenum, + "

    " + ] ).draw( false ); + $("#myTable_"+moduleName).attr('data-numElem', curr_numElem+1); } } $("#load_more_json_button1").removeAttr('disabled'); @@ -122,7 +137,6 @@ $("#myTable_"+moduleName).attr('data-numElem', "{{ all_path|length }}"); $(document).ready(function(){ $('[data-toggle="tooltip"]').tooltip(); $("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) { - //get_html_and_update_modal(event); get_html_and_update_modal(event, $(this)); }); diff --git a/var/www/modules/search/Flask_search.py b/var/www/modules/search/Flask_search.py index afce2452..818e3e61 100644 --- a/var/www/modules/search/Flask_search.py +++ b/var/www/modules/search/Flask_search.py @@ -22,8 +22,10 @@ import Flask_config app = Flask_config.app cfg = Flask_config.cfg r_serv_pasteName = Flask_config.r_serv_pasteName +r_serv_metadata = Flask_config.r_serv_metadata max_preview_char = Flask_config.max_preview_char max_preview_modal = Flask_config.max_preview_modal +bootstrap_label = Flask_config.bootstrap_label baseindexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path")) @@ -95,6 +97,7 @@ def search(): c = [] #preview of the paste content paste_date = [] paste_size = [] + paste_tags = [] index_name = request.form['index_name'] num_elem_to_get = 50 @@ -119,13 +122,15 @@ def search(): # Search full line schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) + print(selected_index) ix = index.open_dir(selected_index) with ix.searcher() as searcher: query = QueryParser("content", ix.schema).parse(" ".join(q)) results = searcher.search_page(query, 1, pagelen=num_elem_to_get) for x in results: r.append(x.items()[0][1]) - paste = Paste.Paste(x.items()[0][1]) + path = x.items()[0][1] + paste = Paste.Paste(path) content = paste.get_p_content() content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 c.append(content[0:content_range]) @@ -133,6 +138,24 @@ def search(): curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] paste_date.append(curr_date) paste_size.append(paste._get_p_size()) + p_tags = r_serv_metadata.smembers('tag:'+path) + l_tags = [] + for tag in p_tags: + complete_tag = tag + tag = tag.split('=') + if len(tag) > 1: + if tag[1] != '': + tag = tag[1][1:-1] + # no value + else: + tag = tag[0][1:-1] + # use for custom tags + else: + tag = tag[0] + + l_tags.append( (tag, complete_tag) ) + + paste_tags.append(l_tags) results = searcher.search(query) num_res = len(results) @@ -142,6 +165,8 @@ def search(): query=request.form['query'], paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal, num_res=num_res, index_min=index_min, index_max=index_max, + bootstrap_label=bootstrap_label, + paste_tags=paste_tags, index_list=get_index_list(selected_index) ) @@ -165,6 +190,7 @@ def get_more_search_result(): preview_array = [] date_array = [] size_array = [] + list_tags = [] schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) @@ -173,8 +199,9 @@ def get_more_search_result(): query = QueryParser("content", ix.schema).parse(" ".join(q)) results = searcher.search_page(query, page_offset, num_elem_to_get) for x in results: - path_array.append(x.items()[0][1]) - paste = Paste.Paste(x.items()[0][1]) + path = x.items()[0][1] + path_array.append(path) + paste = Paste.Paste(path) content = paste.get_p_content() content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 preview_array.append(content[0:content_range]) @@ -182,11 +209,30 @@ def get_more_search_result(): curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] date_array.append(curr_date) size_array.append(paste._get_p_size()) + p_tags = r_serv_metadata.smembers('tag:'+path) + l_tags = [] + for tag in p_tags: + tag = tag.split('=') + if len(tag) > 1: + if tag[1] != '': + tag = tag[1][1:-1] + # no value + else: + tag = tag[0][1:-1] + # use for custom tags + else: + tag = tag[0] + + l_tags.append(tag) + list_tags.append(l_tags) + to_return = {} to_return["path_array"] = path_array to_return["preview_array"] = preview_array to_return["date_array"] = date_array to_return["size_array"] = size_array + to_return["list_tags"] = list_tags + to_return["bootstrap_label"] = bootstrap_label if len(path_array) < num_elem_to_get: #pagelength to_return["moreData"] = False else: diff --git a/var/www/modules/search/templates/search.html b/var/www/modules/search/templates/search.html index 43895a9f..bed826dd 100644 --- a/var/www/modules/search/templates/search.html +++ b/var/www/modules/search/templates/search.html @@ -75,7 +75,7 @@
    - Index: + Index: +
    + +
    + + +
    +

    + +
    + +
    + +
    + + +
    +
    + + +
    + + + + + {% for tag in list_tags %} + {{ tag[0] }} + + + + + {% endfor %} + + + @@ -49,6 +169,7 @@
    +
    {% if duplicate_list|length == 0 %} @@ -86,7 +207,86 @@
    + + + + + + + + + + + diff --git a/var/www/static/css/tags.css b/var/www/static/css/tags.css new file mode 100644 index 00000000..baa0f673 --- /dev/null +++ b/var/www/static/css/tags.css @@ -0,0 +1,226 @@ +.tag-ctn{ + position: relative; + height: 30px; + padding: 0; + margin-bottom: 0px; + font-size: 14px; + line-height: 20px; + color: #555555; + -webkit-border-radius: 3px; + -moz-border-radius: 3px; + border-radius: 3px; + background-color: #ffffff; + border: 1px solid #cccccc; + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + -webkit-transition: border linear 0.2s, box-shadow linear 0.2s; + -moz-transition: border linear 0.2s, box-shadow linear 0.2s; + -o-transition: border linear 0.2s, box-shadow linear 0.2s; + transition: border linear 0.2s, box-shadow linear 0.2s; + cursor: default; + display: block; +} +.tag-ctn-invalid{ + border: 1px solid #CC0000; +} +.tag-ctn-readonly{ + cursor: pointer; +} +.tag-ctn-disabled{ + cursor: not-allowed; + background-color: #eeeeee; +} +.tag-ctn-bootstrap-focus, +.tag-ctn-bootstrap-focus .tag-res-ctn{ + border-color: rgba(82, 168, 236, 0.8) !important; + /* IE6-9 */ + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6) !important; + -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6) !important; + box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6) !important; + border-bottom-left-radius: 0; + border-bottom-right-radius: 0; +} +.tag-ctn input{ + border: 0; + box-shadow: none; + -webkit-transition: none; + outline: none; + display: block; + padding: 4px 6px; + line-height: normal; + overflow: hidden; + height: auto; + border-radius: 0; + float: left; + margin: 2px 0 2px 2px; +} +.tag-ctn-disabled input{ + cursor: not-allowed; + background-color: #eeeeee; +} +.tag-ctn .tag-input-readonly{ + cursor: pointer; +} +.tag-ctn .tag-empty-text{ + color: #DDD; +} +.tag-ctn input:focus{ + border: 0; + box-shadow: none; + -webkit-transition: none; + background: #FFF; +} +.tag-ctn .tag-trigger{ + float: right; + width: 27px; + height:100%; + position:absolute; + right:0; + border-left: 1px solid #CCC; + background: #EEE; + cursor: pointer; +} +.tag-ctn .tag-trigger .tag-trigger-ico { + display: inline-block; + width: 0; + height: 0; + vertical-align: top; + border-top: 4px solid gray; + border-right: 4px solid transparent; + border-left: 4px solid transparent; + content: ""; + margin-left: 9px; + margin-top: 13px; +} +.tag-ctn .tag-trigger:hover{ + background: -moz-linear-gradient(100% 100% 90deg, #e3e3e3, #f1f1f1); + background: -webkit-gradient(linear, 0% 0%, 0% 100%, from(#f1f1f1), to(#e3e3e3)); +} +.tag-ctn .tag-trigger:hover .tag-trigger-ico{ + background-position: 0 -4px; +} +.tag-ctn-disabled .tag-trigger{ + cursor: not-allowed; + background-color: #eeeeee; +} +.tag-ctn-bootstrap-focus{ + border-bottom: 1px solid #CCC; +} +.tag-res-ctn{ + position: relative; + background: #FFF; + overflow-y: auto; + z-index: 9999; + -webkit-border-radius: 3px; + -moz-border-radius: 3px; + border-radius: 3px; + border: 1px solid #CCC; + left: -1px; + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + -webkit-transition: border linear 0.2s, box-shadow linear 0.2s; + -moz-transition: border linear 0.2s, box-shadow linear 0.2s; + -o-transition: border linear 0.2s, box-shadow linear 0.2s; + transition: border linear 0.2s, box-shadow linear 0.2s; + border-top: 0; + border-top-left-radius: 0; + border-top-right-radius: 0; +} +.tag-res-ctn .tag-res-group{ + line-height: 23px; + text-align: left; + padding: 2px 5px; + font-weight: bold; + border-bottom: 1px dotted #CCC; + border-top: 1px solid #CCC; + background: #f3edff; + color: #333; +} +.tag-res-ctn .tag-res-item{ + line-height: 25px; + text-align: left; + padding: 2px 5px; + color: #666; + cursor: pointer; +} +.tag-res-ctn .tag-res-item-grouped{ + padding-left: 15px; +} +.tag-res-ctn .tag-res-odd{ + background: #F3F3F3; +} +.tag-res-ctn .tag-res-item-active{ + background-color: #3875D7; + filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#3875D7', endColorstr='#2A62BC', GradientType=0 ); + background-image: -webkit-gradient(linear, 0 0, 0 100%, color-stop(20%, #3875D7), color-stop(90%, #2A62BC)); + background-image: -webkit-linear-gradient(top, #3875D7 20%, #2A62BC 90%); + background-image: -moz-linear-gradient(top, #3875D7 20%, #2A62BC 90%); + background-image: -o-linear-gradient(top, #3875D7 20%, #2A62BC 90%); + background-image: linear-gradient(#3875D7 20%, #2A62BC 90%); + color: #fff; +} +.tag-sel-ctn{ + overflow: auto; + line-height: 22px; + padding-right:27px; +} +.tag-sel-ctn .tag-sel-item{ + background: #555; + color: #EEE; + float: left; + font-size: 12px; + padding: 0 5px; + border-radius: 3px; + margin-left: 5px; + margin-top: 4px; +} +.tag-sel-ctn .tag-sel-text{ + background: #FFF; + color: #666; + padding-right: 0; + margin-left: 0; + font-size: 14px; + font-weight: normal; +} +.tag-res-ctn .tag-res-item em{ + font-style: normal; + background: #565656; + color: #FFF; +} +.tag-sel-ctn .tag-sel-item:hover{ + background: #565656; +} +.tag-sel-ctn .tag-sel-text:hover{ + background: #FFF; +} +.tag-sel-ctn .tag-sel-item-active{ + border: 1px solid red; + background: #757575; +} +.tag-ctn .tag-sel-ctn .tag-sel-item{ + margin-top: 3px; +} +.tag-stacked .tag-sel-item{ + float: inherit; +} +.tag-sel-ctn .tag-sel-item .tag-close-btn{ + width: 7px; + cursor: pointer; + height: 7px; + float: right; + margin: 8px 2px 0 10px; + background-image: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAcAAAAOCAYAAADjXQYbAAAACXBIWXMAAAsTAAALEwEAmpwYAAAKT2lDQ1BQaG90b3Nob3AgSUNDIHByb2ZpbGUAAHjanVNnVFPpFj333vRCS4iAlEtvUhUIIFJCi4AUkSYqIQkQSoghodkVUcERRUUEG8igiAOOjoCMFVEsDIoK2AfkIaKOg6OIisr74Xuja9a89+bN/rXXPues852zzwfACAyWSDNRNYAMqUIeEeCDx8TG4eQuQIEKJHAAEAizZCFz/SMBAPh+PDwrIsAHvgABeNMLCADATZvAMByH/w/qQplcAYCEAcB0kThLCIAUAEB6jkKmAEBGAYCdmCZTAKAEAGDLY2LjAFAtAGAnf+bTAICd+Jl7AQBblCEVAaCRACATZYhEAGg7AKzPVopFAFgwABRmS8Q5ANgtADBJV2ZIALC3AMDOEAuyAAgMADBRiIUpAAR7AGDIIyN4AISZABRG8lc88SuuEOcqAAB4mbI8uSQ5RYFbCC1xB1dXLh4ozkkXKxQ2YQJhmkAuwnmZGTKBNA/g88wAAKCRFRHgg/P9eM4Ors7ONo62Dl8t6r8G/yJiYuP+5c+rcEAAAOF0ftH+LC+zGoA7BoBt/qIl7gRoXgugdfeLZrIPQLUAoOnaV/Nw+H48PEWhkLnZ2eXk5NhKxEJbYcpXff5nwl/AV/1s+X48/Pf14L7iJIEyXYFHBPjgwsz0TKUcz5IJhGLc5o9H/LcL//wd0yLESWK5WCoU41EScY5EmozzMqUiiUKSKcUl0v9k4t8s+wM+3zUAsGo+AXuRLahdYwP2SycQWHTA4vcAAPK7b8HUKAgDgGiD4c93/+8//UegJQCAZkmScQAAXkQkLlTKsz/HCAAARKCBKrBBG/TBGCzABhzBBdzBC/xgNoRCJMTCQhBCCmSAHHJgKayCQiiGzbAdKmAv1EAdNMBRaIaTcA4uwlW4Dj1wD/phCJ7BKLyBCQRByAgTYSHaiAFiilgjjggXmYX4IcFIBBKLJCDJiBRRIkuRNUgxUopUIFVIHfI9cgI5h1xGupE7yAAygvyGvEcxlIGyUT3UDLVDuag3GoRGogvQZHQxmo8WoJvQcrQaPYw2oefQq2gP2o8+Q8cwwOgYBzPEbDAuxsNCsTgsCZNjy7EirAyrxhqwVqwDu4n1Y8+xdwQSgUXACTYEd0IgYR5BSFhMWE7YSKggHCQ0EdoJNwkDhFHCJyKTqEu0JroR+cQYYjIxh1hILCPWEo8TLxB7iEPENyQSiUMyJ7mQAkmxpFTSEtJG0m5SI+ksqZs0SBojk8naZGuyBzmULCAryIXkneTD5DPkG+Qh8lsKnWJAcaT4U+IoUspqShnlEOU05QZlmDJBVaOaUt2ooVQRNY9aQq2htlKvUYeoEzR1mjnNgxZJS6WtopXTGmgXaPdpr+h0uhHdlR5Ol9BX0svpR+iX6AP0dwwNhhWDx4hnKBmbGAcYZxl3GK+YTKYZ04sZx1QwNzHrmOeZD5lvVVgqtip8FZHKCpVKlSaVGyovVKmqpqreqgtV81XLVI+pXlN9rkZVM1PjqQnUlqtVqp1Q61MbU2epO6iHqmeob1Q/pH5Z/YkGWcNMw09DpFGgsV/jvMYgC2MZs3gsIWsNq4Z1gTXEJrHN2Xx2KruY/R27iz2qqaE5QzNKM1ezUvOUZj8H45hx+Jx0TgnnKKeX836K3hTvKeIpG6Y0TLkxZVxrqpaXllirSKtRq0frvTau7aedpr1Fu1n7gQ5Bx0onXCdHZ4/OBZ3nU9lT3acKpxZNPTr1ri6qa6UbobtEd79up+6Ynr5egJ5Mb6feeb3n+hx9L/1U/W36p/VHDFgGswwkBtsMzhg8xTVxbzwdL8fb8VFDXcNAQ6VhlWGX4YSRudE8o9VGjUYPjGnGXOMk423GbcajJgYmISZLTepN7ppSTbmmKaY7TDtMx83MzaLN1pk1mz0x1zLnm+eb15vft2BaeFostqi2uGVJsuRaplnutrxuhVo5WaVYVVpds0atna0l1rutu6cRp7lOk06rntZnw7Dxtsm2qbcZsOXYBtuutm22fWFnYhdnt8Wuw+6TvZN9un2N/T0HDYfZDqsdWh1+c7RyFDpWOt6azpzuP33F9JbpL2dYzxDP2DPjthPLKcRpnVOb00dnF2e5c4PziIuJS4LLLpc+Lpsbxt3IveRKdPVxXeF60vWdm7Obwu2o26/uNu5p7ofcn8w0nymeWTNz0MPIQ+BR5dE/C5+VMGvfrH5PQ0+BZ7XnIy9jL5FXrdewt6V3qvdh7xc+9j5yn+M+4zw33jLeWV/MN8C3yLfLT8Nvnl+F30N/I/9k/3r/0QCngCUBZwOJgUGBWwL7+Hp8Ib+OPzrbZfay2e1BjKC5QRVBj4KtguXBrSFoyOyQrSH355jOkc5pDoVQfujW0Adh5mGLw34MJ4WHhVeGP45wiFga0TGXNXfR3ENz30T6RJZE3ptnMU85ry1KNSo+qi5qPNo3ujS6P8YuZlnM1VidWElsSxw5LiquNm5svt/87fOH4p3iC+N7F5gvyF1weaHOwvSFpxapLhIsOpZATIhOOJTwQRAqqBaMJfITdyWOCnnCHcJnIi/RNtGI2ENcKh5O8kgqTXqS7JG8NXkkxTOlLOW5hCepkLxMDUzdmzqeFpp2IG0yPTq9MYOSkZBxQqohTZO2Z+pn5mZ2y6xlhbL+xW6Lty8elQfJa7OQrAVZLQq2QqboVFoo1yoHsmdlV2a/zYnKOZarnivN7cyzytuQN5zvn//tEsIS4ZK2pYZLVy0dWOa9rGo5sjxxedsK4xUFK4ZWBqw8uIq2Km3VT6vtV5eufr0mek1rgV7ByoLBtQFr6wtVCuWFfevc1+1dT1gvWd+1YfqGnRs+FYmKrhTbF5cVf9go3HjlG4dvyr+Z3JS0qavEuWTPZtJm6ebeLZ5bDpaql+aXDm4N2dq0Dd9WtO319kXbL5fNKNu7g7ZDuaO/PLi8ZafJzs07P1SkVPRU+lQ27tLdtWHX+G7R7ht7vPY07NXbW7z3/T7JvttVAVVN1WbVZftJ+7P3P66Jqun4lvttXa1ObXHtxwPSA/0HIw6217nU1R3SPVRSj9Yr60cOxx++/p3vdy0NNg1VjZzG4iNwRHnk6fcJ3/ceDTradox7rOEH0x92HWcdL2pCmvKaRptTmvtbYlu6T8w+0dbq3nr8R9sfD5w0PFl5SvNUyWna6YLTk2fyz4ydlZ19fi753GDborZ752PO32oPb++6EHTh0kX/i+c7vDvOXPK4dPKy2+UTV7hXmq86X23qdOo8/pPTT8e7nLuarrlca7nuer21e2b36RueN87d9L158Rb/1tWeOT3dvfN6b/fF9/XfFt1+cif9zsu72Xcn7q28T7xf9EDtQdlD3YfVP1v+3Njv3H9qwHeg89HcR/cGhYPP/pH1jw9DBY+Zj8uGDYbrnjg+OTniP3L96fynQ89kzyaeF/6i/suuFxYvfvjV69fO0ZjRoZfyl5O/bXyl/erA6xmv28bCxh6+yXgzMV70VvvtwXfcdx3vo98PT+R8IH8o/2j5sfVT0Kf7kxmTk/8EA5jz/GMzLdsAAAAEZ0FNQQAAsY58+1GTAAAAIGNIUk0AAHolAACAgwAA+f8AAIDpAAB1MAAA6mAAADqYAAAXb5JfxUYAAABSSURBVHjahI7BCQAwCAOTzpThHMHh3Kl9CVos9XckFwQAuPtGuWTWwMwaczKzyHsqg6+5JqMJr28BABHRwmTWQFJjTmYWOU1L4tdck9GE17dnALGAS+kAR/u2AAAAAElFTkSuQmCC); + +} +.tag-sel-ctn .tag-sel-item .tag-close-btn:hover{ + background-position: 0 -7px; +} +.tag-helper{ + color: #AAA; + font-size: 10px; + position: absolute; + top: -17px; + right: 0; +} diff --git a/var/www/static/js/tags.js b/var/www/static/js/tags.js new file mode 100644 index 00000000..47b84061 --- /dev/null +++ b/var/www/static/js/tags.js @@ -0,0 +1,1516 @@ +/** + * All auto suggestion boxes are fucked up or badly written. + * This is an attempt to create something that doesn't suck... + * + * Requires: jQuery + * + * Author: Nicolas Bize + * Date: Feb. 8th 2013 + * Version: 1.3.1 + * Licence: TagSuggest is licenced under MIT licence (https://www.opensource.org/licenses/mit-license.php) + */ +(function($) +{ + "use strict"; + var TagSuggest = function(element, options) + { + var ms = this; + + /** + * Initializes the TagSuggest component + * @param defaults - see config below + */ + var defaults = { + /********** CONFIGURATION PROPERTIES ************/ + /** + * @cfg {Boolean} allowFreeEntries + *

    Restricts or allows the user to validate typed entries.

    + * Defaults to true. + */ + allowFreeEntries: false, + + /** + * @cfg {String} cls + *

    A custom CSS class to apply to the field's underlying element.

    + * Defaults to ''. + */ + cls: '', + + /** + * @cfg {Array / String / Function} data + * JSON Data source used to populate the combo box. 3 options are available here:
    + *

    No Data Source (default)
    + * When left null, the combo box will not suggest anything. It can still enable the user to enter + * multiple entries if allowFreeEntries is * set to true (default).

    + *

    Static Source
    + * You can pass an array of JSON objects, an array of strings or even a single CSV string as the + * data source.
    For ex. data: [* {id:0,name:"Paris"}, {id: 1, name: "New York"}]
    + * You can also pass any json object with the results property containing the json array.

    + *

    Url
    + * You can pass the url from which the component will fetch its JSON data.
    Data will be fetched + * using a POST ajax request that will * include the entered text as 'query' parameter. The results + * fetched from the server can be:
    + * - an array of JSON objects (ex: [{id:...,name:...},{...}])
    + * - a string containing an array of JSON objects ready to be parsed (ex: "[{id:...,name:...},{...}]")
    + * - a JSON object whose data will be contained in the results property + * (ex: {results: [{id:...,name:...},{...}]

    + *

    Function
    + * You can pass a function which returns an array of JSON objects (ex: [{id:...,name:...},{...}])
    + * The function can return the JSON data or it can use the first argument as function to handle the data.
    + * Only one (callback function or return value) is needed for the function to succeed.
    + * See the following example:
    + * function (response) { var myjson = [{name: 'test', id: 1}]; response(myjson); return myjson; }

    + * Defaults to null + */ + data: null, + + /** + * @cfg {Object} dataParams + *

    Additional parameters to the ajax call

    + * Defaults to {} + */ + dataUrlParams: {}, + + /** + * @cfg {Boolean} disabled + *

    Start the component in a disabled state.

    + * Defaults to false. + */ + disabled: false, + + /** + * @cfg {String} displayField + *

    name of JSON object property displayed in the combo list

    + * Defaults to name. + */ + displayField: 'name', + + /** + * @cfg {Boolean} editable + *

    Set to false if you only want mouse interaction. In that case the combo will + * automatically expand on focus.

    + * Defaults to true. + */ + editable: true, + + /** + * @cfg {String} emptyText + *

    The default placeholder text when nothing has been entered

    + * Defaults to 'Type or click here' or just 'Click here' if not editable. + */ + emptyText: function() { + return cfg.editable ? 'Select Tags' : ''; + }, + + /** + * @cfg {String} emptyTextCls + *

    A custom CSS class to style the empty text

    + * Defaults to 'tag-empty-text'. + */ + emptyTextCls: 'tag-empty-text', + + /** + * @cfg {Boolean} expanded + *

    Set starting state for combo.

    + * Defaults to false. + */ + expanded: false, + + /** + * @cfg {Boolean} expandOnFocus + *

    Automatically expands combo on focus.

    + * Defaults to false. + */ + expandOnFocus: function() { + return cfg.editable ? true : true; + }, + + /** + * @cfg {String} groupBy + *

    JSON property by which the list should be grouped

    + * Defaults to null + */ + groupBy: null, + + /** + * @cfg {Boolean} hideTrigger + *

    Set to true to hide the trigger on the right

    + * Defaults to false. + */ + hideTrigger: false, + + /** + * @cfg {Boolean} highlight + *

    Set to true to highlight search input within displayed suggestions

    + * Defaults to true. + */ + highlight: true, + + /** + * @cfg {String} id + *

    A custom ID for this component

    + * Defaults to 'tag-ctn-{n}' with n positive integer + */ + id: function() { + return 'tag-ctn-' + $('div[id^="tag-ctn"]').length; + }, + + /** + * @cfg {String} infoMsgCls + *

    A class that is added to the info message appearing on the top-right part of the component

    + * Defaults to '' + */ + infoMsgCls: '', + + /** + * @cfg {Object} inputCfg + *

    Additional parameters passed out to the INPUT tag. Enables usage of AngularJS's custom tags for ex.

    + * Defaults to {} + */ + inputCfg: {}, + + /** + * @cfg {String} invalidCls + *

    The class that is applied to show that the field is invalid

    + * Defaults to tag-ctn-invalid + */ + invalidCls: 'tag-ctn-invalid', + + /** + * @cfg {Boolean} matchCase + *

    Set to true to filter data results according to case. Useless if the data is fetched remotely

    + * Defaults to false. + */ + matchCase: false, + + /** + * @cfg {Integer} maxDropHeight (in px) + *

    Once expanded, the combo's height will take as much room as the # of available results. + * In case there are too many results displayed, this will fix the drop down height.

    + * Defaults to 290 px. + */ + maxDropHeight: 290, + + /** + * @cfg {Integer} maxEntryLength + *

    Defines how long the user free entry can be. Set to null for no limit.

    + * Defaults to null. + */ + maxEntryLength: null, + + /** + * @cfg {String} maxEntryRenderer + *

    A function that defines the helper text when the max entry length has been surpassed.

    + * Defaults to function(v){return 'Please reduce your entry by ' + v + ' character' + (v > 1 ? 's':'');} + */ + maxEntryRenderer: function(v) { + return 'Please reduce your entry by ' + v + ' character' + (v > 1 ? 's':''); + }, + + /** + * @cfg {Integer} maxSuggestions + *

    The maximum number of results displayed in the combo drop down at once.

    + * Defaults to null. + */ + maxSuggestions: null, + + /** + * @cfg {Integer} maxSelection + *

    The maximum number of items the user can select if multiple selection is allowed. + * Set to null to remove the limit.

    + * Defaults to 10. + */ + maxSelection: null, + + /** + * @cfg {Function} maxSelectionRenderer + *

    A function that defines the helper text when the max selection amount has been reached. The function has a single + * parameter which is the number of selected elements.

    + * Defaults to function(v){return 'You cannot choose more than ' + v + ' item' + (v > 1 ? 's':'');} + */ + maxSelectionRenderer: function(v) { + return 'You cannot choose more than ' + v + ' item' + (v > 1 ? 's':''); + }, + + /** + * @cfg {String} method + *

    The method used by the ajax request.

    + * Defaults to 'POST' + */ + method: 'POST', + + /** + * @cfg {Integer} minChars + *

    The minimum number of characters the user must type before the combo expands and offers suggestions. + * Defaults to 0. + */ + minChars: 0, + + /** + * @cfg {Function} minCharsRenderer + *

    A function that defines the helper text when not enough letters are set. The function has a single + * parameter which is the difference between the required amount of letters and the current one.

    + * Defaults to function(v){return 'Please type ' + v + ' more character' + (v > 1 ? 's':'');} + */ + minCharsRenderer: function(v) { + return 'Please type ' + v + ' more character' + (v > 1 ? 's':''); + }, + + /** + * @cfg {String} name + *

    The name used as a form element.

    + * Defaults to 'null' + */ + name: null, + + /** + * @cfg {String} noSuggestionText + *

    The text displayed when there are no suggestions.

    + * Defaults to 'No suggestions" + */ + noSuggestionText: 'No suggestions', + + /** + * @cfg {Boolean} preselectSingleSuggestion + *

    If a single suggestion comes out, it is preselected.

    + * Defaults to true. + */ + preselectSingleSuggestion: true, + + /** + * @cfg (function) renderer + *

    A function used to define how the items will be presented in the combo

    + * Defaults to null. + */ + renderer: null, + + /** + * @cfg {Boolean} required + *

    Whether or not this field should be required

    + * Defaults to false + */ + required: false, + + /** + * @cfg {Boolean} resultAsString + *

    Set to true to render selection as comma separated string

    + * Defaults to false. + */ + resultAsString: false, + + /** + * @cfg {String} resultsField + *

    Name of JSON object property that represents the list of suggested objets

    + * Defaults to results + */ + resultsField: 'results', + + /** + * @cfg {String} selectionCls + *

    A custom CSS class to add to a selected item

    + * Defaults to ''. + */ + selectionCls: '', + + /** + * @cfg {String} selectionPosition + *

    Where the selected items will be displayed. Only 'right', 'bottom' and 'inner' are valid values

    + * Defaults to 'inner', meaning the selected items will appear within the input box itself. + */ + selectionPosition: 'inner', + + /** + * @cfg (function) selectionRenderer + *

    A function used to define how the items will be presented in the tag list

    + * Defaults to null. + */ + selectionRenderer: null, + + /** + * @cfg {Boolean} selectionStacked + *

    Set to true to stack the selectioned items when positioned on the bottom + * Requires the selectionPosition to be set to 'bottom'

    + * Defaults to false. + */ + selectionStacked: false, + + /** + * @cfg {String} sortDir + *

    Direction used for sorting. Only 'asc' and 'desc' are valid values

    + * Defaults to 'asc'. + */ + sortDir: 'asc', + + /** + * @cfg {String} sortOrder + *

    name of JSON object property for local result sorting. + * Leave null if you do not wish the results to be ordered or if they are already ordered remotely.

    + * + * Defaults to null. + */ + sortOrder: null, + + /** + * @cfg {Boolean} strictSuggest + *

    If set to true, suggestions will have to start by user input (and not simply contain it as a substring)

    + * Defaults to false. + */ + strictSuggest: false, + + /** + * @cfg {String} style + *

    Custom style added to the component container.

    + * + * Defaults to ''. + */ + style: '', + + /** + * @cfg {Boolean} toggleOnClick + *

    If set to true, the combo will expand / collapse when clicked upon

    + * Defaults to false. + */ + toggleOnClick: false, + + + /** + * @cfg {Integer} typeDelay + *

    Amount (in ms) between keyboard registers.

    + * + * Defaults to 400 + */ + typeDelay: 400, + + /** + * @cfg {Boolean} useTabKey + *

    If set to true, tab won't blur the component but will be registered as the ENTER key

    + * Defaults to false. + */ + useTabKey: false, + + /** + * @cfg {Boolean} useCommaKey + *

    If set to true, using comma will validate the user's choice

    + * Defaults to true. + */ + useCommaKey: true, + + + /** + * @cfg {Boolean} useZebraStyle + *

    Determines whether or not the results will be displayed with a zebra table style

    + * Defaults to true. + */ + useZebraStyle: true, + + /** + * @cfg {String/Object/Array} value + *

    initial value for the field

    + * Defaults to null. + */ + value: null, + + /** + * @cfg {String} valueField + *

    name of JSON object property that represents its underlying value

    + * Defaults to id. + */ + valueField: 'id', + + /** + * @cfg {Integer} width (in px) + *

    Width of the component

    + * Defaults to underlying element width. + */ + width: function() { + return $(this).width(); + } + }; + + var conf = $.extend({},options); + var cfg = $.extend(true, {}, defaults, conf); + + // some init stuff + if ($.isFunction(cfg.emptyText)) { + cfg.emptyText = cfg.emptyText.call(this); + } + if ($.isFunction(cfg.expandOnFocus)) { + cfg.expandOnFocus = cfg.expandOnFocus.call(this); + } + if ($.isFunction(cfg.id)) { + cfg.id = cfg.id.call(this); + } + + /********** PUBLIC METHODS ************/ + /** + * Add one or multiple json items to the current selection + * @param items - json object or array of json objects + * @param isSilent - (optional) set to true to suppress 'selectionchange' event from being triggered + */ + this.addToSelection = function(items, isSilent) + { + if (!cfg.maxSelection || _selection.length < cfg.maxSelection) { + if (!$.isArray(items)) { + items = [items]; + } + var valuechanged = false; + $.each(items, function(index, json) { + if ($.inArray(json[cfg.valueField], ms.getValue()) === -1) { + _selection.push(json); + valuechanged = true; + } + }); + if(valuechanged === true) { + self._renderSelection(); + this.empty(); + if (isSilent !== true) { + $(this).trigger('selectionchange', [this, this.getSelectedItems()]); + } + } + } + }; + + /** + * Clears the current selection + * @param isSilent - (optional) set to true to suppress 'selectionchange' event from being triggered + */ + this.clear = function(isSilent) + { + this.removeFromSelection(_selection.slice(0), isSilent); // clone array to avoid concurrency issues + }; + + /** + * Collapse the drop down part of the combo + */ + this.collapse = function() + { + if (cfg.expanded === true) { + this.combobox.detach(); + cfg.expanded = false; + $(this).trigger('collapse', [this]); + } + }; + + /** + * Set the component in a disabled state. + */ + this.disable = function() + { + this.container.addClass('tag-ctn-disabled'); + cfg.disabled = true; + ms.input.attr('disabled', true); + }; + + /** + * Empties out the combo user text + */ + this.empty = function(){ + this.input.removeClass(cfg.emptyTextCls); + this.input.val(''); + }; + + /** + * Set the component in a enable state. + */ + this.enable = function() + { + this.container.removeClass('tag-ctn-disabled'); + cfg.disabled = false; + ms.input.attr('disabled', false); + }; + + /** + * Expand the drop drown part of the combo. + */ + this.expand = function() + { + if (!cfg.expanded && (this.input.val().length >= cfg.minChars || this.combobox.children().size() > 0)) { + this.combobox.appendTo(this.container); + self._processSuggestions(); + cfg.expanded = true; + $(this).trigger('expand', [this]); + } + }; + + /** + * Retrieve component enabled status + */ + this.isDisabled = function() + { + return cfg.disabled; + }; + + /** + * Checks whether the field is valid or not + * @return {boolean} + */ + this.isValid = function() + { + return cfg.required === false || _selection.length > 0; + }; + + /** + * Gets the data params for current ajax request + */ + this.getDataUrlParams = function() + { + return cfg.dataUrlParams; + }; + + /** + * Gets the name given to the form input + */ + this.getName = function() + { + return cfg.name; + }; + + /** + * Retrieve an array of selected json objects + * @return {Array} + */ + this.getSelectedItems = function() + { + return _selection; + }; + + /** + * Retrieve the current text entered by the user + */ + this.getRawValue = function(){ + return ms.input.val() !== cfg.emptyText ? ms.input.val() : ''; + }; + + /** + * Retrieve an array of selected values + */ + this.getValue = function() + { + return $.map(_selection, function(o) { + return o[cfg.valueField]; + }); + }; + + /** + * Remove one or multiples json items from the current selection + * @param items - json object or array of json objects + * @param isSilent - (optional) set to true to suppress 'selectionchange' event from being triggered + */ + this.removeFromSelection = function(items, isSilent) + { + if (!$.isArray(items)) { + items = [items]; + } + var valuechanged = false; + $.each(items, function(index, json) { + var i = $.inArray(json[cfg.valueField], ms.getValue()); + if (i > -1) { + _selection.splice(i, 1); + valuechanged = true; + } + }); + if (valuechanged === true) { + self._renderSelection(); + if(isSilent !== true){ + $(this).trigger('selectionchange', [this, this.getSelectedItems()]); + } + if(cfg.expandOnFocus){ + ms.expand(); + } + if(cfg.expanded) { + self._processSuggestions(); + } + } + }; + + /** + * Set up some combo data after it has been rendered + * @param data + */ + this.setData = function(data){ + cfg.data = data; + self._processSuggestions(); + }; + + /** + * Sets the name for the input field so it can be fetched in the form + * @param name + */ + this.setName = function(name){ + cfg.name = name; + if(ms._valueContainer){ + ms._valueContainer.name = name; + } + }; + + /** + * Sets a value for the combo box. Value must be a value or an array of value with data type matching valueField one. + * @param data + */ + this.setValue = function(data) + { + var values = data, items = []; + if(!$.isArray(data)){ + if(typeof(data) === 'string'){ + if(data.indexOf('[') > -1){ + values = eval(data); + } else if(data.indexOf(',') > -1){ + values = data.split(','); + } + } else { + values = [data]; + } + } + + $.each(_cbData, function(index, obj) { + if($.inArray(obj[cfg.valueField], values) > -1) { + items.push(obj); + } + }); + if(items.length > 0) { + this.addToSelection(items); + } + }; + + /** + * Sets data params for subsequent ajax requests + * @param params + */ + this.setDataUrlParams = function(params) + { + cfg.dataUrlParams = $.extend({},params); + }; + + /********** PRIVATE ************/ + var _selection = [], // selected objects + _comboItemHeight = 0, // height for each combo item. + _timer, + _hasFocus = false, + _groups = null, + _cbData = [], + _ctrlDown = false; + + var self = { + + /** + * Empties the result container and refills it with the array of json results in input + * @private + */ + _displaySuggestions: function(data) { + ms.combobox.empty(); + + var resHeight = 0, // total height taken by displayed results. + nbGroups = 0; + + if(_groups === null) { + self._renderComboItems(data); + resHeight = _comboItemHeight * data.length; + } + else { + for(var grpName in _groups) { + nbGroups += 1; + $('
    ', { + 'class': 'tag-res-group', + html: grpName + }).appendTo(ms.combobox); + self._renderComboItems(_groups[grpName].items, true); + } + resHeight = _comboItemHeight * (data.length + nbGroups); + } + + if(resHeight < ms.combobox.height() || resHeight <= cfg.maxDropHeight) { + ms.combobox.height(resHeight); + } + else if(resHeight >= ms.combobox.height() && resHeight > cfg.maxDropHeight) { + ms.combobox.height(cfg.maxDropHeight); + } + + if(data.length === 1 && cfg.preselectSingleSuggestion === true) { + ms.combobox.children().filter(':last').addClass('tag-res-item-active'); + } + + if(data.length === 0 && ms.getRawValue() !== "") { + self._updateHelper(cfg.noSuggestionText); + ms.collapse(); + } + }, + + /** + * Returns an array of json objects from an array of strings. + * @private + */ + _getEntriesFromStringArray: function(data) { + var json = []; + $.each(data, function(index, s) { + var entry = {}; + entry[cfg.displayField] = entry[cfg.valueField] = $.trim(s); + json.push(entry); + }); + return json; + }, + + /** + * Replaces html with highlighted html according to case + * @param html + * @private + */ + _highlightSuggestion: function(html) { + var q = ms.input.val() !== cfg.emptyText ? ms.input.val() : ''; + if(q.length === 0) { + return html; // nothing entered as input + } + + if(cfg.matchCase === true) { + html = html.replace(new RegExp('(' + q + ')(?!([^<]+)?>)','g'), '$1'); + } + else { + html = html.replace(new RegExp('(' + q + ')(?!([^<]+)?>)','gi'), '$1'); + } + return html; + }, + + /** + * Moves the selected cursor amongst the list item + * @param dir - 'up' or 'down' + * @private + */ + _moveSelectedRow: function(dir) { + if(!cfg.expanded) { + ms.expand(); + } + var list, start, active, scrollPos; + list = ms.combobox.find(".tag-res-item"); + if(dir === 'down') { + start = list.eq(0); + } + else { + start = list.filter(':last'); + } + active = ms.combobox.find('.tag-res-item-active:first'); + if(active.length > 0) { + if(dir === 'down') { + start = active.nextAll('.tag-res-item').first(); + if(start.length === 0) { + start = list.eq(0); + } + scrollPos = ms.combobox.scrollTop(); + ms.combobox.scrollTop(0); + if(start[0].offsetTop + start.outerHeight() > ms.combobox.height()) { + ms.combobox.scrollTop(scrollPos + _comboItemHeight); + } + } + else { + start = active.prevAll('.tag-res-item').first(); + if(start.length === 0) { + start = list.filter(':last'); + ms.combobox.scrollTop(_comboItemHeight * list.length); + } + if(start[0].offsetTop < ms.combobox.scrollTop()) { + ms.combobox.scrollTop(ms.combobox.scrollTop() - _comboItemHeight); + } + } + } + list.removeClass("tag-res-item-active"); + start.addClass("tag-res-item-active"); + }, + + /** + * According to given data and query, sort and add suggestions in their container + * @private + */ + _processSuggestions: function(source) { + var json = null, data = source || cfg.data; + if(data !== null) { + if(typeof(data) === 'function'){ + data = data.call(ms); + } + if(typeof(data) === 'string' && data.indexOf(',') < 0) { // get results from ajax + + $(ms).trigger('beforeload', [ms]); + var params = $.extend({query: ms.input.val()}, cfg.dataUrlParams); + $.ajax({ + type: cfg.method, + url: data, + data: params, + success: function(asyncData){ + json = typeof(asyncData) === 'string' ? JSON.parse(asyncData) : asyncData; + self._processSuggestions(json); + $(ms).trigger('load', [ms, json]); + }, + error: function(){ + throw("Could not reach server"); + } + }); + return; + } else if(typeof(data) === 'string' && data.indexOf(',') > -1) { // results from csv string + + _cbData = self._getEntriesFromStringArray(data.split(',')); + } else { // results from local array + + if(data.length > 0 && typeof(data[0]) === 'string') { // results from array of strings + + _cbData = self._getEntriesFromStringArray(data); + } else { // regular json array or json object with results property + + _cbData = data[cfg.resultsField] || data; + + } + } + self._displaySuggestions(self._sortAndTrim(_cbData)); + + } + }, + + /** + * Render the component to the given input DOM element + * @private + */ + _render: function(el) { + $(ms).trigger('beforerender', [ms]); + var w = $.isFunction(cfg.width) ? cfg.width.call(el) : cfg.width; + // holds the main div, will relay the focus events to the contained input element. + ms.container = $('
    ', { + id: cfg.id, + 'class': 'tag-ctn ' + cfg.cls + + (cfg.disabled === true ? ' tag-ctn-disabled' : '') + + (cfg.editable === true ? '' : ' tag-ctn-readonly'), + style: cfg.style + }).width(w); + ms.container.focus($.proxy(handlers._onFocus, this)); + ms.container.blur($.proxy(handlers._onBlur, this)); + ms.container.keydown($.proxy(handlers._onKeyDown, this)); + ms.container.keyup($.proxy(handlers._onKeyUp, this)); + + // holds the input field + ms.input = $('', $.extend({ + id: 'tag-input-' + $('input[id^="tag-input"]').length, + type: 'text', + 'class': cfg.emptyTextCls + (cfg.editable === true ? '' : ' tag-input-readonly'), + value: cfg.emptyText, + readonly: !cfg.editable, + disabled: cfg.disabled + }, cfg.inputCfg)).width(w - (cfg.hideTrigger ? 16 : 42)); + + ms.input.focus($.proxy(handlers._onInputFocus, this)); + ms.input.click($.proxy(handlers._onInputClick, this)); + + // holds the trigger on the right side + if(cfg.hideTrigger === false) { + ms.trigger = $('
    ', { + id: 'tag-trigger-' + $('div[id^="tag-trigger"]').length, + 'class': 'tag-trigger', + html: '
    ' + }); + ms.trigger.click($.proxy(handlers._onTriggerClick, this)); + ms.container.append(ms.trigger); + } + + // holds the suggestions. will always be placed on focus + ms.combobox = $('
    ', { + id: 'tag-res-ctn-' + $('div[id^="tag-res-ctn"]').length, + 'class': 'tag-res-ctn ' + }).width(w).height(cfg.maxDropHeight); + + // bind the onclick and mouseover using delegated events (needs jQuery >= 1.7) + ms.combobox.on('click', 'div.tag-res-item', $.proxy(handlers._onComboItemSelected, this)); + ms.combobox.on('mouseover', 'div.tag-res-item', $.proxy(handlers._onComboItemMouseOver, this)); + + ms.selectionContainer = $('
    ', { + id: 'tag-sel-ctn-' + $('div[id^="tag-sel-ctn"]').length, + 'class': 'tag-sel-ctn' + }); + ms.selectionContainer.click($.proxy(handlers._onFocus, this)); + + if(cfg.selectionPosition === 'inner') { + ms.selectionContainer.append(ms.input); + } + else { + ms.container.append(ms.input); + } + + ms.helper = $('
    ', { + 'class': 'tag-helper ' + cfg.infoMsgCls + }); + self._updateHelper(); + ms.container.append(ms.helper); + + + // Render the whole thing + $(el).replaceWith(ms.container); + + switch(cfg.selectionPosition) { + case 'bottom': + ms.selectionContainer.insertAfter(ms.container); + if(cfg.selectionStacked === true) { + ms.selectionContainer.width(ms.container.width()); + ms.selectionContainer.addClass('tag-stacked'); + } + break; + case 'right': + ms.selectionContainer.insertAfter(ms.container); + ms.container.css('float', 'left'); + break; + default: + ms.container.append(ms.selectionContainer); + break; + } + + self._processSuggestions(); + if(cfg.value !== null) { + ms.setValue(cfg.value); + self._renderSelection(); + } + + $(ms).trigger('afterrender', [ms]); + $("body").click(function(e) { + if(ms.container.hasClass('tag-ctn-bootstrap-focus') && + ms.container.has(e.target).length === 0 && + e.target.className.indexOf('tag-res-item') < 0 && + e.target.className.indexOf('tag-close-btn') < 0 && + ms.container[0] !== e.target) { + handlers._onBlur(); + } + }); + + if(cfg.expanded === true) { + cfg.expanded = false; + ms.expand(); + } + }, + + _renderComboItems: function(items, isGrouped) { + var ref = this, html = ''; + $.each(items, function(index, value) { + var displayed = cfg.renderer !== null ? cfg.renderer.call(ref, value) : value[cfg.displayField]; + var resultItemEl = $('
    ', { + 'class': 'tag-res-item ' + (isGrouped ? 'tag-res-item-grouped ':'') + + (index % 2 === 1 && cfg.useZebraStyle === true ? 'tag-res-odd' : ''), + html: cfg.highlight === true ? self._highlightSuggestion(displayed) : displayed, + 'data-json': JSON.stringify(value) + }); + resultItemEl.click($.proxy(handlers._onComboItemSelected, ref)); + resultItemEl.mouseover($.proxy(handlers._onComboItemMouseOver, ref)); + html += $('
    ').append(resultItemEl).html(); + }); + ms.combobox.append(html); + _comboItemHeight = ms.combobox.find('.tag-res-item:first').outerHeight(); + }, + + /** + * Renders the selected items into their container. + * @private + */ + _renderSelection: function() { + var ref = this, w = 0, inputOffset = 0, items = [], + asText = cfg.resultAsString === true && !_hasFocus; + + ms.selectionContainer.find('.tag-sel-item').remove(); + if(ms._valueContainer !== undefined) { + ms._valueContainer.remove(); + } + + $.each(_selection, function(index, value){ + + var selectedItemEl, delItemEl, + selectedItemHtml = cfg.selectionRenderer !== null ? cfg.selectionRenderer.call(ref, value) : value[cfg.displayField]; + // tag representing selected value + if(asText === true) { + selectedItemEl = $('
    ', { + 'class': 'tag-sel-item tag-sel-text ' + cfg.selectionCls, + html: selectedItemHtml + (index === (_selection.length - 1) ? '' : ',') + }).data('json', value); + } + else { + selectedItemEl = $('
    ', { + 'class': 'tag-sel-item ' + cfg.selectionCls, + html: selectedItemHtml + }).data('json', value); + + if(cfg.disabled === false){ + // small cross img + delItemEl = $('', { + 'class': 'tag-close-btn' + }).data('json', value).appendTo(selectedItemEl); + + delItemEl.click($.proxy(handlers._onTagTriggerClick, ref)); + } + } + + items.push(selectedItemEl); + }); + + ms.selectionContainer.prepend(items); + ms._valueContainer = $('', { + type: 'hidden', + name: cfg.name, + value: JSON.stringify(ms.getValue()) + }); + ms._valueContainer.appendTo(ms.selectionContainer); + + if(cfg.selectionPosition === 'inner') { + ms.input.width(0); + inputOffset = ms.input.offset().left - ms.selectionContainer.offset().left; + w = ms.container.width() - inputOffset - 42; + ms.input.width(w); + ms.container.height(ms.selectionContainer.height()); + } + + if(_selection.length === cfg.maxSelection){ + self._updateHelper(cfg.maxSelectionRenderer.call(this, _selection.length)); + } else { + ms.helper.hide(); + } + }, + + /** + * Select an item either through keyboard or mouse + * @param item + * @private + */ + _selectItem: function(item) { + if(cfg.maxSelection === 1){ + _selection = []; + } + ms.addToSelection(item.data('json')); + item.removeClass('tag-res-item-active'); + if(cfg.expandOnFocus === false || _selection.length === cfg.maxSelection){ + ms.collapse(); + } + if(!_hasFocus){ + ms.input.focus(); + } else if(_hasFocus && (cfg.expandOnFocus || _ctrlDown)){ + self._processSuggestions(); + if(_ctrlDown){ + ms.expand(); + } + } + }, + + /** + * Sorts the results and cut them down to max # of displayed results at once + * @private + */ + _sortAndTrim: function(data) { + var q = ms.getRawValue(), + filtered = [], + newSuggestions = [], + selectedValues = ms.getValue(); + // filter the data according to given input + if(q.length > 0) { + $.each(data, function(index, obj) { + var name = obj[cfg.displayField]; + if((cfg.matchCase === true && name.indexOf(q) > -1) || + (cfg.matchCase === false && name.toLowerCase().indexOf(q.toLowerCase()) > -1)) { + if(cfg.strictSuggest === false || name.toLowerCase().indexOf(q.toLowerCase()) === 0) { + filtered.push(obj); + } + } + }); + } + else { + filtered = data; + } + // take out the ones that have already been selected + $.each(filtered, function(index, obj) { + if($.inArray(obj[cfg.valueField], selectedValues) === -1) { + newSuggestions.push(obj); + } + }); + // sort the data + if(cfg.sortOrder !== null) { + newSuggestions.sort(function(a,b) { + if(a[cfg.sortOrder] < b[cfg.sortOrder]) { + return cfg.sortDir === 'asc' ? -1 : 1; + } + if(a[cfg.sortOrder] > b[cfg.sortOrder]) { + return cfg.sortDir === 'asc' ? 1 : -1; + } + return 0; + }); + } + // trim it down + if(cfg.maxSuggestions && cfg.maxSuggestions > 0) { + newSuggestions = newSuggestions.slice(0, cfg.maxSuggestions); + } + // build groups + if(cfg.groupBy !== null) { + _groups = {}; + $.each(newSuggestions, function(index, value) { + if(_groups[value[cfg.groupBy]] === undefined) { + _groups[value[cfg.groupBy]] = {title: value[cfg.groupBy], items: [value]}; + } + else { + _groups[value[cfg.groupBy]].items.push(value); + } + }); + } + return newSuggestions; + }, + + /** + * Update the helper text + * @private + */ + _updateHelper: function(html) { + ms.helper.html(html); + if(!ms.helper.is(":visible")) { + ms.helper.fadeIn(); + } + } + }; + + var handlers = { + /** + * Triggered when blurring out of the component + * @private + */ + _onBlur: function() { + ms.container.removeClass('tag-ctn-bootstrap-focus'); + ms.collapse(); + _hasFocus = false; + if(ms.getRawValue() !== '' && cfg.allowFreeEntries === true){ + var obj = {}; + obj[cfg.displayField] = obj[cfg.valueField] = ms.getRawValue(); + ms.addToSelection(obj); + } + self._renderSelection(); + + if(ms.isValid() === false) { + ms.container.addClass('tag-ctn-invalid'); + } + + if(ms.input.val() === '' && _selection.length === 0) { + ms.input.addClass(cfg.emptyTextCls); + ms.input.val(cfg.emptyText); + } + else if(ms.input.val() !== '' && cfg.allowFreeEntries === false) { + ms.empty(); + self._updateHelper(''); + } + + if(ms.input.is(":focus")) { + $(ms).trigger('blur', [ms]); + } + }, + + /** + * Triggered when hovering an element in the combo + * @param e + * @private + */ + _onComboItemMouseOver: function(e) { + ms.combobox.children().removeClass('tag-res-item-active'); + $(e.currentTarget).addClass('tag-res-item-active'); + }, + + /** + * Triggered when an item is chosen from the list + * @param e + * @private + */ + _onComboItemSelected: function(e) { + self._selectItem($(e.currentTarget)); + }, + + /** + * Triggered when focusing on the container div. Will focus on the input field instead. + * @private + */ + _onFocus: function() { + ms.input.focus(); + }, + + /** + * Triggered when clicking on the input text field + * @private + */ + _onInputClick: function(){ + if (ms.isDisabled() === false && _hasFocus) { + if (cfg.toggleOnClick === true) { + if (cfg.expanded){ + ms.collapse(); + } else { + ms.expand(); + } + } + } + }, + + /** + * Triggered when focusing on the input text field. + * @private + */ + _onInputFocus: function() { + if(ms.isDisabled() === false && !_hasFocus) { + _hasFocus = true; + ms.container.addClass('tag-ctn-bootstrap-focus'); + ms.container.removeClass(cfg.invalidCls); + + if(ms.input.val() === cfg.emptyText) { + ms.empty(); + } + + var curLength = ms.getRawValue().length; + if(cfg.expandOnFocus === true){ + ms.expand(); + } + + if(_selection.length === cfg.maxSelection) { + self._updateHelper(cfg.maxSelectionRenderer.call(this, _selection.length)); + } else if(curLength < cfg.minChars) { + self._updateHelper(cfg.minCharsRenderer.call(this, cfg.minChars - curLength)); + } + + self._renderSelection(); + $(ms).trigger('focus', [ms]); + } + }, + + /** + * Triggered when the user presses a key while the component has focus + * This is where we want to handle all keys that don't require the user input field + * since it hasn't registered the key hit yet + * @param e keyEvent + * @private + */ + _onKeyDown: function(e) { + // check how tab should be handled + var active = ms.combobox.find('.tag-res-item-active:first'), + freeInput = ms.input.val() !== cfg.emptyText ? ms.input.val() : ''; + $(ms).trigger('keydown', [ms, e]); + + if(e.keyCode === 9 && (cfg.useTabKey === false || + (cfg.useTabKey === true && active.length === 0 && ms.input.val().length === 0))) { + handlers._onBlur(); + return; + } + switch(e.keyCode) { + case 8: //backspace + if(freeInput.length === 0 && ms.getSelectedItems().length > 0 && cfg.selectionPosition === 'inner') { + _selection.pop(); + self._renderSelection(); + $(ms).trigger('selectionchange', [ms, ms.getSelectedItems()]); + ms.input.focus(); + e.preventDefault(); + } + break; + case 9: // tab + case 188: // esc + case 13: // enter + e.preventDefault(); + break; + case 17: // ctrl + _ctrlDown = true; + break; + case 40: // down + e.preventDefault(); + self._moveSelectedRow("down"); + break; + case 38: // up + e.preventDefault(); + self._moveSelectedRow("up"); + break; + default: + if(_selection.length === cfg.maxSelection) { + e.preventDefault(); + } + break; + } + }, + + /** + * Triggered when a key is released while the component has focus + * @param e + * @private + */ + _onKeyUp: function(e) { + var freeInput = ms.getRawValue(), + inputValid = $.trim(ms.input.val()).length > 0 && ms.input.val() !== cfg.emptyText && + (!cfg.maxEntryLength || $.trim(ms.input.val()).length <= cfg.maxEntryLength), + selected, + obj = {}; + + $(ms).trigger('keyup', [ms, e]); + + clearTimeout(_timer); + + // collapse if escape, but keep focus. + if(e.keyCode === 27 && cfg.expanded) { + ms.combobox.height(0); + } + // ignore a bunch of keys + if((e.keyCode === 9 && cfg.useTabKey === false) || (e.keyCode > 13 && e.keyCode < 32)) { + if(e.keyCode === 17){ + _ctrlDown = false; + } + return; + } + switch(e.keyCode) { + case 40:case 38: // up, down + e.preventDefault(); + break; + case 13:case 9:case 188:// enter, tab, comma + if(e.keyCode !== 188 || cfg.useCommaKey === true) { + e.preventDefault(); + if(cfg.expanded === true){ // if a selection is performed, select it and reset field + selected = ms.combobox.find('.tag-res-item-active:first'); + if(selected.length > 0) { + self._selectItem(selected); + return; + } + } + // if no selection or if freetext entered and free entries allowed, add new obj to selection + if(inputValid === true && cfg.allowFreeEntries === true) { + obj[cfg.displayField] = obj[cfg.valueField] = freeInput; + ms.addToSelection(obj); + ms.collapse(); // reset combo suggestions + ms.input.focus(); + } + break; + } + default: + if(_selection.length === cfg.maxSelection){ + self._updateHelper(cfg.maxSelectionRenderer.call(this, _selection.length)); + } + else { + if(freeInput.length < cfg.minChars) { + self._updateHelper(cfg.minCharsRenderer.call(this, cfg.minChars - freeInput.length)); + if(cfg.expanded === true) { + ms.collapse(); + } + } + else if(cfg.maxEntryLength && freeInput.length > cfg.maxEntryLength) { + self._updateHelper(cfg.maxEntryRenderer.call(this, freeInput.length - cfg.maxEntryLength)); + if(cfg.expanded === true) { + ms.collapse(); + } + } + else { + ms.helper.hide(); + if(cfg.minChars <= freeInput.length){ + _timer = setTimeout(function() { + if(cfg.expanded === true) { + self._processSuggestions(); + } else { + ms.expand(); + } + }, cfg.typeDelay); + } + } + } + break; + } + }, + + /** + * Triggered when clicking upon cross for deletion + * @param e + * @private + */ + _onTagTriggerClick: function(e) { + ms.removeFromSelection($(e.currentTarget).data('json')); + }, + + /** + * Triggered when clicking on the small trigger in the right + * @private + */ + _onTriggerClick: function() { + if(ms.isDisabled() === false && !(cfg.expandOnFocus === true && _selection.length === cfg.maxSelection)) { + $(ms).trigger('triggerclick', [ms]); + if(cfg.expanded === true) { + ms.collapse(); + } else { + var curLength = ms.getRawValue().length; + if(curLength >= cfg.minChars){ + ms.input.focus(); + ms.expand(); + } else { + self._updateHelper(cfg.minCharsRenderer.call(this, cfg.minChars - curLength)); + } + } + } + } + }; + + // startup point + if(element !== null) { + self._render(element); + } + }; + + $.fn.tagSuggest = function(options) { + var obj = $(this); + + if(obj.size() === 1 && obj.data('tagSuggest')) { + return obj.data('tagSuggest'); + } + + obj.each(function(i) { + // assume $(this) is an element + var cntr = $(this); + + // Return early if this element already has a plugin instance + if(cntr.data('tagSuggest')){ + return; + } + + if(this.nodeName.toLowerCase() === 'select'){ // rendering from select + options.data = []; + options.value = []; + $.each(this.children, function(index, child){ + if(child.nodeName && child.nodeName.toLowerCase() === 'option'){ + options.data.push({id: child.value, name: child.text}); + if(child.selected){ + options.value.push(child.value); + } + } + }); + + } + + var def = {}; + // set values from DOM container element + $.each(this.attributes, function(i, att){ + def[att.name] = att.value; + }); + var field = new TagSuggest(this, $.extend(options, def)); + cntr.data('tagSuggest', field); + field.container.data('tagSuggest', field); + }); + + if(obj.size() === 1) { + return obj.data('tagSuggest'); + } + return obj; + }; + +// $.fn.tagSuggest.defaults = {}; +})(jQuery); + + + /*$(document).ready(function() { + var jsonData = []; + var fruits = 'Apple,Orange,Banana,Strawberry'.split(','); + //Default values + for(var i=0;i