diff --git a/.gitignore b/.gitignore index 28ed8063..ac4837a3 100644 --- a/.gitignore +++ b/.gitignore @@ -31,15 +31,24 @@ var/www/static/ !var/www/static/js/trendingchart.js var/www/templates/header.html var/www/submitted +var/www/server.crt +var/www/server.key # Local config -bin/packages/config.cfg -bin/packages/config.cfg.backup configs/keys +bin/packages/core.cfg +bin/packages/config.cfg.backup +configs/core.cfg +configs/core.cfg.backup configs/update.cfg update/current_version files +# Helper +bin/helper/gen_cert/rootCA.* +bin/helper/gen_cert/server.* + + # Pystemon archives pystemon/archives diff --git a/HOWTO.md b/HOWTO.md index 00b7017c..9e72c77e 100644 --- a/HOWTO.md +++ b/HOWTO.md @@ -25,7 +25,7 @@ Feed data to AIL: 3. Launch pystemon ``` ./pystemon ``` -4. Edit your configuration file ```bin/packages/config.cfg``` and modify the pystemonpath path accordingly +4. Edit your configuration file ```configs/core.cfg``` and modify the pystemonpath path accordingly 5. Launch pystemon-feeder ``` ./bin/feeder/pystemon-feeder.py ``` @@ -123,7 +123,7 @@ There are two types of installation. You can install a *local* or a *remote* Spl (for a linux docker, the localhost IP is *172.17.0.1*; Should be adapted for other platform) - Restart the tor proxy: ``sudo service tor restart`` -3. *(AIL host)* Edit the ``/bin/packages/config.cfg`` file: +3. *(AIL host)* Edit the ``/configs/core.cfg`` file: - In the crawler section, set ``activate_crawler`` to ``True`` - Change the IP address of Splash servers if needed (remote only) - Set ``splash_onion_port`` according to your Splash servers port numbers that will be used. @@ -134,7 +134,7 @@ There are two types of installation. You can install a *local* or a *remote* Spl - *(Splash host)* Launch all Splash servers with: ```sudo ./bin/torcrawler/launch_splash_crawler.sh -f -p -n ``` -With ```` and ```` matching those specified at ``splash_onion_port`` in the configuration file of point 3 (``/bin/packages/config.cfg``) +With ```` and ```` matching those specified at ``splash_onion_port`` in the configuration file of point 3 (``/configs/core.cfg``) All Splash dockers are launched inside the ``Docker_Splash`` screen. You can use ``sudo screen -r Docker_Splash`` to connect to the screen session and check all Splash servers status. @@ -148,7 +148,7 @@ All Splash dockers are launched inside the ``Docker_Splash`` screen. You can use - ```crawler_hidden_services_install.sh -y``` - Add the following line in ``SOCKSPolicy accept 172.17.0.0/16`` in ``/etc/tor/torrc`` - ```sudo service tor restart``` -- set activate_crawler to True in ``/bin/packages/config.cfg`` +- set activate_crawler to True in ``/configs/core.cfg`` #### Start - ```sudo ./bin/torcrawler/launch_splash_crawler.sh -f $AIL_HOME/configs/docker/splash_onion/etc/splash/proxy-profiles/ -p 8050 -n 1``` @@ -166,4 +166,3 @@ Then starting the crawler service (if you follow the procedure above) ##### Python 3 Upgrade To upgrade from an existing AIL installation, you have to launch [python3_upgrade.sh](./python3_upgrade.sh), this script will delete and create a new virtual environment. The script **will upgrade the packages but won't keep your previous data** (neverthless the data is copied into a directory called `old`). If you install from scratch, you don't require to launch the [python3_upgrade.sh](./python3_upgrade.sh). - diff --git a/OVERVIEW.md b/OVERVIEW.md index 39824bad..40eefa41 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -38,6 +38,21 @@ Redis and ARDB overview | failed_login_ip:**ip** | **nb login failed** | TTL | failed_login_user_id:**user_id** | **nb login failed** | TTL +##### Item Import: + +| Key | Value | +| ------ | ------ | +| **uuid**:nb_total | **nb total** | TTL *(if imported)* +| **uuid**:nb_end | **nb** | TTL *(if imported)* +| **uuid**:nb_sucess | **nb success** | TTL *(if imported)* +| **uuid**:end | **0 (in progress) or (item imported)** | TTL *(if imported)* +| **uuid**:processing | **process status: 0 or 1** | TTL *(if imported)* +| **uuid**:error | **error message** | TTL *(if imported)* + +| Set Key | Value | +| ------ | ------ | +| **uuid**:paste_submit_link | **item_path** | TTL *(if imported)* + ## DB0 - Core: ##### Update keys: @@ -92,47 +107,91 @@ Redis and ARDB overview | ------ | ------ | | misp_module:**module name** | **module dict** | -## DB2 - TermFreq: - -##### Set: +##### Item Import: | Key | Value | | ------ | ------ | -| TrackedSetTermSet | **tracked_term** | -| TrackedSetSet | **tracked_set** | -| TrackedRegexSet | **tracked_regex** | -| | | -| tracked_**tracked_term** | **item_path** | -| set_**tracked_set** | **item_path** | -| regex_**tracked_regex** | **item_path** | -| | | -| TrackedNotifications | **tracked_trem / set / regex** | -| | | -| TrackedNotificationTags_**tracked_trem / set / regex** | **tag** | -| | | -| TrackedNotificationEmails_**tracked_trem / set / regex** | **email** | +| **uuid**:isfile | **boolean** | +| **uuid**:paste_content | **item_content** | -##### Zset: +## DB2 - TermFreq: + +| Set Key | Value | +| ------ | ------ | +| submitted:uuid | **uuid** | +| **uuid**:ltags | **tag** | +| **uuid**:ltagsgalaxies | **tag** | + +## DB3 - Leak Hunter: + +##### Tracker metadata: +| Hset - Key | Field | Value | +| ------ | ------ | ------ | +| tracker:**uuid** | tracker | **tacked word/set/regex** | +| | type | **word/set/regex** | +| | date | **date added** | +| | user_id | **created by user_id** | +| | dashboard | **0/1 Display alert on dashboard** | +| | description | **Tracker description** | +| | level | **0/1 Tracker visibility** | + +##### Tracker by user_id (visibility level: user only): +| Set - Key | Value | +| ------ | ------ | +| user:tracker:**user_id** | **uuid - tracker uuid** | +| user:tracker:**user_id**:**word/set/regex - tracker type** | **uuid - tracker uuid** | + +##### Global Tracker (visibility level: all users): +| Set - Key | Value | +| ------ | ------ | +| gobal:tracker | **uuid - tracker uuid** | +| gobal:tracker:**word/set/regex - tracker type** | **uuid - tracker uuid** | + +##### All Tracker by type: +| Set - Key | Value | +| ------ | ------ | +| all:tracker:**word/set/regex - tracker type** | **tracked item** | + +| Set - Key | Value | +| ------ | ------ | +| all:tracker_uuid:**tracker type**:**tracked item** | **uuid - tracker uuid** | + +##### All Tracked items: +| Set - Key | Value | +| ------ | ------ | +| tracker:item:**uuid**:**date** | **item_id** | + +##### All Tracked tags: +| Set - Key | Value | +| ------ | ------ | +| tracker:tags:**uuid** | **tag** | + +##### All Tracked mail: +| Set - Key | Value | +| ------ | ------ | +| tracker:mail:**uuid** | **mail** | + +##### Refresh Tracker: +| Key | Value | +| ------ | ------ | +| tracker:refresh:word | **last refreshed epoch** | +| tracker:refresh:set | - | +| tracker:refresh:regex | - | + +##### Zset Stat Tracker: | Key | Field | Value | | ------ | ------ | ------ | -| per_paste_TopTermFreq_set_month | **term** | **nb_seen** | -| per_paste_TopTermFreq_set_week | **term** | **nb_seen** | -| per_paste_TopTermFreq_set_day_**epoch** | **term** | **nb_seen** | -| | | | -| TopTermFreq_set_month | **term** | **nb_seen** | -| TopTermFreq_set_week | **term** | **nb_seen** | -| TopTermFreq_set_day_**epoch** | **term** | **nb_seen** | +| tracker:stat:**uuid** | **date** | **nb_seen** | - -##### Hset: +##### Stat token: | Key | Field | Value | | ------ | ------ | ------ | -| TrackedTermDate | **tracked_term** | **epoch** | -| TrackedSetDate | **tracked_set** | **epoch** | -| TrackedRegexDate | **tracked_regex** | **epoch** | +| stat_token_total_by_day:**date** | **word** | **nb_seen** | | | | | -| BlackListTermDate | **blacklisted_term** | **epoch** | -| | | | -| **epoch** | **term** | **nb_seen** | +| stat_token_per_item_by_day:**date** | **word** | **nb_seen** | + +| Set - Key | Value | +| ------ | ------ | +| stat_token_history | **date** | ## DB6 - Tags: @@ -214,6 +273,9 @@ Redis and ARDB overview | set_pgpdump_name:*name* | *item_path* | | | | | set_pgpdump_mail:*mail* | *item_path* | +| | | +| | | +| set_domain_pgpdump_**pgp_type**:**key** | **domain** | ##### Hset date: | Key | Field | Value | @@ -241,11 +303,20 @@ Redis and ARDB overview | item_pgpdump_name:*item_path* | *name* | | | | | item_pgpdump_mail:*item_path* | *mail* | +| | | +| | | +| domain_pgpdump_**pgp_type**:**domain** | **key** | #### Cryptocurrency Supported cryptocurrency: - bitcoin +- bitcoin-cash +- dash +- etherum +- litecoin +- monero +- zcash ##### Hset: | Key | Field | Value | @@ -256,7 +327,8 @@ Supported cryptocurrency: ##### set: | Key | Value | | ------ | ------ | -| set_cryptocurrency_**cryptocurrency name**:**cryptocurrency address** | **item_path** | +| set_cryptocurrency_**cryptocurrency name**:**cryptocurrency address** | **item_path** | PASTE +| domain_cryptocurrency_**cryptocurrency name**:**cryptocurrency address** | **domain** | DOMAIN ##### Hset date: | Key | Field | Value | @@ -271,8 +343,14 @@ Supported cryptocurrency: ##### set: | Key | Value | | ------ | ------ | -| item_cryptocurrency_**cryptocurrency name**:**item_path** | **cryptocurrency address** | +| item_cryptocurrency_**cryptocurrency name**:**item_path** | **cryptocurrency address** | PASTE +| domain_cryptocurrency_**cryptocurrency name**:**item_path** | **cryptocurrency address** | DOMAIN +#### HASH +| Key | Value | +| ------ | ------ | +| hash_domain:**domain** | **hash** | +| domain_hash:**hash** | **domain** | ## DB9 - Crawler: @@ -315,6 +393,20 @@ Supported cryptocurrency: } ``` +##### CRAWLER QUEUES: +| SET - Key | Value | +| ------ | ------ | +| onion_crawler_queue | **url**;**item_id** | RE-CRAWL +| regular_crawler_queue | - | +| | | +| onion_crawler_priority_queue | **url**;**item_id** | USER +| regular_crawler_priority_queue | - | +| | | +| onion_crawler_discovery_queue | **url**;**item_id** | DISCOVER +| regular_crawler_discovery_queue | - | + +##### TO CHANGE: + ARDB overview ----------------------------------------- SENTIMENT ------------------------------------ diff --git a/README.md b/README.md index 7a191ef7..e96241a9 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,8 @@ Features * Multiple feed support * Each module can process and reprocess the information already processed by AIL * Detecting and extracting URLs including their geographical location (e.g. IP address location) -* Extracting and validating potential leak of credit cards numbers, credentials, ... -* Extracting and validating email addresses leaked including DNS MX validation +* Extracting and validating potential leaks of credit card numbers, credentials, ... +* Extracting and validating leaked email addresses, including DNS MX validation * Module for extracting Tor .onion addresses (to be further processed for analysis) * Keep tracks of duplicates (and diffing between each duplicate found) * Extracting and validating potential hostnames (e.g. to feed Passive DNS systems) @@ -103,7 +103,7 @@ Starting AIL ```bash cd bin/ -./LAUNCH -l +./LAUNCH.sh -l ``` Eventually you can browse the status of the AIL framework website at the following URL: @@ -119,6 +119,12 @@ Training CIRCL organises training on how to use or extend the AIL framework. AIL training materials are available at [https://www.circl.lu/services/ail-training-materials/](https://www.circl.lu/services/ail-training-materials/). + +API +----- + +The API documentation is available in [doc/README.md](doc/README.md) + HOWTO ----- diff --git a/bin/BankAccount.py b/bin/BankAccount.py index cd58e3c3..16a8a11f 100755 --- a/bin/BankAccount.py +++ b/bin/BankAccount.py @@ -92,8 +92,6 @@ if __name__ == "__main__": publisher.info("BankAccount started") - message = p.get_from_set() - #iban_regex = re.compile(r'\b[A-Za-z]{2}[0-9]{2}(?:[ ]?[0-9]{4}){4}(?:[ ]?[0-9]{1,2})?\b') iban_regex = re.compile(r'\b([A-Za-z]{2}[ \-]?[0-9]{2})(?=(?:[ \-]?[A-Za-z0-9]){9,30})((?:[ \-]?[A-Za-z0-9]{3,5}){2,6})([ \-]?[A-Za-z0-9]{1,3})\b') iban_regex_verify = re.compile(r'^([A-Z]{2})([0-9]{2})([A-Z0-9]{9,30})$') diff --git a/bin/Bitcoin.py b/bin/Bitcoin.py deleted file mode 100755 index a3cfcfc7..00000000 --- a/bin/Bitcoin.py +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* -""" -The Bitcoin Module -============================ - -It trying to extract Bitcoin address and secret key from paste - - ..seealso:: Paste method (get_regex) - -Requirements ------------- - -*Need running Redis instances. (Redis). - -""" - -from packages import Paste -from Helper import Process -from pubsublogger import publisher - -import re -import time -import redis - -from hashlib import sha256 - - -#### thank http://rosettacode.org/wiki/Bitcoin/address_validation#Python for this 2 functions - -def decode_base58(bc, length): - n = 0 - for char in bc: - n = n * 58 + digits58.index(char) - return n.to_bytes(length, 'big') - -def check_bc(bc): - try: - bcbytes = decode_base58(bc, 25) - return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4] - except Exception: - return False -######################################################## - -def search_key(content, message, paste): - bitcoin_address = re.findall(regex_bitcoin_public_address, content) - bitcoin_private_key = re.findall(regex_bitcoin_private_key, content) - date = str(paste._get_p_date()) - validate_address = False - key = False - if(len(bitcoin_address) >0): - #print(message) - for address in bitcoin_address: - if(check_bc(address)): - validate_address = True - print('Bitcoin address found : {}'.format(address)) - if(len(bitcoin_private_key) > 0): - for private_key in bitcoin_private_key: - print('Bitcoin private key found : {}'.format(private_key)) - key = True - # build bitcoin correlation - save_cryptocurrency_data('bitcoin', date, message, address) - - if(validate_address): - p.populate_set_out(message, 'Duplicate') - to_print = 'Bitcoin found: {} address and {} private Keys'.format(len(bitcoin_address), len(bitcoin_private_key)) - print(to_print) - publisher.warning(to_print) - - msg = 'infoleak:automatic-detection="bitcoin-address";{}'.format(message) - p.populate_set_out(msg, 'Tags') - - if(key): - msg = 'infoleak:automatic-detection="bitcoin-private-key";{}'.format(message) - p.populate_set_out(msg, 'Tags') - - to_print = 'Bitcoin;{};{};{};'.format(paste.p_source, paste.p_date, - paste.p_name) - publisher.warning('{}Detected {} Bitcoin private key;{}'.format( - to_print, len(bitcoin_private_key),paste.p_rel_path)) - -def save_cryptocurrency_data(cryptocurrency_name, date, item_path, cryptocurrency_address): - # create basic medata - if not serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)): - serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'first_seen', date) - serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date) - else: - last_seen = serv_metadata.hget('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen') - if not last_seen: - serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date) - else: - if int(last_seen) < int(date): - serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date) - - # global set - serv_metadata.sadd('set_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), item_path) - - # daily - serv_metadata.hincrby('cryptocurrency:{}:{}'.format(cryptocurrency_name, date), cryptocurrency_address, 1) - - # all type - serv_metadata.zincrby('cryptocurrency_all:{}'.format(cryptocurrency_name), cryptocurrency_address, 1) - - # item_metadata - serv_metadata.sadd('item_cryptocurrency_{}:{}'.format(cryptocurrency_name, item_path), cryptocurrency_address) - -if __name__ == "__main__": - publisher.port = 6380 - publisher.channel = "Script" - - config_section = 'Bitcoin' - - # Setup the I/O queues - p = Process(config_section) - - serv_metadata = redis.StrictRedis( - host=p.config.get("ARDB_Metadata", "host"), - port=p.config.getint("ARDB_Metadata", "port"), - db=p.config.getint("ARDB_Metadata", "db"), - decode_responses=True) - - # Sent to the logging a description of the module - publisher.info("Run Keys module ") - - digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' - - regex_bitcoin_public_address = re.compile(r'(?', path) - p.populate_set_out("New Term added", 'CurveManageTopSets') - - # Send a notification only when the member is in the set - if term in server_term.smembers(TrackedTermsNotificationEnabled_Name): - - # create mail body - mail_body = ("AIL Framework,\n" - "New occurrence for term: " + term + "\n" - ''+full_paste_url + path) - - # Send to every associated email adress - for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term): - sendEmailNotification(email, 'Term', mail_body) - - # tag paste - for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + term): - msg = '{};{}'.format(tag, path) - p.populate_set_out(msg, 'Tags') - - -def getValueOverRange(word, startDate, num_day): - to_return = 0 - for timestamp in range(startDate, startDate - num_day*oneDay, -oneDay): - value = server_term.hget(timestamp, word) - to_return += int(value) if value is not None else 0 - return to_return - - - -if __name__ == "__main__": - publisher.port = 6380 - publisher.channel = "Script" - - config_section = 'Curve' - p = Process(config_section) - - # REDIS # - r_serv1 = redis.StrictRedis( - host=p.config.get("ARDB_Curve", "host"), - port=p.config.get("ARDB_Curve", "port"), - db=p.config.get("ARDB_Curve", "db"), - decode_responses=True) - - server_term = redis.StrictRedis( - host=p.config.get("ARDB_TermFreq", "host"), - port=p.config.get("ARDB_TermFreq", "port"), - db=p.config.get("ARDB_TermFreq", "db"), - decode_responses=True) - - # FUNCTIONS # - publisher.info("Script Curve started") - - # create direct link in mail - full_paste_url = p.config.get("Notifications", "ail_domain") + full_paste_url - - # FILE CURVE SECTION # - csv_path = os.path.join(os.environ['AIL_HOME'], - p.config.get("Directories", "wordtrending_csv")) - wordfile_path = os.path.join(os.environ['AIL_HOME'], - p.config.get("Directories", "wordsfile")) - - message = p.get_from_set() - prec_filename = None - generate_new_graph = False - - # Term Frequency - top_termFreq_setName_day = ["TopTermFreq_set_day_", 1] - top_termFreq_setName_week = ["TopTermFreq_set_week", 7] - top_termFreq_setName_month = ["TopTermFreq_set_month", 31] - - while True: - - if message is not None: - generate_new_graph = True - - filename, word, score = message.split() - temp = filename.split('/') - date = temp[-4] + temp[-3] + temp[-2] - timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0)) - curr_set = top_termFreq_setName_day[0] + str(timestamp) - - - low_word = word.lower() - #Old curve with words in file - r_serv1.hincrby(low_word, date, int(score)) - - # Update redis - #consider the num of occurence of this term - curr_word_value = int(server_term.hincrby(timestamp, low_word, int(score))) - #1 term per paste - curr_word_value_perPaste = int(server_term.hincrby("per_paste_" + str(timestamp), low_word, int(1))) - - # Add in set only if term is not in the blacklist - if low_word not in server_term.smembers(BlackListTermsSet_Name): - #consider the num of occurence of this term - server_term.zincrby(curr_set, low_word, float(score)) - #1 term per paste - server_term.zincrby("per_paste_" + curr_set, low_word, float(1)) - - #Add more info for tracked terms - check_if_tracked_term(low_word, filename) - - #send to RegexForTermsFrequency - to_send = "{} {} {}".format(filename, timestamp, word) - p.populate_set_out(to_send, 'RegexForTermsFrequency') - - else: - - if generate_new_graph: - generate_new_graph = False - print('Building graph') - today = datetime.date.today() - year = today.year - month = today.month - - lib_words.create_curve_with_word_file(r_serv1, csv_path, - wordfile_path, year, - month) - - publisher.debug("Script Curve is Idling") - print("sleeping") - time.sleep(10) - message = p.get_from_set() diff --git a/bin/CurveManageTopSets.py b/bin/CurveManageTopSets.py deleted file mode 100755 index 4eaf9c3f..00000000 --- a/bin/CurveManageTopSets.py +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* -""" - -This module manage top sets for terms frequency. -Every 'refresh_rate' update the weekly and monthly set - -""" - -import redis -import time -import datetime -import copy -from pubsublogger import publisher -from packages import lib_words -import datetime -import calendar -import os -import configparser - -# Config Variables -Refresh_rate = 60*5 #sec -BlackListTermsSet_Name = "BlackListSetTermSet" -TrackedTermsSet_Name = "TrackedSetTermSet" -top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set -oneDay = 60*60*24 -num_day_month = 31 -num_day_week = 7 - -top_termFreq_setName_day = ["TopTermFreq_set_day_", 1] -top_termFreq_setName_week = ["TopTermFreq_set_week", 7] -top_termFreq_setName_month = ["TopTermFreq_set_month", 31] -top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] - - -def manage_top_set(): - startDate = datetime.datetime.now() - startDate = startDate.replace(hour=0, minute=0, second=0, microsecond=0) - startDate = calendar.timegm(startDate.timetuple()) - blacklist_size = int(server_term.scard(BlackListTermsSet_Name)) - - dico = {} - dico_per_paste = {} - - # Retreive top data (max_card + blacklist_size) from days sets - for timestamp in range(startDate, startDate - top_termFreq_setName_month[1]*oneDay, -oneDay): - curr_set = top_termFreq_setName_day[0] + str(timestamp) - array_top_day = server_term.zrevrangebyscore(curr_set, '+inf', '-inf', withscores=True, start=0, num=top_term_freq_max_set_cardinality+blacklist_size) - array_top_day_per_paste = server_term.zrevrangebyscore("per_paste_" + curr_set, '+inf', '-inf', withscores=True, start=0, num=top_term_freq_max_set_cardinality+blacklist_size) - - for word, value in array_top_day: - if word not in server_term.smembers(BlackListTermsSet_Name): - if word in dico.keys(): - dico[word] += value - else: - dico[word] = value - - for word, value in array_top_day_per_paste: - if word not in server_term.smembers(BlackListTermsSet_Name): - if word in dico_per_paste.keys(): - dico_per_paste[word] += value - else: - dico_per_paste[word] = value - - if timestamp == startDate - num_day_week*oneDay: - dico_week = copy.deepcopy(dico) - dico_week_per_paste = copy.deepcopy(dico_per_paste) - - # convert dico into sorted array - array_month = [] - for w, v in dico.items(): - array_month.append((w, v)) - array_month.sort(key=lambda tup: -tup[1]) - array_month = array_month[0:20] - - array_week = [] - for w, v in dico_week.items(): - array_week.append((w, v)) - array_week.sort(key=lambda tup: -tup[1]) - array_week = array_week[0:20] - - # convert dico_per_paste into sorted array - array_month_per_paste = [] - for w, v in dico_per_paste.items(): - array_month_per_paste.append((w, v)) - array_month_per_paste.sort(key=lambda tup: -tup[1]) - array_month_per_paste = array_month_per_paste[0:20] - - array_week_per_paste = [] - for w, v in dico_week_per_paste.items(): - array_week_per_paste.append((w, v)) - array_week_per_paste.sort(key=lambda tup: -tup[1]) - array_week_per_paste = array_week_per_paste[0:20] - - - # suppress every terms in top sets - for curr_set, curr_num_day in top_termFreq_set_array[1:3]: - for w in server_term.zrange(curr_set, 0, -1): - server_term.zrem(curr_set, w) - for w in server_term.zrange("per_paste_" + curr_set, 0, -1): - server_term.zrem("per_paste_" + curr_set, w) - - # Add top term from sorted array in their respective sorted sets - for elem in array_week: - server_term.zadd(top_termFreq_setName_week[0], float(elem[1]), elem[0]) - for elem in array_week_per_paste: - server_term.zadd("per_paste_" + top_termFreq_setName_week[0], float(elem[1]), elem[0]) - - for elem in array_month: - server_term.zadd(top_termFreq_setName_month[0], float(elem[1]), elem[0]) - for elem in array_month_per_paste: - server_term.zadd("per_paste_" + top_termFreq_setName_month[0], float(elem[1]), elem[0]) - - timestamp = int(time.mktime(datetime.datetime.now().timetuple())) - value = str(timestamp) + ", " + "-" - r_temp.set("MODULE_"+ "CurveManageTopSets" + "_" + str(os.getpid()), value) - print("refreshed module") - - - -if __name__ == '__main__': - # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) - # Port of the redis instance used by pubsublogger - publisher.port = 6380 - # Script is the default channel used for the modules. - publisher.channel = 'Script' - - configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') - if not os.path.exists(configfile): - raise Exception('Unable to find the configuration file. \ - Did you set environment variables? \ - Or activate the virtualenv.') - - cfg = configparser.ConfigParser() - cfg.read(configfile) - - - # For Module Manager - r_temp = redis.StrictRedis( - host=cfg.get('RedisPubSub', 'host'), - port=cfg.getint('RedisPubSub', 'port'), - db=cfg.getint('RedisPubSub', 'db'), - decode_responses=True) - - timestamp = int(time.mktime(datetime.datetime.now().timetuple())) - value = str(timestamp) + ", " + "-" - r_temp.set("MODULE_"+ "CurveManageTopSets" + "_" + str(os.getpid()), value) - r_temp.sadd("MODULE_TYPE_"+ "CurveManageTopSets" , str(os.getpid())) - - server_term = redis.StrictRedis( - host=cfg.get("ARDB_TermFreq", "host"), - port=cfg.getint("ARDB_TermFreq", "port"), - db=cfg.getint("ARDB_TermFreq", "db"), - decode_responses=True) - - publisher.info("Script Curve_manage_top_set started") - - # Sent to the logging a description of the module - publisher.info("Manage the top sets with the data created by the module curve.") - - manage_top_set() - - while True: - # Get one message from the input queue (module only work if linked with a queue) - time.sleep(Refresh_rate) # sleep a long time then manage the set - manage_top_set() diff --git a/bin/DbCleaner.py b/bin/DbCleaner.py new file mode 100755 index 00000000..e0cf6512 --- /dev/null +++ b/bin/DbCleaner.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +The DbCleaner Module +=================== + +""" +import os +import sys +import time +import datetime + +from pubsublogger import publisher + +import NotificationHelper + +from packages import Date +from packages import Item +from packages import Term + +def clean_term_db_stat_token(): + all_stat_date = Term.get_all_token_stat_history() + + list_date_to_keep = Date.get_date_range(31) + for date in all_stat_date: + if date not in list_date_to_keep: + # remove history + Term.delete_token_statistics_by_date(date) + + print('Term Stats Cleaned') + + +if __name__ == "__main__": + + publisher.port = 6380 + publisher.channel = "Script" + publisher.info("DbCleaner started") + + # low priority + time.sleep(180) + + daily_cleaner = True + current_date = datetime.datetime.now().strftime("%Y%m%d") + + while True: + + if daily_cleaner: + clean_term_db_stat_token() + daily_cleaner = False + else: + sys.exit(0) + time.sleep(600) + + new_date = datetime.datetime.now().strftime("%Y%m%d") + if new_date != current_date: + current_date = new_date + daily_cleaner = True diff --git a/bin/Decoder.py b/bin/Decoder.py index 76228dfb..82133de7 100755 --- a/bin/Decoder.py +++ b/bin/Decoder.py @@ -18,6 +18,7 @@ from pubsublogger import publisher from Helper import Process from packages import Paste +from packages import Item import re import signal @@ -120,6 +121,12 @@ def save_hash(decoder_name, message, date, decoded): serv_metadata.zincrby('nb_seen_hash:'+hash, message, 1)# hash - paste map serv_metadata.zincrby(decoder_name+'_hash:'+hash, message, 1) # number of b64 on this paste + # Domain Object + if Item.is_crawled(message): + domain = Item.get_item_domain(message) + serv_metadata.sadd('hash_domain:{}'.format(domain), hash) # domain - hash map + serv_metadata.sadd('domain_hash:{}'.format(hash), domain) # hash - domain map + def save_hash_on_disk(decode, type, hash, json_data): diff --git a/bin/Dir.py b/bin/Dir.py deleted file mode 100755 index d76a7ad5..00000000 --- a/bin/Dir.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import argparse -import redis -from pubsublogger import publisher -from packages.lib_words import create_dirfile -import configparser - - -def main(): - """Main Function""" - - # CONFIG # - cfg = configparser.ConfigParser() - cfg.read('./packages/config.cfg') - - parser = argparse.ArgumentParser( - description='''This script is a part of the Analysis Information Leak - framework. It create a redis list called "listfile" which contain - the absolute filename of all the files from the directory given in - the argument "directory".''', - epilog='Example: ./Dir.py /home/2013/03/') - - parser.add_argument('directory', type=str, - help='The directory to run inside', action='store') - - parser.add_argument('-db', type=int, default=0, - help='The name of the Redis DB (default 0)', - choices=[0, 1, 2, 3, 4], action='store') - - parser.add_argument('-ow', help='trigger the overwritting mode', - action='store_true') - - args = parser.parse_args() - - r_serv = redis.StrictRedis(host=cfg.get("Redis_Queues", "host"), - port=cfg.getint("Redis_Queues", "port"), - db=cfg.getint("Redis_Queues", "db"), - decode_responses=True) - - publisher.port = 6380 - publisher.channel = "Script" - - create_dirfile(r_serv, args.directory, args.ow) - -if __name__ == "__main__": - main() diff --git a/bin/Helper.py b/bin/Helper.py index 52097ef6..cda26ce5 100755 --- a/bin/Helper.py +++ b/bin/Helper.py @@ -20,10 +20,10 @@ import datetime import json -class PubSub(object): +class PubSub(object): ## TODO: remove config, use ConfigLoader by default def __init__(self): - configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + configfile = os.path.join(os.environ['AIL_HOME'], 'configs/core.cfg') if not os.path.exists(configfile): raise Exception('Unable to find the configuration file. \ Did you set environment variables? \ @@ -58,7 +58,6 @@ class PubSub(object): for address in addresses.split(','): new_sub = context.socket(zmq.SUB) new_sub.connect(address) - # bytes64 encode bytes to ascii only bytes new_sub.setsockopt_string(zmq.SUBSCRIBE, channel) self.subscribers.append(new_sub) @@ -112,7 +111,7 @@ class PubSub(object): class Process(object): def __init__(self, conf_section, module=True): - configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + configfile = os.path.join(os.environ['AIL_HOME'], 'configs/core.cfg') if not os.path.exists(configfile): raise Exception('Unable to find the configuration file. \ Did you set environment variables? \ diff --git a/bin/IPAddress.py b/bin/IPAddress.py new file mode 100755 index 00000000..f03ee8b3 --- /dev/null +++ b/bin/IPAddress.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +The IP Module +====================== + +This module is consuming the global channel. + +It first performs a regex to find IP addresses and then matches those IPs to +some configured ip ranges. + +The list of IP ranges are expected to be in CIDR format (e.g. 192.168.0.0/16) +and should be defined in the config.cfg file, under the [IP] section + +""" + +import time +import re +from pubsublogger import publisher +from packages import Paste +from Helper import Process +from ipaddress import IPv4Network, IPv4Address + + +def search_ip(message): + paste = Paste.Paste(message) + content = paste.get_p_content() + # regex to find IPs + reg_ip = re.compile(r'^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)', flags=re.MULTILINE) + # list of the regex results in the Paste, may be null + results = reg_ip.findall(content) + matching_ips = [] + + for res in results: + address = IPv4Address(res) + for network in ip_networks: + if address in network: + matching_ips.append(address) + + if len(matching_ips) > 0: + print('{} contains {} IPs'.format(paste.p_name, len(matching_ips))) + publisher.warning('{} contains {} IPs'.format(paste.p_name, len(matching_ips))) + + #Tag message with IP + msg = 'infoleak:automatic-detection="ip";{}'.format(message) + p.populate_set_out(msg, 'Tags') + #Send to duplicate + p.populate_set_out(message, 'Duplicate') + +if __name__ == '__main__': + # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) + # Port of the redis instance used by pubsublogger + publisher.port = 6380 + # Script is the default channel used for the modules. + publisher.channel = 'Script' + + # Section name in bin/packages/modules.cfg + config_section = 'IP' + # Setup the I/O queues + p = Process(config_section) + + ip_networks = [] + for network in p.config.get("IP", "networks").split(","): + ip_networks.append(IPv4Network(network)) + + + # Sent to the logging a description of the module + publisher.info("Run IP module") + + # Endless loop getting messages from the input queue + while True: + # Get one message from the input queue + message = p.get_from_set() + if message is None: + publisher.debug("{} queue is empty, waiting".format(config_section)) + time.sleep(1) + continue + + # Do something with the message from the queue + search_ip(message) + diff --git a/bin/Keys.py b/bin/Keys.py index eb06601a..237f807c 100755 --- a/bin/Keys.py +++ b/bin/Keys.py @@ -121,6 +121,13 @@ def search_key(paste): p.populate_set_out(msg, 'Tags') find = True + if '-----BEGIN PUBLIC KEY-----' in content: + publisher.warning('{} has a public key message'.format(paste.p_name)) + + msg = 'infoleak:automatic-detection="public-key";{}'.format(message) + p.populate_set_out(msg, 'Tags') + find = True + # pgp content if get_pgp_content: p.populate_set_out(message, 'PgpDump') diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index e4175b90..406c4a77 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -66,8 +66,8 @@ function helptext { "$DEFAULT" This script launch: "$CYAN" - - All the ZMQ queuing modules. - - All the ZMQ processing modules. + - All the queuing modules. + - All the processing modules. - All Redis in memory servers. - All ARDB on disk servers. "$DEFAULT" @@ -76,12 +76,15 @@ function helptext { Usage: ----- LAUNCH.sh - [-l | --launchAuto] - [-k | --killAll] - [-u | --update] - [-c | --configUpdate] - [-t | --thirdpartyUpdate] - [-h | --help] + [-l | --launchAuto] LAUNCH DB + Scripts + [-k | --killAll] Kill DB + Scripts + [-ks | --killscript] Scripts + [-u | --update] Update AIL + [-c | --crawler] LAUNCH Crawlers + [-f | --launchFeeder] LAUNCH Pystemon feeder + [-t | --thirdpartyUpdate] Update Web + [-m | --menu] Display Advanced Menu + [-h | --help] Help " } @@ -143,7 +146,7 @@ function launching_scripts { screen -dmS "Script_AIL" sleep 0.1 - echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT + echo -e $GREEN"\t* Launching scripts"$DEFAULT screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x" sleep 0.1 @@ -153,14 +156,10 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Lines" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Lines.py; read x" - sleep 0.1 screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./DomClassifier.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Categ.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tokenize" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tokenize.py; read x" - sleep 0.1 screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x" @@ -175,13 +174,9 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "Credential" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Credential.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Curve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Curve.py; read x" + screen -S "Script_AIL" -X screen -t "TermTrackerMod" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./TermTrackerMod.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CurveManageTopSets.py; read x" - sleep 0.1 - screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexForTermsFrequency.py; read x" - sleep 0.1 - screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SetForTermsFrequency.py; read x" + screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x" sleep 0.1 @@ -191,7 +186,9 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Decoder.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Bitcoin.py; read x" + screen -S "Script_AIL" -X screen -t "Cryptocurrency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cryptocurrencies.py; read x" + sleep 0.1 + screen -S "Script_AIL" -X screen -t "Tools" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tools.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Phone.py; read x" sleep 0.1 @@ -213,15 +210,19 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SentimentAnalysis.py; read x" sleep 0.1 + screen -S "Script_AIL" -X screen -t "DbCleaner" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./DbCleaner.py; read x" + sleep 0.1 screen -S "Script_AIL" -X screen -t "UpdateBackground" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./update-background.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./submit_paste.py; read x" + sleep 0.1 + screen -S "Script_AIL" -X screen -t "IPAddress" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./IPAddress.py; read x" } function launching_crawler { if [[ ! $iscrawler ]]; then - CONFIG=$AIL_BIN/packages/config.cfg + CONFIG=$AIL_HOME/configs/core.cfg lport=$(awk '/^\[Crawler\]/{f=1} f==1&&/^splash_port/{print $3;exit}' "${CONFIG}") IFS='-' read -ra PORTS <<< "$lport" @@ -404,6 +405,18 @@ function launch_feeder { fi } +function killscript { + if [[ $islogged || $isqueued || $isscripted || $isflasked || $isfeeded || $iscrawler ]]; then + echo -e $GREEN"Killing Script"$DEFAULT + kill $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler + sleep 0.2 + echo -e $ROSE`screen -ls`$DEFAULT + echo -e $GREEN"\t* $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler killed."$DEFAULT + else + echo -e $RED"\t* No script to kill"$DEFAULT + fi +} + function killall { if [[ $isredis || $isardb || $islogged || $isqueued || $isscripted || $isflasked || $isfeeded || $iscrawler ]]; then if [[ $isredis ]]; then @@ -463,76 +476,82 @@ function launch_all { launch_flask; } -#If no params, display the menu +function menu_display { + + options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Flask" "Killall" "Shutdown" "Update" "Update-config" "Update-thirdparty") + + menu() { + echo "What do you want to Launch?:" + for i in ${!options[@]}; do + printf "%3d%s) %s\n" $((i+1)) "${choices[i]:- }" "${options[i]}" + done + [[ "$msg" ]] && echo "$msg"; : + } + + prompt="Check an option (again to uncheck, ENTER when done): " + + while menu && read -rp "$prompt" numinput && [[ "$numinput" ]]; do + for num in $numinput; do + [[ "$num" != *[![:digit:]]* ]] && (( num > 0 && num <= ${#options[@]} )) || { + msg="Invalid option: $num"; break + } + ((num--)); msg="${options[num]} was ${choices[num]:+un}checked" + [[ "${choices[num]}" ]] && choices[num]="" || choices[num]="+" + done + done + + for i in ${!options[@]}; do + if [[ "${choices[i]}" ]]; then + case ${options[i]} in + Redis) + launch_redis; + ;; + Ardb) + launch_ardb; + ;; + Logs) + launch_logs; + ;; + Queues) + launch_queues; + ;; + Scripts) + launch_scripts; + ;; + Flask) + launch_flask; + ;; + Crawler) + launching_crawler; + ;; + Killall) + killall; + ;; + Shutdown) + shutdown; + ;; + Update) + update; + ;; + Update-config) + checking_configuration; + ;; + Update-thirdparty) + update_thirdparty; + ;; + esac + fi + done + + exit + +} + + +#If no params, display the help [[ $@ ]] || { helptext; - - options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Flask" "Killall" "Shutdown" "Update" "Update-config" "Update-thirdparty") - - menu() { - echo "What do you want to Launch?:" - for i in ${!options[@]}; do - printf "%3d%s) %s\n" $((i+1)) "${choices[i]:- }" "${options[i]}" - done - [[ "$msg" ]] && echo "$msg"; : - } - - prompt="Check an option (again to uncheck, ENTER when done): " - while menu && read -rp "$prompt" numinput && [[ "$numinput" ]]; do - for num in $numinput; do - [[ "$num" != *[![:digit:]]* ]] && (( num > 0 && num <= ${#options[@]} )) || { - msg="Invalid option: $num"; break - } - ((num--)); msg="${options[num]} was ${choices[num]:+un}checked" - [[ "${choices[num]}" ]] && choices[num]="" || choices[num]="+" - done - done - - for i in ${!options[@]}; do - if [[ "${choices[i]}" ]]; then - case ${options[i]} in - Redis) - launch_redis; - ;; - Ardb) - launch_ardb; - ;; - Logs) - launch_logs; - ;; - Queues) - launch_queues; - ;; - Scripts) - launch_scripts; - ;; - Flask) - launch_flask; - ;; - Crawler) - launching_crawler; - ;; - Killall) - killall; - ;; - Shutdown) - shutdown; - ;; - Update) - update; - ;; - Update-config) - checking_configuration; - ;; - Update-thirdparty) - update_thirdparty; - ;; - esac - fi - done - - exit } #echo "$@" @@ -553,6 +572,10 @@ while [ "$1" != "" ]; do ;; -k | --killAll ) killall; ;; + -ks | --killscript ) killscript; + ;; + -m | --menu ) menu_display; + ;; -u | --update ) update; ;; -t | --thirdpartyUpdate ) update_thirdparty; @@ -565,7 +588,6 @@ while [ "$1" != "" ]; do exit ;; -kh | --khelp ) helptext; - ;; * ) helptext exit 1 diff --git a/bin/Lines.py b/bin/Lines.py deleted file mode 100755 index e4187dc7..00000000 --- a/bin/Lines.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -""" -The ZMQ_PubSub_Lines Module -============================ - -This module is consuming the Redis-list created by the ZMQ_PubSub_Line_Q -Module. - -It perform a sorting on the line's length and publish/forward them to -differents channels: - -*Channel 1 if max length(line) < max -*Channel 2 if max length(line) > max - -The collected informations about the processed pastes -(number of lines and maximum length line) are stored in Redis. - -..note:: Module ZMQ_Something_Q and ZMQ_Something are closely bound, always put -the same Subscriber name in both of them. - -Requirements ------------- - -*Need running Redis instances. (LevelDB & Redis) -*Need the ZMQ_PubSub_Line_Q Module running to be able to work properly. - -""" -import argparse -import time -from packages import Paste -from pubsublogger import publisher - -from Helper import Process - -if __name__ == '__main__': - publisher.port = 6380 - publisher.channel = 'Script' - - config_section = 'Lines' - p = Process(config_section) - - # SCRIPT PARSER # - parser = argparse.ArgumentParser( - description='This script is a part of the Analysis Information \ - Leak framework.') - - parser.add_argument( - '-max', type=int, default=500, - help='The limit between "short lines" and "long lines"', - action='store') - - args = parser.parse_args() - - # FUNCTIONS # - tmp_string = "Lines script Subscribed to channel {} and Start to publish \ - on channel Longlines, Shortlines" - publisher.info(tmp_string) - - while True: - try: - message = p.get_from_set() - print(message) - if message is not None: - PST = Paste.Paste(message) - else: - publisher.debug("Tokeniser is idling 10s") - time.sleep(10) - continue - - # FIXME do it in the paste class - lines_infos = PST.get_lines_info() - PST.save_attribute_redis("p_nb_lines", lines_infos[0]) - PST.save_attribute_redis("p_max_length_line", lines_infos[1]) - - # FIXME Not used. - PST.store.sadd("Pastes_Objects", PST.p_rel_path) - print(PST.p_rel_path) - if lines_infos[1] < args.max: - p.populate_set_out( PST.p_rel_path , 'LinesShort') - else: - p.populate_set_out( PST.p_rel_path , 'LinesLong') - except IOError: - print("CRC Checksum Error on : ", PST.p_rel_path) diff --git a/bin/MISP_The_Hive_feeder.py b/bin/MISP_The_Hive_feeder.py index 2bc33431..33a8841f 100755 --- a/bin/MISP_The_Hive_feeder.py +++ b/bin/MISP_The_Hive_feeder.py @@ -8,20 +8,20 @@ module This module send tagged pastes to MISP or THE HIVE Project """ - -import redis -import sys import os +import sys +import uuid +import redis import time import json -import configparser from pubsublogger import publisher from Helper import Process from packages import Paste import ailleakObject -import uuid +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader from pymisp import PyMISP @@ -133,26 +133,10 @@ if __name__ == "__main__": config_section = 'MISP_The_hive_feeder' - configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') - if not os.path.exists(configfile): - raise Exception('Unable to find the configuration file. \ - Did you set environment variables? \ - Or activate the virtualenv.') + config_loader = ConfigLoader.ConfigLoader() - cfg = configparser.ConfigParser() - cfg.read(configfile) - - r_serv_db = redis.StrictRedis( - host=cfg.get("ARDB_DB", "host"), - port=cfg.getint("ARDB_DB", "port"), - db=cfg.getint("ARDB_DB", "db"), - decode_responses=True) - - r_serv_metadata = redis.StrictRedis( - host=cfg.get("ARDB_Metadata", "host"), - port=cfg.getint("ARDB_Metadata", "port"), - db=cfg.getint("ARDB_Metadata", "db"), - decode_responses=True) + r_serv_db = config_loader.get_redis_conn("ARDB_DB") + r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") # set sensor uuid uuid_ail = r_serv_db.get('ail:uuid') @@ -212,7 +196,9 @@ if __name__ == "__main__": refresh_time = 3 ## FIXME: remove it - PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + config_loader = None + time_1 = time.time() while True: diff --git a/bin/Mixer.py b/bin/Mixer.py index cbb39676..fd8bedc5 100755 --- a/bin/Mixer.py +++ b/bin/Mixer.py @@ -29,16 +29,20 @@ Every data coming from a named feed can be sent to a pre-processing module befor The mapping can be done via the variable FEED_QUEUE_MAPPING """ +import os +import sys + import base64 import hashlib -import os import time from pubsublogger import publisher import redis -import configparser from Helper import Process +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + # CONFIG # refresh_time = 30 @@ -52,37 +56,22 @@ if __name__ == '__main__': p = Process(config_section) - configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') - if not os.path.exists(configfile): - raise Exception('Unable to find the configuration file. \ - Did you set environment variables? \ - Or activate the virtualenv.') - - cfg = configparser.ConfigParser() - cfg.read(configfile) + config_loader = ConfigLoader.ConfigLoader() # REDIS # - server = redis.StrictRedis( - host=cfg.get("Redis_Mixer_Cache", "host"), - port=cfg.getint("Redis_Mixer_Cache", "port"), - db=cfg.getint("Redis_Mixer_Cache", "db"), - decode_responses=True) - - server_cache = redis.StrictRedis( - host=cfg.get("Redis_Log_submit", "host"), - port=cfg.getint("Redis_Log_submit", "port"), - db=cfg.getint("Redis_Log_submit", "db"), - decode_responses=True) + server = config_loader.get_redis_conn("Redis_Mixer_Cache") + server_cache = config_loader.get_redis_conn("Redis_Log_submit") # LOGGING # publisher.info("Feed Script started to receive & publish.") # OTHER CONFIG # - operation_mode = cfg.getint("Module_Mixer", "operation_mode") - ttl_key = cfg.getint("Module_Mixer", "ttl_duplicate") - default_unnamed_feed_name = cfg.get("Module_Mixer", "default_unnamed_feed_name") + operation_mode = config_loader.get_config_int("Module_Mixer", "operation_mode") + ttl_key = config_loader.get_config_int("Module_Mixer", "ttl_duplicate") + default_unnamed_feed_name = config_loader.get_config_str("Module_Mixer", "default_unnamed_feed_name") - PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes")) + '/' + PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/' + config_loader = None # STATS # processed_paste = 0 diff --git a/bin/ModuleInformation.py b/bin/ModuleInformation.py deleted file mode 100755 index 807cb87e..00000000 --- a/bin/ModuleInformation.py +++ /dev/null @@ -1,354 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -''' - -This module can be use to see information of running modules. -These information are logged in "logs/moduleInfo.log" - -It can also try to manage them by killing inactive one. -However, it does not support mutliple occurence of the same module -(It will kill the first one obtained by get) - - -''' - -import time -import datetime -import redis -import os -import signal -import argparse -from subprocess import PIPE, Popen -import configparser -import json -from terminaltables import AsciiTable -import textwrap -from colorama import Fore, Back, Style, init -import curses - -# CONFIG VARIABLES -kill_retry_threshold = 60 #1m -log_filename = "../logs/moduleInfo.log" -command_search_pid = "ps a -o pid,cmd | grep {}" -command_search_name = "ps a -o pid,cmd | grep {}" -command_restart_module = "screen -S \"Script\" -X screen -t \"{}\" bash -c \"./{}.py; read x\"" - -init() #Necesary for colorama -printarrayGlob = [None]*14 -printarrayGlob.insert(0, ["Time", "Module", "PID", "Action"]) -lastTimeKillCommand = {} - -#Curses init -#stdscr = curses.initscr() -#curses.cbreak() -#stdscr.keypad(1) - -# GLOBAL -last_refresh = 0 - - -def getPid(module): - p = Popen([command_search_pid.format(module+".py")], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True) - for line in p.stdout: - print(line) - splittedLine = line.split() - if 'python2' in splittedLine: - return int(splittedLine[0]) - return None - -def clearRedisModuleInfo(): - for k in server.keys("MODULE_*"): - server.delete(k) - inst_time = datetime.datetime.fromtimestamp(int(time.time())) - printarrayGlob.insert(1, [inst_time, "*", "-", "Cleared redis module info"]) - printarrayGlob.pop() - -def cleanRedis(): - for k in server.keys("MODULE_TYPE_*"): - moduleName = k[12:].split('_')[0] - for pid in server.smembers(k): - flag_pid_valid = False - proc = Popen([command_search_name.format(pid)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True) - for line in proc.stdout: - splittedLine = line.split() - if ('python2' in splittedLine or 'python' in splittedLine) and "./"+moduleName+".py" in splittedLine: - flag_pid_valid = True - - if not flag_pid_valid: - print(flag_pid_valid, 'cleaning', pid, 'in', k) - server.srem(k, pid) - inst_time = datetime.datetime.fromtimestamp(int(time.time())) - printarrayGlob.insert(1, [inst_time, moduleName, pid, "Cleared invalid pid in " + k]) - printarrayGlob.pop() - #time.sleep(5) - - -def kill_module(module, pid): - print('') - print('-> trying to kill module:', module) - - if pid is None: - print('pid was None') - printarrayGlob.insert(1, [0, module, pid, "PID was None"]) - printarrayGlob.pop() - pid = getPid(module) - else: #Verify that the pid is at least in redis - if server.exists("MODULE_"+module+"_"+str(pid)) == 0: - return - - lastTimeKillCommand[pid] = int(time.time()) - if pid is not None: - try: - os.kill(pid, signal.SIGUSR1) - except OSError: - print(pid, 'already killed') - inst_time = datetime.datetime.fromtimestamp(int(time.time())) - printarrayGlob.insert(1, [inst_time, module, pid, "Already killed"]) - printarrayGlob.pop() - return - time.sleep(1) - if getPid(module) is None: - print(module, 'has been killed') - print('restarting', module, '...') - p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True) - inst_time = datetime.datetime.fromtimestamp(int(time.time())) - printarrayGlob.insert(1, [inst_time, module, pid, "Killed"]) - printarrayGlob.insert(1, [inst_time, module, "?", "Restarted"]) - printarrayGlob.pop() - printarrayGlob.pop() - - else: - print('killing failed, retrying...') - inst_time = datetime.datetime.fromtimestamp(int(time.time())) - printarrayGlob.insert(1, [inst_time, module, pid, "Killing #1 failed."]) - printarrayGlob.pop() - - time.sleep(1) - os.kill(pid, signal.SIGUSR1) - time.sleep(1) - if getPid(module) is None: - print(module, 'has been killed') - print('restarting', module, '...') - p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True) - inst_time = datetime.datetime.fromtimestamp(int(time.time())) - printarrayGlob.insert(1, [inst_time, module, pid, "Killed"]) - printarrayGlob.insert(1, [inst_time, module, "?", "Restarted"]) - printarrayGlob.pop() - printarrayGlob.pop() - else: - print('killing failed!') - inst_time = datetime.datetime.fromtimestamp(int(time.time())) - printarrayGlob.insert(1, [inst_time, module, pid, "Killing failed!"]) - printarrayGlob.pop() - else: - print('Module does not exist') - inst_time = datetime.datetime.fromtimestamp(int(time.time())) - printarrayGlob.insert(1, [inst_time, module, pid, "Killing failed, module not found"]) - printarrayGlob.pop() - #time.sleep(5) - cleanRedis() - -def get_color(time, idle): - if time is not None: - temp = time.split(':') - time = int(temp[0])*3600 + int(temp[1])*60 + int(temp[2]) - - if time >= args.treshold: - if not idle: - return Back.RED + Style.BRIGHT - else: - return Back.MAGENTA + Style.BRIGHT - elif time > args.treshold/2: - return Back.YELLOW + Style.BRIGHT - else: - return Back.GREEN + Style.BRIGHT - else: - return Style.RESET_ALL - -def waiting_refresh(): - global last_refresh - if time.time() - last_refresh < args.refresh: - return False - else: - last_refresh = time.time() - return True - - - -if __name__ == "__main__": - - parser = argparse.ArgumentParser(description='Show info concerning running modules and log suspected stucked modules. May be use to automatically kill and restart stucked one.') - parser.add_argument('-r', '--refresh', type=int, required=False, default=1, help='Refresh rate') - parser.add_argument('-t', '--treshold', type=int, required=False, default=60*10*1, help='Refresh rate') - parser.add_argument('-k', '--autokill', type=int, required=False, default=0, help='Enable auto kill option (1 for TRUE, anything else for FALSE)') - parser.add_argument('-c', '--clear', type=int, required=False, default=0, help='Clear the current module information (Used to clear data from old launched modules)') - - args = parser.parse_args() - - configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') - if not os.path.exists(configfile): - raise Exception('Unable to find the configuration file. \ - Did you set environment variables? \ - Or activate the virtualenv.') - - cfg = configparser.ConfigParser() - cfg.read(configfile) - - # REDIS # - server = redis.StrictRedis( - host=cfg.get("Redis_Queues", "host"), - port=cfg.getint("Redis_Queues", "port"), - db=cfg.getint("Redis_Queues", "db"), - decode_responses=True) - - if args.clear == 1: - clearRedisModuleInfo() - - lastTime = datetime.datetime.now() - - module_file_array = set() - no_info_modules = {} - path_allmod = os.path.join(os.environ['AIL_HOME'], 'doc/all_modules.txt') - with open(path_allmod, 'r') as module_file: - for line in module_file: - module_file_array.add(line[:-1]) - - cleanRedis() - - while True: - if waiting_refresh(): - - #key = '' - #while key != 'q': - # key = stdsrc.getch() - # stdscr.refresh() - - all_queue = set() - printarray1 = [] - printarray2 = [] - printarray3 = [] - for queue, card in server.hgetall("queues").items(): - all_queue.add(queue) - key = "MODULE_" + queue + "_" - keySet = "MODULE_TYPE_" + queue - array_module_type = [] - - for moduleNum in server.smembers(keySet): - value = server.get(key + str(moduleNum)) - if value is not None: - timestamp, path = value.split(", ") - if timestamp is not None and path is not None: - startTime_readable = datetime.datetime.fromtimestamp(int(timestamp)) - processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0] - - if int(card) > 0: - if int((datetime.datetime.now() - startTime_readable).total_seconds()) > args.treshold: - log = open(log_filename, 'a') - log.write(json.dumps([queue, card, str(startTime_readable), str(processed_time_readable), path]) + "\n") - try: - last_kill_try = time.time() - lastTimeKillCommand[moduleNum] - except KeyError: - last_kill_try = kill_retry_threshold+1 - if args.autokill == 1 and last_kill_try > kill_retry_threshold : - kill_module(queue, int(moduleNum)) - - array_module_type.append([get_color(processed_time_readable, False) + str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path) + get_color(None, False)]) - - else: - printarray2.append([get_color(processed_time_readable, True) + str(queue), str(moduleNum), str(card), str(startTime_readable), str(processed_time_readable), str(path) + get_color(None, True)]) - array_module_type.sort(lambda x,y: cmp(x[4], y[4]), reverse=True) - for e in array_module_type: - printarray1.append(e) - - for curr_queue in module_file_array: - if curr_queue not in all_queue: - printarray3.append([curr_queue, "Not running"]) - else: - if len(list(server.smembers('MODULE_TYPE_'+curr_queue))) == 0: - if curr_queue not in no_info_modules: - no_info_modules[curr_queue] = int(time.time()) - printarray3.append([curr_queue, "No data"]) - else: - #If no info since long time, try to kill - if args.autokill == 1: - if int(time.time()) - no_info_modules[curr_queue] > args.treshold: - kill_module(curr_queue, None) - no_info_modules[curr_queue] = int(time.time()) - printarray3.append([curr_queue, "Stuck or idle, restarting in " + str(abs(args.treshold - (int(time.time()) - no_info_modules[curr_queue]))) + "s"]) - else: - printarray3.append([curr_queue, "Stuck or idle, restarting disabled"]) - - ## FIXME To add: - ## Button KILL Process using Curses - - printarray1.sort(key=lambda x: x[0][9:], reverse=False) - printarray2.sort(key=lambda x: x[0][9:], reverse=False) - printarray1.insert(0,["Queue", "PID", "Amount", "Paste start time", "Processing time for current paste (H:M:S)", "Paste hash"]) - printarray2.insert(0,["Queue", "PID","Amount", "Paste start time", "Time since idle (H:M:S)", "Last paste hash"]) - printarray3.insert(0,["Queue", "State"]) - - os.system('clear') - t1 = AsciiTable(printarray1, title="Working queues") - t1.column_max_width(1) - if not t1.ok: - longest_col = t1.column_widths.index(max(t1.column_widths)) - max_length_col = t1.column_max_width(longest_col) - if max_length_col > 0: - for i, content in enumerate(t1.table_data): - if len(content[longest_col]) > max_length_col: - temp = '' - for l in content[longest_col].splitlines(): - if len(l) > max_length_col: - temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n' - else: - temp += l + '\n' - content[longest_col] = temp.strip() - t1.table_data[i] = content - - t2 = AsciiTable(printarray2, title="Idling queues") - t2.column_max_width(1) - if not t2.ok: - longest_col = t2.column_widths.index(max(t2.column_widths)) - max_length_col = t2.column_max_width(longest_col) - if max_length_col > 0: - for i, content in enumerate(t2.table_data): - if len(content[longest_col]) > max_length_col: - temp = '' - for l in content[longest_col].splitlines(): - if len(l) > max_length_col: - temp += '\n'.join(textwrap.wrap(l, max_length_col)) + '\n' - else: - temp += l + '\n' - content[longest_col] = temp.strip() - t2.table_data[i] = content - - t3 = AsciiTable(printarray3, title="Not running queues") - t3.column_max_width(1) - - printarray4 = [] - for elem in printarrayGlob: - if elem is not None: - printarray4.append(elem) - - t4 = AsciiTable(printarray4, title="Last actions") - t4.column_max_width(1) - - legend_array = [["Color", "Meaning"], [Back.RED+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(args.treshold)+Style.RESET_ALL], [Back.MAGENTA+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(args.treshold)+" while idle"+Style.RESET_ALL], [Back.YELLOW+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time >=" +str(args.treshold/2)+Style.RESET_ALL], [Back.GREEN+Style.BRIGHT+" "*10+Style.RESET_ALL, "Time <" +str(args.treshold)]] - legend = AsciiTable(legend_array, title="Legend") - legend.column_max_width(1) - - print(legend.table) - print('\n') - print(t1.table) - print('\n') - print(t2.table) - print('\n') - print(t3.table) - print('\n') - print(t4.table9) - - if (datetime.datetime.now() - lastTime).total_seconds() > args.refresh*5: - lastTime = datetime.datetime.now() - cleanRedis() - #time.sleep(args.refresh) diff --git a/bin/ModuleStats.py b/bin/ModuleStats.py index 6743cdca..cfdb82f7 100755 --- a/bin/ModuleStats.py +++ b/bin/ModuleStats.py @@ -9,7 +9,6 @@ import time import datetime import redis import os -from packages import lib_words from packages.Date import Date from pubsublogger import publisher from Helper import Process diff --git a/bin/ModulesInformationV2.py b/bin/ModulesInformationV2.py index cef6301c..def7509f 100755 --- a/bin/ModulesInformationV2.py +++ b/bin/ModulesInformationV2.py @@ -10,13 +10,16 @@ from asciimatics.event import Event from asciimatics.event import KeyboardEvent, MouseEvent import sys, os import time, datetime -import argparse, configparser +import argparse import json import redis import psutil from subprocess import PIPE, Popen from packages import Paste +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + # CONFIG VARIABLES kill_retry_threshold = 60 #1m log_filename = "../logs/moduleInfo.log" @@ -798,21 +801,11 @@ if __name__ == "__main__": args = parser.parse_args() - configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') - if not os.path.exists(configfile): - raise Exception('Unable to find the configuration file. \ - Did you set environment variables? \ - Or activate the virtualenv.') - - cfg = configparser.ConfigParser() - cfg.read(configfile) + config_loader = ConfigLoader.ConfigLoader() # REDIS # - server = redis.StrictRedis( - host=cfg.get("Redis_Queues", "host"), - port=cfg.getint("Redis_Queues", "port"), - db=cfg.getint("Redis_Queues", "db"), - decode_responses=True) + server = config_loader.get_redis_conn("Redis_Queues") + config_loader = None if args.clear == 1: clearRedisModuleInfo() diff --git a/bin/NotificationHelper.py b/bin/NotificationHelper.py index 1bccd314..02568a1e 100755 --- a/bin/NotificationHelper.py +++ b/bin/NotificationHelper.py @@ -1,46 +1,34 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* -import argparse -import configparser -import traceback import os +import sys + +import argparse +import traceback import smtplib from pubsublogger import publisher from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + """ This module allows the global configuration and management of notification settings and methods. """ -# CONFIG # -configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') +config_loader = ConfigLoader.ConfigLoader() publisher.port = 6380 publisher.channel = "Script" -# notifications enabled/disabled -TrackedTermsNotificationEnabled_Name = "TrackedNotifications" - -# associated notification email addresses for a specific term` -# Keys will be e.g. TrackedNotificationEmails -TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_" - def sendEmailNotification(recipient, alert_name, content): - if not os.path.exists(configfile): - raise Exception('Unable to find the configuration file. \ - Did you set environment variables? \ - Or activate the virtualenv?') - - cfg = configparser.ConfigParser() - cfg.read(configfile) - - sender = cfg.get("Notifications", "sender") - sender_host = cfg.get("Notifications", "sender_host") - sender_port = cfg.getint("Notifications", "sender_port") - sender_pw = cfg.get("Notifications", "sender_pw") + sender = config_loader.get_config_str("Notifications", "sender") + sender_host = config_loader.get_config_str("Notifications", "sender_host") + sender_port = config_loader.get_config_int("Notifications", "sender_port") + sender_pw = config_loader.get_config_str("Notifications", "sender_pw") if sender_pw == 'None': sender_pw = None diff --git a/bin/PgpDump.py b/bin/PgpDump.py index 4b7ec629..a269734f 100755 --- a/bin/PgpDump.py +++ b/bin/PgpDump.py @@ -21,6 +21,8 @@ from bs4 import BeautifulSoup from Helper import Process from packages import Paste +from packages import Pgp + class TimeoutException(Exception): pass @@ -117,31 +119,6 @@ def extract_id_from_output(pgp_dump_outpout): key_id = key_id.replace(key_id_str, '', 1) set_key.add(key_id) -def save_pgp_data(type_pgp, date, item_path, data): - # create basic medata - if not serv_metadata.exists('pgpdump_metadata_{}:{}'.format(type_pgp, data)): - serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'first_seen', date) - serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date) - else: - last_seen = serv_metadata.hget('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen') - if not last_seen: - serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date) - else: - if int(last_seen) < int(date): - serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date) - - # global set - serv_metadata.sadd('set_pgpdump_{}:{}'.format(type_pgp, data), item_path) - - # daily - serv_metadata.hincrby('pgpdump:{}:{}'.format(type_pgp, date), data, 1) - - # all type - serv_metadata.zincrby('pgpdump_all:{}'.format(type_pgp), data, 1) - - # item_metadata - serv_metadata.sadd('item_pgpdump_{}:{}'.format(type_pgp, item_path), data) - if __name__ == '__main__': # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) @@ -236,12 +213,12 @@ if __name__ == '__main__': for key_id in set_key: print(key_id) - save_pgp_data('key', date, message, key_id) + Pgp.save_pgp_data('key', date, message, key_id) for name_id in set_name: print(name_id) - save_pgp_data('name', date, message, name_id) + Pgp.save_pgp_data('name', date, message, name_id) for mail_id in set_mail: print(mail_id) - save_pgp_data('mail', date, message, mail_id) + Pgp.save_pgp_data('mail', date, message, mail_id) diff --git a/bin/Queues_Monitoring.py b/bin/Queues_Monitoring.py deleted file mode 100755 index 3f0462ab..00000000 --- a/bin/Queues_Monitoring.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import redis -import argparse -import configparser -import time -import os -from pubsublogger import publisher -import texttable - - -def main(): - """Main Function""" - - # CONFIG # - cfg = configparser.ConfigParser() - cfg.read('./packages/config.cfg') - - # SCRIPT PARSER # - parser = argparse.ArgumentParser( - description='''This script is a part of the Assisted Information Leak framework.''', - epilog='''''') - - parser.add_argument('-db', type=int, default=0, - help='The name of the Redis DB (default 0)', - choices=[0, 1, 2, 3, 4], action='store') - - # REDIS # - r_serv = redis.StrictRedis( - host=cfg.get("Redis_Queues", "host"), - port=cfg.getint("Redis_Queues", "port"), - db=cfg.getint("Redis_Queues", "db"), - decode_responses=True) - - # LOGGING # - publisher.port = 6380 - publisher.channel = "Queuing" - - while True: - table = texttable.Texttable() - table.header(["Queue name", "#Items"]) - row = [] - for queue in r_serv.smembers("queues"): - current = r_serv.llen(queue) - current = current - r_serv.llen(queue) - row.append((queue, r_serv.llen(queue))) - - time.sleep(0.5) - row.sort() - table.add_rows(row, header=False) - os.system('clear') - print(table.draw()) - - -if __name__ == "__main__": - main() diff --git a/bin/RegexForTermsFrequency.py b/bin/RegexForTermsFrequency.py deleted file mode 100755 index cd8102c1..00000000 --- a/bin/RegexForTermsFrequency.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* -""" -This Module is used for term frequency. -It processes every paste coming from the global module and test the regexs -supplied in the term webpage. - -""" -import redis -import time -from pubsublogger import publisher -from packages import Paste -import calendar -import re -import signal -import time -from Helper import Process -# Email notifications -from NotificationHelper import * - - -class TimeoutException(Exception): - pass - - -def timeout_handler(signum, frame): - raise TimeoutException - -signal.signal(signal.SIGALRM, timeout_handler) - -# Config Variables -DICO_REFRESH_TIME = 60 # s - -BlackListTermsSet_Name = "BlackListSetTermSet" -TrackedTermsSet_Name = "TrackedSetTermSet" -TrackedRegexSet_Name = "TrackedRegexSet" - -top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set -oneDay = 60*60*24 -top_termFreq_setName_day = ["TopTermFreq_set_day_", 1] -top_termFreq_setName_week = ["TopTermFreq_set_week", 7] -top_termFreq_setName_month = ["TopTermFreq_set_month", 31] -top_termFreq_set_array = [top_termFreq_setName_day, top_termFreq_setName_week, top_termFreq_setName_month] - -TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_" - -# create direct link in mail -full_paste_url = "/showsavedpaste/?paste=" - - -def refresh_dicos(): - dico_regex = {} - dico_regexname_to_redis = {} - for regex_str in server_term.smembers(TrackedRegexSet_Name): - dico_regex[regex_str[1:-1]] = re.compile(regex_str[1:-1]) - dico_regexname_to_redis[regex_str[1:-1]] = regex_str - - return dico_regex, dico_regexname_to_redis - -if __name__ == "__main__": - publisher.port = 6380 - publisher.channel = "Script" - - config_section = 'RegexForTermsFrequency' - p = Process(config_section) - max_execution_time = p.config.getint(config_section, "max_execution_time") - - # REDIS # - server_term = redis.StrictRedis( - host=p.config.get("ARDB_TermFreq", "host"), - port=p.config.get("ARDB_TermFreq", "port"), - db=p.config.get("ARDB_TermFreq", "db"), - decode_responses=True) - - # FUNCTIONS # - publisher.info("RegexForTermsFrequency script started") - - # create direct link in mail - full_paste_url = p.config.get("Notifications", "ail_domain") + full_paste_url - - # compile the regex - dico_refresh_cooldown = time.time() - dico_regex, dico_regexname_to_redis = refresh_dicos() - - message = p.get_from_set() - - # Regex Frequency - while True: - - if message is not None: - if time.time() - dico_refresh_cooldown > DICO_REFRESH_TIME: - dico_refresh_cooldown = time.time() - dico_regex, dico_regexname_to_redis = refresh_dicos() - print('dico got refreshed') - - filename = message - temp = filename.split('/') - timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0)) - - curr_set = top_termFreq_setName_day[0] + str(timestamp) - paste = Paste.Paste(filename) - content = paste.get_p_content() - - # iterate the word with the regex - for regex_str, compiled_regex in dico_regex.items(): - - signal.alarm(max_execution_time) - try: - matched = compiled_regex.search(content) - except TimeoutException: - print ("{0} processing timeout".format(paste.p_rel_path)) - continue - else: - signal.alarm(0) - - if matched is not None: # there is a match - print('regex matched {}'.format(regex_str)) - matched = matched.group(0) - regex_str_complete = "/" + regex_str + "/" - # Add in Regex track set only if term is not in the blacklist - if regex_str_complete not in server_term.smembers(BlackListTermsSet_Name): - # Send a notification only when the member is in the set - if regex_str_complete in server_term.smembers(TrackedTermsNotificationEnabled_Name): - - # create mail body - mail_body = ("AIL Framework,\n" - "New occurrence for regex: " + regex_str + "\n" - ''+full_paste_url + filename) - - # Send to every associated email adress - for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + regex_str_complete): - sendEmailNotification(email, 'Term', mail_body) - - # tag paste - for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + regex_str_complete): - msg = '{};{}'.format(tag, filename) - p.populate_set_out(msg, 'Tags') - - set_name = 'regex_' + dico_regexname_to_redis[regex_str] - new_to_the_set = server_term.sadd(set_name, filename) - new_to_the_set = True if new_to_the_set == 1 else False - - # consider the num of occurence of this term - regex_value = int(server_term.hincrby(timestamp, dico_regexname_to_redis[regex_str], int(1))) - # 1 term per paste - if new_to_the_set: - regex_value_perPaste = int(server_term.hincrby("per_paste_" + str(timestamp), dico_regexname_to_redis[regex_str], int(1))) - server_term.zincrby("per_paste_" + curr_set, dico_regexname_to_redis[regex_str], float(1)) - server_term.zincrby(curr_set, dico_regexname_to_redis[regex_str], float(1)) - else: - pass - - else: - publisher.debug("Script RegexForTermsFrequency is Idling") - print("sleeping") - time.sleep(5) - message = p.get_from_set() diff --git a/bin/RegexTracker.py b/bin/RegexTracker.py new file mode 100755 index 00000000..2f7e5b9f --- /dev/null +++ b/bin/RegexTracker.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +This Module is used for regex tracking. +It processes every paste coming from the global module and test the regexs +supplied in the term webpage. + +""" +import os +import re +import sys +import time +import signal + +from Helper import Process +from pubsublogger import publisher + +import NotificationHelper + +from packages import Item +from packages import Term + +full_item_url = "/showsavedpaste/?paste=" +mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}" + +dict_regex_tracked = Term.get_regex_tracked_words_dict() +last_refresh = time.time() + +class TimeoutException(Exception): + pass +def timeout_handler(signum, frame): + raise TimeoutException +signal.signal(signal.SIGALRM, timeout_handler) + +def new_term_found(term, term_type, item_id, item_date): + uuid_list = Term.get_term_uuid_list(term, 'regex') + print('new tracked term found: {} in {}'.format(term, item_id)) + + for term_uuid in uuid_list: + Term.add_tracked_item(term_uuid, item_id, item_date) + + tags_to_add = Term.get_term_tags(term_uuid) + for tag in tags_to_add: + msg = '{};{}'.format(tag, item_id) + p.populate_set_out(msg, 'Tags') + + mail_to_notify = Term.get_term_mails(term_uuid) + if mail_to_notify: + mail_body = mail_body_template.format(term, item_id, full_item_url, item_id) + for mail in mail_to_notify: + NotificationHelper.sendEmailNotification(mail, 'Term Tracker', mail_body) + +if __name__ == "__main__": + publisher.port = 6380 + publisher.channel = "Script" + publisher.info("Script RegexTracker started") + + config_section = 'RegexTracker' + p = Process(config_section) + max_execution_time = p.config.getint(config_section, "max_execution_time") + + ull_item_url = p.config.get("Notifications", "ail_domain") + full_item_url + + # Regex Frequency + while True: + + item_id = p.get_from_set() + + if item_id is not None: + + item_date = Item.get_item_date(item_id) + item_content = Item.get_item_content(item_id) + + for regex in dict_regex_tracked: + + signal.alarm(max_execution_time) + try: + matched = dict_regex_tracked[regex].search(item_content) + except TimeoutException: + print ("{0} processing timeout".format(item_id)) + continue + else: + signal.alarm(0) + + if matched: + new_term_found(regex, 'regex', item_id, item_date) + + + else: + time.sleep(5) + + # refresh Tracked term + if last_refresh < Term.get_tracked_term_last_updated_by_type('regex'): + dict_regex_tracked = Term.get_regex_tracked_words_dict() + last_refresh = time.time() + print('Tracked set refreshed') diff --git a/bin/Repartition_graph.py b/bin/Repartition_graph.py deleted file mode 100755 index 5aa146a2..00000000 --- a/bin/Repartition_graph.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/python3 -# -*-coding:UTF-8 -* - -import redis -import argparse -import configparser -from datetime import datetime -from pubsublogger import publisher - -import matplotlib.pyplot as plt - - -def main(): - """Main Function""" - - # CONFIG # - cfg = configparser.ConfigParser() - cfg.read('./packages/config.cfg') - - # SCRIPT PARSER # - parser = argparse.ArgumentParser( - description='''This script is a part of the Analysis Information Leak framework.''', - epilog='''''') - - parser.add_argument('-f', type=str, metavar="filename", default="figure", - help='The absolute path name of the "figure.png"', - action='store') - parser.add_argument('-y', '--year', type=int, required=False, default=None, help='The date related to the DB') - - args = parser.parse_args() - - # REDIS # - # port generated automatically depending on the date - curYear = datetime.now().year if args.year is None else args.year - r_serv = redis.StrictRedis( - host=cfg.get("ARDB_Hashs", "host"), - port=cfg.getint("ARDB_Hashs", "port"), - db=curYear, - decode_responses=True) - - # LOGGING # - publisher.port = 6380 - publisher.channel = "Graph" - - # FUNCTIONS # - publisher.info("""Creating the Repartition Graph""") - - total_list = [] - codepad_list = [] - pastie_list = [] - pastebin_list = [] - for hash in r_serv.keys(): - total_list.append(r_serv.scard(hash)) - - code = 0 - pastie = 0 - pastebin = 0 - for paste in r_serv.smembers(hash): - source = paste.split("/")[5] - - if source == "codepad.org": - code = code + 1 - elif source == "pastie.org": - pastie = pastie + 1 - elif source == "pastebin.com": - pastebin = pastebin + 1 - - codepad_list.append(code) - pastie_list.append(pastie) - pastebin_list.append(pastebin) - - codepad_list.sort(reverse=True) - pastie_list.sort(reverse=True) - pastebin_list.sort(reverse=True) - - total_list.sort(reverse=True) - - plt.plot(codepad_list, 'b', label='Codepad.org') - plt.plot(pastebin_list, 'g', label='Pastebin.org') - plt.plot(pastie_list, 'y', label='Pastie.org') - plt.plot(total_list, 'r', label='Total') - - plt.xscale('log') - plt.xlabel('Hashs') - plt.ylabel('Occur[Hash]') - plt.title('Repartition') - plt.legend() - plt.grid() - plt.tight_layout() - - plt.savefig(args.f+".png", dpi=None, facecolor='w', edgecolor='b', - orientation='portrait', papertype=None, format="png", - transparent=False, bbox_inches=None, pad_inches=0.1, - frameon=True) - -if __name__ == "__main__": - main() diff --git a/bin/SQLInjectionDetection.py b/bin/SQLInjectionDetection.py index ffb21003..3d8761a3 100755 --- a/bin/SQLInjectionDetection.py +++ b/bin/SQLInjectionDetection.py @@ -14,7 +14,6 @@ It test different possibility to makes some sqlInjection. import time import datetime import redis -import string import urllib.request import re from pubsublogger import publisher @@ -22,131 +21,41 @@ from Helper import Process from packages import Paste from pyfaup.faup import Faup -# Config Var - -regex_injection = [] -word_injection = [] -word_injection_suspect = [] - -# Classic atome injection -regex_injection1 = "([[AND |OR ]+[\'|\"]?[0-9a-zA-Z]+[\'|\"]?=[\'|\"]?[0-9a-zA-Z]+[\'|\"]?])" -regex_injection.append(regex_injection1) - -# Time-based attack -regex_injection2 = ["SLEEP\([0-9]+", "BENCHMARK\([0-9]+", "WAIT FOR DELAY ", "WAITFOR DELAY"] -regex_injection2 = re.compile('|'.join(regex_injection2)) -regex_injection.append(regex_injection2) - -# Interesting keyword -word_injection1 = [" IF ", " ELSE ", " CASE ", " WHEN ", " END ", " UNION ", "SELECT ", " FROM ", " ORDER BY ", " WHERE ", " DELETE ", " DROP ", " UPDATE ", " EXEC "] -word_injection.append(word_injection1) - -# Database special keywords -word_injection2 = ["@@version", "POW(", "BITAND(", "SQUARE("] -word_injection.append(word_injection2) - -# Html keywords -word_injection3 = [" + - - - - {% include 'nav_bar.html' %} - -
-
- - {% include 'decoded/menu_sidebar.html' %} - -
- -
-
-

{{ hash }} :

-
    -
  • -
    -
    - - - - - - - - - - - - - - - - - - - -
    Estimated typeFirst_seenLast_seenSize (Kb)Nb seen
      {{ estimated_type }}{{ first_seen }}{{ last_seen }}{{ size }}{{ nb_seen_in_all_pastes }}
    -
    -
    -
    -
    -
    -
    • -
- - {% if vt_enabled %} - {% if not b64_vt %} - - - - {% else %} - VT Report - {% endif %} - - {% else %} - Virus Total submission is disabled - {% endif %} - - - - -
-
- -
-
- -
-
- Graph -
-
-
-
-
-
-
- -
- -
-
- Encoding -
-
- {% for encoding in list_hash_decoder %} - - {% endfor %} -
-
- -
-
- Graph -
-
- - - -
    -
    • -
  • -

    Double click on a node to open Hash/Paste

    - - Current Hash
    - - Hashes
    - - Pastes -

    -
    • -
  • - Hash Types: -
    • -
  • - Application
    - Audio
    - Image
    - Text
    - Other -
    • -
-
-
-
-
- -
-
- Graph -
-
-
-
-
-
- -
-
-
- - - - - - - - - - - - - - diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py index 12fe3177..3daa76b8 100644 --- a/var/www/modules/hiddenServices/Flask_hiddenServices.py +++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py @@ -13,7 +13,7 @@ import json from pyfaup.faup import Faup from flask import Flask, render_template, jsonify, request, send_file, Blueprint, redirect, url_for -from Role_Manager import login_admin, login_analyst +from Role_Manager import login_admin, login_analyst, no_cache from flask_login import login_required from Date import Date @@ -23,7 +23,6 @@ from HiddenServices import HiddenServices import Flask_config app = Flask_config.app -cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_cache = Flask_config.r_cache r_serv_onion = Flask_config.r_serv_onion @@ -731,94 +730,10 @@ def show_domains_by_daterange(): date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down, domains_tags=domains_tags, type=service_type, bootstrap_label=bootstrap_label) -@hiddenServices.route("/crawlers/show_domain", methods=['GET']) -@login_required -@login_analyst -def show_domain(): - domain = request.args.get('domain') - epoch = request.args.get('epoch') - try: - epoch = int(epoch) - except: - epoch = None - port = request.args.get('port') - faup.decode(domain) - unpack_url = faup.get() - - ## TODO: # FIXME: remove me - try: - domain = unpack_url['domain'].decode() - except: - domain = unpack_url['domain'] - - if not port: - if unpack_url['port']: - try: - port = unpack_url['port'].decode() - except: - port = unpack_url['port'] - else: - port = 80 - try: - port = int(port) - except: - port = 80 - type = get_type_domain(domain) - if domain is None or not r_serv_onion.exists('{}_metadata:{}'.format(type, domain)): - return '404' - # # TODO: FIXME return 404 - - last_check = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'last_check') - if last_check is None: - last_check = '********' - last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8]) - first_seen = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'first_seen') - if first_seen is None: - first_seen = '********' - first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8]) - ports = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'ports') - origin_paste = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'paste_parent') - - h = HiddenServices(domain, type, port=port) - item_core = h.get_domain_crawled_core_item(epoch=epoch) - if item_core: - l_pastes = h.get_last_crawled_pastes(item_root=item_core['root_item']) - else: - l_pastes = [] - dict_links = h.get_all_links(l_pastes) - if l_pastes: - status = True - else: - status = False - last_check = '{} - {}'.format(last_check, time.strftime('%H:%M.%S', time.gmtime(epoch))) - screenshot = h.get_domain_random_screenshot(l_pastes) - if screenshot: - screenshot = screenshot[0] - else: - screenshot = 'None' - - domain_tags = h.get_domain_tags() - - origin_paste_name = h.get_origin_paste_name() - origin_paste_tags = unpack_paste_tags(r_serv_metadata.smembers('tag:{}'.format(origin_paste))) - paste_tags = [] - for path in l_pastes: - p_tags = r_serv_metadata.smembers('tag:'+path) - paste_tags.append(unpack_paste_tags(p_tags)) - - domain_history = h.extract_epoch_from_history(h.get_domain_crawled_history()) - - return render_template("showDomain.html", domain=domain, last_check=last_check, first_seen=first_seen, - l_pastes=l_pastes, paste_tags=paste_tags, bootstrap_label=bootstrap_label, - dict_links=dict_links, port=port, epoch=epoch, - ports=ports, domain_history=domain_history, - origin_paste_tags=origin_paste_tags, status=status, - origin_paste=origin_paste, origin_paste_name=origin_paste_name, - domain_tags=domain_tags, screenshot=screenshot) - @hiddenServices.route("/crawlers/download_domain", methods=['GET']) @login_required @login_analyst +@no_cache def download_domain(): domain = request.args.get('domain') epoch = request.args.get('epoch') diff --git a/var/www/modules/hiddenServices/templates/Crawler_Splash_last_by_type.html b/var/www/modules/hiddenServices/templates/Crawler_Splash_last_by_type.html index bafe3ecf..f793b3a9 100644 --- a/var/www/modules/hiddenServices/templates/Crawler_Splash_last_by_type.html +++ b/var/www/modules/hiddenServices/templates/Crawler_Splash_last_by_type.html @@ -67,7 +67,7 @@ title="{{metadata_domain['domain']}}" data-content="port: {{metadata_domain['port']}}
epoch: {{metadata_domain['epoch']}}"> - {{ metadata_domain['domain_name'] }} + {{ metadata_domain['domain_name'] }} {{'{}/{}/{}'.format(metadata_domain['first_seen'][0:4], metadata_domain['first_seen'][4:6], metadata_domain['first_seen'][6:8])}} {{'{}/{}/{}'.format(metadata_domain['last_check'][0:4], metadata_domain['last_check'][4:6], metadata_domain['last_check'][6:8])}}
@@ -205,7 +205,7 @@ function refresh_list_crawled(){ var newRow = tableRef.insertRow(tableRef.rows.length); var newCell = newRow.insertCell(0); - newCell.innerHTML = ""+data_domain['domain']+""; + newCell.innerHTML = ""+data_domain['domain']+""; newCell = newRow.insertCell(1); newCell.innerHTML = ""+data_domain['first_seen'].substr(0, 4)+"/"+data_domain['first_seen'].substr(4, 2)+"/"+data_domain['first_seen'].substr(6, 2)+"" @@ -243,7 +243,7 @@ function refresh_list_crawled(){ newCell.innerHTML = ""+crawler['crawler_info']+""; newCell = newRow.insertCell(1); - newCell.innerHTML = ""+crawler['crawling_domain']+""; + newCell.innerHTML = ""+crawler['crawling_domain']+""; newCell = newRow.insertCell(2); newCell.innerHTML = "
"+crawler['status_info']+"
"; diff --git a/var/www/modules/hiddenServices/templates/Crawler_auto.html b/var/www/modules/hiddenServices/templates/Crawler_auto.html index 87dd7569..87b74c36 100644 --- a/var/www/modules/hiddenServices/templates/Crawler_auto.html +++ b/var/www/modules/hiddenServices/templates/Crawler_auto.html @@ -45,7 +45,7 @@ {% for metadata_domain in last_domains %} - {{ metadata_domain['domain_name'] }} + {{ metadata_domain['domain_name'] }} {{'{}/{}/{}'.format(metadata_domain['first_seen'][0:4], metadata_domain['first_seen'][4:6], metadata_domain['first_seen'][6:8])}} {{'{}/{}/{}'.format(metadata_domain['last_check'][0:4], metadata_domain['last_check'][4:6], metadata_domain['last_check'][6:8])}}
@@ -76,7 +76,7 @@ {% for metadata_domain in auto_crawler_domain_onions_metadata %} - {{ metadata_domain['url'] }} + {{ metadata_domain['url'] }} @@ -110,7 +110,7 @@ {% for metadata_domain in auto_crawler_domain_regular_metadata %} - {{ metadata_domain['url'] }} + {{ metadata_domain['url'] }} diff --git a/var/www/modules/hiddenServices/templates/domains.html b/var/www/modules/hiddenServices/templates/domains.html index c816ae17..5854a8f4 100644 --- a/var/www/modules/hiddenServices/templates/domains.html +++ b/var/www/modules/hiddenServices/templates/domains.html @@ -64,7 +64,7 @@ {% for domain in domains_by_day[date] %} - {{ domain }} + {{ domain }}
{% for tag in domain_metadata[domain]['tags'] %} diff --git a/var/www/modules/hiddenServices/templates/showDomain.html b/var/www/modules/hiddenServices/templates/showDomain.html deleted file mode 100644 index 4230ec5a..00000000 --- a/var/www/modules/hiddenServices/templates/showDomain.html +++ /dev/null @@ -1,268 +0,0 @@ - - - - Show Domain - AIL - - - - - - - - - - - - - - - {% include 'nav_bar.html' %} - -
-
- - {% include 'crawler/menu_sidebar.html' %} - -
- - - -
- -
-
-
- - {% if status %} -
- - UP -
- {% else %} -
- - DOWN -
- {% endif %} - -

{{ domain }} :

-
-
- - - - - - - - - - - - - - - -
First SeenLast CheckPorts
{{ first_seen }}{{ last_check }}{{ ports }}
- - Origin Paste: - {% if origin_paste_name=='manual' or origin_paste_name=='auto' %} - {{ origin_paste_name }} - {%else%} - {{ origin_paste_name }} - {%endif%} -
- {% for tag in origin_paste_tags %} - - {{ tag[0] }} - - {% endfor %} -
-
- -
-
- -
- {% for tag in domain_tags %} - - {{ tag }} {{ domain_tags[tag] }} - - {% endfor %} -
-
-
- - {% if l_pastes %} - - - - - - - - {% for path in l_pastes %} - - - - {% endfor %} - -
Crawled Pastes
- -
{{ dict_links[path] }}
- -
- {% for tag in paste_tags[loop.index0] %} - - {{ tag[0] }} - - {% endfor %} -
-
- {%endif%} - - -
- - - -
- - {% if domain_history %} - - - - - - - - {% for epoch_item in domain_history %} - - - {% endfor %} - -
Domain History
- -
-
{{domain}}
- {% if epoch_item[2] %} -
UP
- {% else %} -
DOWN
- {% endif %} -
{{ epoch_item[1] }}
-
- -
- {%endif%} - -
- -
-
-
-
-
- -
-
- -
-
-
-
- -
- - -
{{dict_links[screenshot['item']]}}
- - -
-
- -
-
-
-
- - - - - - - - - diff --git a/var/www/modules/hunter/Flask_hunter.py b/var/www/modules/hunter/Flask_hunter.py new file mode 100644 index 00000000..e9028b36 --- /dev/null +++ b/var/www/modules/hunter/Flask_hunter.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Flask functions and routes for tracked items +''' +import json +import redis +import datetime +import calendar +import flask +from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, Response, escape + +from Role_Manager import login_admin, login_analyst +from flask_login import login_required, current_user + +import re +from pprint import pprint +import Levenshtein + +# --------------------------------------------------------------- + +import Paste +import Term + +# ============ VARIABLES ============ +import Flask_config + +app = Flask_config.app +baseUrl = Flask_config.baseUrl +r_serv_term = Flask_config.r_serv_term +r_serv_cred = Flask_config.r_serv_cred +r_serv_db = Flask_config.r_serv_db +bootstrap_label = Flask_config.bootstrap_label + +hunter = Blueprint('hunter', __name__, template_folder='templates') + +# ============ FUNCTIONS ============ + +# ============ ROUTES ============ + +@hunter.route("/trackers") +@login_required +@login_analyst +def tracked_menu(): + user_id = current_user.get_id() + user_term = Term.get_all_user_tracked_terms(user_id) + global_term = Term.get_all_global_tracked_terms() + return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label) + +@hunter.route("/trackers/word") +@login_required +@login_analyst +def tracked_menu_word(): + filter_type = 'word' + user_id = current_user.get_id() + user_term = Term.get_all_user_tracked_terms(user_id, filter_type='word') + global_term = Term.get_all_global_tracked_terms(filter_type='word') + return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) + +@hunter.route("/trackers/set") +@login_required +@login_analyst +def tracked_menu_set(): + filter_type = 'set' + user_id = current_user.get_id() + user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type) + global_term = Term.get_all_global_tracked_terms(filter_type=filter_type) + return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) + +@hunter.route("/trackers/regex") +@login_required +@login_analyst +def tracked_menu_regex(): + filter_type = 'regex' + user_id = current_user.get_id() + user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type) + global_term = Term.get_all_global_tracked_terms(filter_type=filter_type) + return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) + + +@hunter.route("/tracker/add", methods=['GET', 'POST']) +@login_required +@login_analyst +def add_tracked_menu(): + if request.method == 'POST': + term = request.form.get("term") + term_type = request.form.get("tracker_type") + nb_words = request.form.get("nb_word", 1) + description = request.form.get("description", '') + level = request.form.get("level", 0) + tags = request.form.get("tags", []) + mails = request.form.get("mails", []) + + if level == 'on': + level = 1 + + if mails: + mails = mails.split() + if tags: + tags = tags.split() + + input_dict = {"term": term, "type": term_type, "nb_words": nb_words, "tags": tags, "mails": mails, "level": level, "description": description} + user_id = current_user.get_id() + res = Term.parse_json_term_to_add(input_dict, user_id) + if res[1] == 200: + return redirect(url_for('hunter.tracked_menu')) + else: + ## TODO: use modal + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + else: + return render_template("Add_tracker.html") + +@hunter.route("/tracker/show_tracker") +@login_required +@login_analyst +def show_tracker(): + user_id = current_user.get_id() + term_uuid = request.args.get('uuid', None) + res = Term.check_term_uuid_valid_access(term_uuid, user_id) + if res: # invalid access + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + + if date_from: + date_from = date_from.replace('-', '') + if date_to: + date_to = date_to.replace('-', '') + + tracker_metadata = Term.get_term_metedata(term_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sparkline=True) + + if date_from: + res = Term.parse_get_tracker_term_item({'uuid': term_uuid, 'date_from': date_from, 'date_to': date_to}, user_id) + if res[1] !=200: + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + tracker_metadata['items'] = res[0]['items'] + tracker_metadata['date_from'] = res[0]['date_from'] + tracker_metadata['date_to'] = res[0]['date_to'] + else: + tracker_metadata['items'] = [] + tracker_metadata['date_from'] = '' + tracker_metadata['date_to'] = '' + + return render_template("showTracker.html", tracker_metadata=tracker_metadata, bootstrap_label=bootstrap_label) + +@hunter.route("/tracker/update_tracker_description", methods=['POST']) +@login_required +@login_analyst +def update_tracker_description(): + user_id = current_user.get_id() + term_uuid = request.form.get('uuid') + res = Term.check_term_uuid_valid_access(term_uuid, user_id) + if res: # invalid access + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + description = escape( str(request.form.get('description', '')) ) + Term.replace_tracker_description(term_uuid, description) + return redirect(url_for('hunter.show_tracker', uuid=term_uuid)) + +@hunter.route("/tracker/update_tracker_tags", methods=['POST']) +@login_required +@login_analyst +def update_tracker_tags(): + user_id = current_user.get_id() + term_uuid = request.form.get('uuid') + res = Term.check_term_uuid_valid_access(term_uuid, user_id) + if res: # invalid access + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + tags = request.form.get('tags') + if tags: + tags = tags.split() + else: + tags = [] + Term.replace_tracked_term_tags(term_uuid, tags) + return redirect(url_for('hunter.show_tracker', uuid=term_uuid)) + +@hunter.route("/tracker/update_tracker_mails", methods=['POST']) +@login_required +@login_analyst +def update_tracker_mails(): + user_id = current_user.get_id() + term_uuid = request.form.get('uuid') + res = Term.check_term_uuid_valid_access(term_uuid, user_id) + if res: # invalid access + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + mails = request.form.get('mails') + if mails: + mails = mails.split() + else: + mails = [] + res = Term.replace_tracked_term_mails(term_uuid, mails) + if res: # invalid mail + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + return redirect(url_for('hunter.show_tracker', uuid=term_uuid)) + +@hunter.route("/tracker/delete", methods=['GET']) +@login_required +@login_analyst +def delete_tracker(): + user_id = current_user.get_id() + term_uuid = request.args.get('uuid') + res = Term.parse_tracked_term_to_delete({'uuid': term_uuid}, user_id) + if res[1] !=200: + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + return redirect(url_for('hunter.tracked_menu')) + +@hunter.route("/tracker/get_json_tracker_stats", methods=['GET']) +@login_required +@login_analyst +def get_json_tracker_stats(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + + if date_from: + date_from = date_from.replace('-', '') + if date_to: + date_to = date_to.replace('-', '') + + tracker_uuid = request.args.get('uuid') + + if date_from and date_to: + res = Term.get_list_tracked_term_stats_by_day([tracker_uuid], date_from=date_from, date_to=date_to) + else: + res = Term.get_list_tracked_term_stats_by_day([tracker_uuid]) + return jsonify(res) + +# ========= REGISTRATION ========= +app.register_blueprint(hunter, url_prefix=baseUrl) diff --git a/var/www/modules/hunter/templates/Add_tracker.html b/var/www/modules/hunter/templates/Add_tracker.html new file mode 100644 index 00000000..08613209 --- /dev/null +++ b/var/www/modules/hunter/templates/Add_tracker.html @@ -0,0 +1,159 @@ + + + + + AIL-Framework + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'hunter/menu_sidebar.html' %} + +
+ +
+
+
Create a new tracker
+
+
+

Enter a domain and choose what kind of data you want.

+ +
+ +
+
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+
+ + +
+
+
+ +
+ + + +

Terms to track (space separated)

+ +
+
+ +
+
+ +
+
+ + +
+ + +
+ + + + + +
+
+ + +
+ +
+
+ + + + + diff --git a/var/www/modules/hunter/templates/showTracker.html b/var/www/modules/hunter/templates/showTracker.html new file mode 100644 index 00000000..7c149205 --- /dev/null +++ b/var/www/modules/hunter/templates/showTracker.html @@ -0,0 +1,339 @@ + + + + + + + AIL Framework - AIL + + + + + + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'hunter/menu_sidebar.html' %} + +
+ +
+
+

{{ tracker_metadata['uuid'] }}

+
+ {%if tracker_metadata['description']%} + {{ tracker_metadata['description'] }} + {%endif%} + +
+
    +
  • +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    TypeTrackerDate addedLevelCreated byFirst seenLast seenTags Email
    {{ tracker_metadata['type'] }}{{ tracker_metadata['term'] }}{{ tracker_metadata['date'][0:4] }}/{{ tracker_metadata['date'][4:6] }}/{{ tracker_metadata['date'][6:8] }}{{ tracker_metadata['level'] }}{{ tracker_metadata['user_id'] }} + {% if tracker_metadata['first_seen'] %} + {{ tracker_metadata['first_seen'][0:4] }}/{{ tracker_metadata['first_seen'][4:6] }}/{{ tracker_metadata['first_seen'][6:8] }} + {% endif %} + + {% if tracker_metadata['last_seen'] %} + {{ tracker_metadata['last_seen'][0:4] }}/{{ tracker_metadata['last_seen'][4:6] }}/{{ tracker_metadata['last_seen'][6:8] }} + {% endif %} + + {% for tag in tracker_metadata['tags'] %} + + {{ tag }} + + {% endfor %} + + + {% for mail in tracker_metadata['mails'] %} + {{ mail }}
    + {% endfor %} +
    +
    +
    +
    +
    +
    +
    • +
+ +
+
+ +
Update this tracker description:
+
+
+
+
+ +
+ + +
+ +
+ +
+
+ +
All Tags added for this tracker, space separated:
+
+
+
+
+ +
+ + +
+ +
+ +
+
+ +
All E-Mails to Notify for this tracker, space separated:
+
+
+
+
+ +
+ + +
+ +
+ + + + +
+
+ +
+ +
+
+ +
+
+
+
+ +
+
+
+
+
+ +
+
+
+ + + +
+
+ + {%if tracker_metadata['items']%} +
+ + + + + + + + + + {% for item in tracker_metadata['items'] %} + + + + + {% endfor %} + + +
DateItem Id
+ {{item['date'][0:4]}}/{{item['date'][4:6]}}/{{item['date'][6:8]}} + + +
{{ item['id'] }}
+ +
+ {% for tag in item['tags'] %} + + {{ tag }} + + {% endfor %} +
+
+
+ {% endif %} + +
+
+
+ + + + + + + diff --git a/var/www/modules/hunter/templates/trackersManagement.html b/var/www/modules/hunter/templates/trackersManagement.html new file mode 100644 index 00000000..63b7758e --- /dev/null +++ b/var/www/modules/hunter/templates/trackersManagement.html @@ -0,0 +1,206 @@ + + + + + + + + Tracker Management + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'hunter/menu_sidebar.html' %} + +
+ +
+
+
Your {{filter_type}} Trackers
+
+
+ + + + + + + + + + + + + {% for dict_uuid in user_term %} + + + + + + + + + {% endfor %} + +
TypeTrackerFirst seenLast seenEmail notificationsparkline
{{dict_uuid['type']}} + {{dict_uuid['term']}} +
+ {% for tag in dict_uuid['tags'] %} + + {{ tag }} + + {% endfor %} +
+ 		+ {% if dict_uuid['first_seen'] %} + {{dict_uuid['first_seen'][0:4]}}/{{dict_uuid['first_seen'][4:6]}}/{{dict_uuid['first_seen'][6:8]}} + {% endif %} + + {% if dict_uuid['last_seen'] %} + {{dict_uuid['last_seen'][0:4]}}/{{dict_uuid['last_seen'][4:6]}}/{{dict_uuid['last_seen'][6:8]}} + {% endif %} + + {% for mail in dict_uuid['mails'] %} + {{ mail }}
+ {% endfor %} +
+
+
+ +
+
+
Global {{filter_type}} Trackers
+
+
+ + + + + + + + + + + + + {% for dict_uuid in global_term %} + + + + + + + + + {% endfor %} + +
TypeTrackerFirst seenLast seenEmail notificationsparkline
{{dict_uuid['type']}} + {{dict_uuid['term']}} +
+ {% for tag in dict_uuid['tags'] %} + + {{ tag }} + + {% endfor %} +
+ 		+ {% if dict_uuid['first_seen'] %} + {{dict_uuid['first_seen'][0:4]}}/{{dict_uuid['first_seen'][4:6]}}/{{dict_uuid['first_seen'][6:8]}} + {% endif %} + + {% if dict_uuid['last_seen'] %} + {{dict_uuid['last_seen'][0:4]}}/{{dict_uuid['last_seen'][4:6]}}/{{dict_uuid['last_seen'][6:8]}} + {% endif %} + + {% for mail in dict_uuid['mails'] %} + {{ mail }}
+ {% endfor %} +
+
+
+ + + + + + Create New Tracker + + +
+
+
+ + + + + diff --git a/var/www/modules/rawSkeleton/Flask_rawSkeleton.py b/var/www/modules/rawSkeleton/Flask_rawSkeleton.py index d767a83c..dca8f331 100644 --- a/var/www/modules/rawSkeleton/Flask_rawSkeleton.py +++ b/var/www/modules/rawSkeleton/Flask_rawSkeleton.py @@ -14,7 +14,6 @@ from flask_login import login_required import Flask_config app = Flask_config.app -cfg = Flask_config.cfg rawSkeleton = Blueprint('rawSkeleton', __name__, template_folder='templates') diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index 4535da28..cbd93dd6 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -8,10 +8,19 @@ import os import re import sys +import uuid import json import redis import datetime +import Import_helper +import Cryptocurrency +import Pgp +import Item +import Paste +import Tag +import Term + from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response from flask_login import login_required @@ -20,14 +29,15 @@ from functools import wraps # ============ VARIABLES ============ import Flask_config + app = Flask_config.app -cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_cache = Flask_config.r_cache r_serv_db = Flask_config.r_serv_db r_serv_onion = Flask_config.r_serv_onion r_serv_metadata = Flask_config.r_serv_metadata + restApi = Blueprint('restApi', __name__, template_folder='templates') # ============ AUTH FUNCTIONS ============ @@ -36,7 +46,7 @@ def check_token_format(strg, search=re.compile(r'[^a-zA-Z0-9_-]').search): return not bool(search(strg)) def verify_token(token): - if len(token) != 55: + if len(token) != 41: return False if not check_token_format(token): @@ -47,23 +57,44 @@ def verify_token(token): else: return False +def get_user_from_token(token): + return r_serv_db.hget('user:tokens', token) + +def verify_user_role(role, token): + user_id = get_user_from_token(token) + if user_id: + if is_in_role(user_id, role): + return True + else: + return False + else: + return False + +def is_in_role(user_id, role): + if r_serv_db.sismember('user_role:{}'.format(role), user_id): + return True + else: + return False + # ============ DECORATOR ============ -def token_required(funct): - @wraps(funct) - def api_token(*args, **kwargs): - data = authErrors() - if data: - return Response(json.dumps(data[0], indent=2, sort_keys=True), mimetype='application/json'), data[1] - else: - return funct(*args, **kwargs) - return api_token +def token_required(user_role): + def actual_decorator(funct): + @wraps(funct) + def api_token(*args, **kwargs): + data = authErrors(user_role) + if data: + return Response(json.dumps(data[0], indent=2, sort_keys=True), mimetype='application/json'), data[1] + else: + return funct(*args, **kwargs) + return api_token + return actual_decorator def get_auth_from_header(): token = request.headers.get('Authorization').replace(' ', '') # remove space return token -def authErrors(): +def authErrors(user_role): # Check auth if not request.headers.get('Authorization'): return ({'status': 'error', 'reason': 'Authentication needed'}, 401) @@ -71,12 +102,27 @@ def authErrors(): data = None # verify token format + # brute force protection + current_ip = request.remote_addr + login_failed_ip = r_cache.get('failed_login_ip_api:{}'.format(current_ip)) + # brute force by ip + if login_failed_ip: + login_failed_ip = int(login_failed_ip) + if login_failed_ip >= 5: + return ({'status': 'error', 'reason': 'Max Connection Attempts reached, Please wait {}s'.format(r_cache.ttl('failed_login_ip_api:{}'.format(current_ip)))}, 401) + try: authenticated = False if verify_token(token): authenticated = True + # check user role + if not verify_user_role(user_role, token): + data = ({'status': 'error', 'reason': 'Access Forbidden'}, 403) + if not authenticated: + r_cache.incr('failed_login_ip_api:{}'.format(current_ip)) + r_cache.expire('failed_login_ip_api:{}'.format(current_ip), 300) data = ({'status': 'error', 'reason': 'Authentication failed'}, 401) except Exception as e: print(e) @@ -86,8 +132,18 @@ def authErrors(): else: return None +# ============ API CORE ============= + # ============ FUNCTIONS ============ +def is_valid_uuid_v4(header_uuid): + try: + header_uuid=header_uuid.replace('-', '') + uuid_test = uuid.UUID(hex=header_uuid, version=4) + return uuid_test.hex == header_uuid + except: + return False + def one(): return 1 @@ -98,12 +154,391 @@ def one(): # def api(): # return 'api doc' -@restApi.route("api/items", methods=['POST']) -@token_required -def items(): - item = request.args.get('id') +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# POST +# +# { +# "id": item_id, mandatory +# "content": true, +# "tags": true, +# +# +# } +# +# response: { +# "id": "item_id", +# "tags": [], +# } +# +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@restApi.route("api/v1/get/item", methods=['POST']) +@token_required('analyst') +def get_item_id(): + data = request.get_json() + res = Item.get_item(data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - return Response(json.dumps({'test': 2}), mimetype='application/json') +@restApi.route("api/v1/get/item/default", methods=['POST']) +@token_required('analyst') +def get_item_id_basic(): + + data = request.get_json() + item_id = data.get('id', None) + req_data = {'id': item_id, 'date': True, 'content': True, 'tags': True} + res = Item.get_item(req_data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# GET +# +# { +# "id": item_id, mandatory +# } +# +# response: { +# "id": "item_id", +# "tags": [], +# } +# +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@restApi.route("api/v1/get/item/tag", methods=['POST']) +@token_required('analyst') +def get_item_tag(): + + data = request.get_json() + item_id = data.get('id', None) + req_data = {'id': item_id, 'date': False, 'tags': True} + res = Item.get_item(req_data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# POST +# +# { +# "id": item_id, mandatory +# "tags": [tags to add], +# "galaxy": [galaxy to add], +# } +# +# response: { +# "id": "item_id", +# "tags": [tags added], +# } +# +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@restApi.route("api/v1/add/item/tag", methods=['POST']) +@token_required('analyst') +def add_item_tags(): + + data = request.get_json() + if not data: + return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400 + + item_id = data.get('id', None) + tags = data.get('tags', []) + galaxy = data.get('galaxy', []) + + res = Tag.add_items_tag(tags, galaxy, item_id) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# DELETE +# +# { +# "id": item_id, mandatory +# "tags": [tags to delete], +# } +# +# response: { +# "id": "item_id", +# "tags": [tags deleted], +# } +# +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@restApi.route("api/v1/delete/item/tag", methods=['DELETE']) +@token_required('analyst') +def delete_item_tags(): + + data = request.get_json() + if not data: + return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400 + + item_id = data.get('id', None) + tags = data.get('tags', []) + + res = Tag.remove_item_tags(tags, item_id) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# GET +# +# { +# "id": item_id, mandatory +# } +# +# response: { +# "id": "item_id", +# "content": "item content" +# } +# +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@restApi.route("api/v1/get/item/content", methods=['POST']) +@token_required('analyst') +def get_item_content(): + + data = request.get_json() + item_id = data.get('id', None) + req_data = {'id': item_id, 'date': False, 'content': True, 'tags': False} + res = Item.get_item(req_data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + +@restApi.route("api/v1/get/tag/metadata", methods=['POST']) +@token_required('analyst') +def get_tag_metadata(): + data = request.get_json() + tag = data.get('tag', None) + if not Tag.is_tag_in_all_tag(tag): + return Response(json.dumps({'status': 'error', 'reason':'Tag not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404 + metadata = Tag.get_tag_metadata(tag) + return Response(json.dumps(metadata, indent=2, sort_keys=True), mimetype='application/json'), 200 + +@restApi.route("api/v1/get/tag/all", methods=['GET']) +@token_required('analyst') +def get_all_tags(): + res = {'tags': Tag.get_all_tags()} + return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200 + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # TRACKER # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@restApi.route("api/v1/add/tracker", methods=['POST']) +@token_required('analyst') +def add_tracker_term(): + data = request.get_json() + user_token = get_auth_from_header() + user_id = get_user_from_token(user_token) + res = Term.parse_json_term_to_add(data, user_id) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/delete/tracker", methods=['DELETE']) +@token_required('analyst') +def delete_tracker_term(): + data = request.get_json() + user_token = get_auth_from_header() + user_id = get_user_from_token(user_token) + res = Term.parse_tracked_term_to_delete(data, user_id) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/tracker/item", methods=['POST']) +@token_required('analyst') +def get_tracker_term_item(): + data = request.get_json() + user_token = get_auth_from_header() + user_id = get_user_from_token(user_token) + res = Term.parse_get_tracker_term_item(data, user_id) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@restApi.route("api/v1/get/cryptocurrency/bitcoin/metadata", methods=['POST']) +@token_required('analyst') +def get_cryptocurrency_bitcoin_metadata(): + data = request.get_json() + crypto_address = data.get('bitcoin', None) + req_data = {'bitcoin': crypto_address, 'metadata': True} + res = Cryptocurrency.get_cryptocurrency(req_data, 'bitcoin') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/cryptocurrency/bitcoin/item", methods=['POST']) +@token_required('analyst') +def get_cryptocurrency_bitcoin_item(): + data = request.get_json() + bitcoin_address = data.get('bitcoin', None) + req_data = {'bitcoin': bitcoin_address, 'items': True} + res = Cryptocurrency.get_cryptocurrency(req_data, 'bitcoin') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # PGP # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@restApi.route("api/v1/get/pgp/key/metadata", methods=['POST']) +@token_required('analyst') +def get_pgp_key_metadata(): + data = request.get_json() + pgp_field = data.get('key', None) + req_data = {'key': pgp_field, 'metadata': True} + res = Pgp.get_pgp(req_data, 'key') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/pgp/mail/metadata", methods=['POST']) +@token_required('analyst') +def get_pgp_mail_metadata(): + data = request.get_json() + pgp_field = data.get('mail', None) + req_data = {'mail': pgp_field, 'metadata': True} + res = Pgp.get_pgp(req_data, 'mail') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/pgp/name/metadata", methods=['POST']) +@token_required('analyst') +def get_pgp_name_metadata(): + data = request.get_json() + pgp_field = data.get('name', None) + req_data = {'name': pgp_field, 'metadata': True} + res = Pgp.get_pgp(req_data, 'name') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/pgp/key/item", methods=['POST']) +@token_required('analyst') +def get_pgp_key_item(): + data = request.get_json() + pgp_field = data.get('key', None) + req_data = {'key': pgp_field, 'items': True} + res = Pgp.get_pgp(req_data, 'key') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/pgp/mail/item", methods=['POST']) +@token_required('analyst') +def get_pgp_mail_item(): + data = request.get_json() + pgp_mail = data.get('mail', None) + req_data = {'mail': pgp_mail, 'items': True} + res = Pgp.get_pgp(req_data, 'mail') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/pgp/name/item", methods=['POST']) +@token_required('analyst') +def get_pgp_name_item(): + data = request.get_json() + pgp_name = data.get('name', None) + req_data = {'name': pgp_name, 'items': True} + res = Pgp.get_pgp(req_data, 'name') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +''' + + + +@restApi.route("api/v1/get/item/cryptocurrency/key", methods=['POST']) +@token_required('analyst') +def get_item_cryptocurrency_bitcoin(): + data = request.get_json() + item_id = data.get('id', None) + req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'key': True}} + res = Item.get_item(req_data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/item/pgp/mail", methods=['POST']) +@token_required('analyst') +def get_item_cryptocurrency_bitcoin(): + data = request.get_json() + item_id = data.get('id', None) + req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'mail': True}} + res = Item.get_item(req_data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/item/pgp/name", methods=['POST']) +@token_required('analyst') +def get_item_cryptocurrency_bitcoin(): + data = request.get_json() + item_id = data.get('id', None) + req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'name': True}} + res = Item.get_item(req_data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] +''' + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + + + + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# +# POST JSON FORMAT +# +# { +# "type": "text", (default value) +# "tags": [], (default value) +# "default_tags": True, (default value) +# "galaxy" [], (default value) +# "text": "", mandatory if type = text +# } +# +# response: {"uuid": "uuid"} +# +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@restApi.route("api/v1/import/item", methods=['POST']) +@token_required('analyst') +def import_item(): + + data = request.get_json() + if not data: + return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400 + + # unpack json + text_to_import = data.get('text', None) + if not text_to_import: + return Response(json.dumps({'status': 'error', 'reason': 'No text supplied'}, indent=2, sort_keys=True), mimetype='application/json'), 400 + + tags = data.get('tags', []) + if not type(tags) is list: + tags = [] + galaxy = data.get('galaxy', []) + if not type(galaxy) is list: + galaxy = [] + + if not Tag.is_valid_tags_taxonomies_galaxy(tags, galaxy): + return Response(json.dumps({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, indent=2, sort_keys=True), mimetype='application/json'), 400 + + default_tags = data.get('default_tags', True) + if default_tags: + tags.append('infoleak:submission="manual"') + + if sys.getsizeof(text_to_import) > 900000: + return Response(json.dumps({'status': 'error', 'reason': 'Size exceeds default'}, indent=2, sort_keys=True), mimetype='application/json'), 413 + + UUID = str(uuid.uuid4()) + Import_helper.create_import_queue(tags, galaxy, text_to_import, UUID) + + return Response(json.dumps({'uuid': UUID}, indent=2, sort_keys=True), mimetype='application/json') + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# GET +# +# { +# "uuid": "uuid", mandatory +# } +# +# response: { +# "status": "in queue"/"in progress"/"imported", +# "items": [all item id] +# } +# +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@restApi.route("api/v1/get/import/item", methods=['POST']) +@token_required('analyst') +def import_item_uuid(): + data = request.get_json() + UUID = data.get('uuid', None) + + # Verify uuid + if not is_valid_uuid_v4(UUID): + return Response(json.dumps({'status': 'error', 'reason': 'Invalid uuid'}), mimetype='application/json'), 400 + + data = Import_helper.check_import_status(UUID) + if data: + return Response(json.dumps(data[0]), mimetype='application/json'), data[1] + + return Response(json.dumps({'status': 'error', 'reason': 'Invalid response'}), mimetype='application/json'), 400 # ========= REGISTRATION ========= app.register_blueprint(restApi, url_prefix=baseUrl) diff --git a/var/www/modules/search/Flask_search.py b/var/www/modules/search/Flask_search.py index 67a518fb..ff5395e6 100644 --- a/var/www/modules/search/Flask_search.py +++ b/var/www/modules/search/Flask_search.py @@ -25,7 +25,7 @@ import time import Flask_config app = Flask_config.app -cfg = Flask_config.cfg +config_loader = Flask_config.config_loader baseUrl = Flask_config.baseUrl r_serv_pasteName = Flask_config.r_serv_pasteName r_serv_metadata = Flask_config.r_serv_metadata @@ -34,9 +34,8 @@ max_preview_modal = Flask_config.max_preview_modal bootstrap_label = Flask_config.bootstrap_label PASTES_FOLDER = Flask_config.PASTES_FOLDER -baseindexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path")) -indexRegister_path = os.path.join(os.environ['AIL_HOME'], - cfg.get("Indexer", "register")) +baseindexpath = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Indexer", "path")) +indexRegister_path = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Indexer", "register")) searches = Blueprint('searches', __name__, template_folder='templates') diff --git a/var/www/modules/sentiment/Flask_sentiment.py b/var/www/modules/sentiment/Flask_sentiment.py index af6c220c..895bd0ee 100644 --- a/var/www/modules/sentiment/Flask_sentiment.py +++ b/var/www/modules/sentiment/Flask_sentiment.py @@ -20,7 +20,6 @@ import Paste import Flask_config app = Flask_config.app -cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_charts = Flask_config.r_serv_charts r_serv_sentiment = Flask_config.r_serv_sentiment diff --git a/var/www/modules/settings/Flask_settings.py b/var/www/modules/settings/Flask_settings.py index a569cbbb..0ad1f43c 100644 --- a/var/www/modules/settings/Flask_settings.py +++ b/var/www/modules/settings/Flask_settings.py @@ -19,7 +19,6 @@ import git_status import Flask_config app = Flask_config.app -cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_db = Flask_config.r_serv_db max_preview_char = Flask_config.max_preview_char diff --git a/var/www/modules/showpaste/Flask_showpaste.py b/var/www/modules/showpaste/Flask_showpaste.py index fb990bbf..a972a346 100644 --- a/var/www/modules/showpaste/Flask_showpaste.py +++ b/var/www/modules/showpaste/Flask_showpaste.py @@ -10,7 +10,7 @@ import os import flask from flask import Flask, render_template, jsonify, request, Blueprint, make_response, Response, send_from_directory, redirect, url_for -from Role_Manager import login_admin, login_analyst +from Role_Manager import login_admin, login_analyst, no_cache from flask_login import login_required import difflib @@ -23,7 +23,6 @@ import requests import Flask_config app = Flask_config.app -cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_pasteName = Flask_config.r_serv_pasteName r_serv_metadata = Flask_config.r_serv_metadata @@ -446,6 +445,7 @@ def showDiff(): @showsavedpastes.route('/screenshot/') @login_required @login_analyst +@no_cache def screenshot(filename): return send_from_directory(SCREENSHOT_FOLDER, filename+'.png', as_attachment=True) diff --git a/var/www/modules/showpaste/templates/show_saved_item_min.html b/var/www/modules/showpaste/templates/show_saved_item_min.html index 4228c6a2..ef9fa465 100644 --- a/var/www/modules/showpaste/templates/show_saved_item_min.html +++ b/var/www/modules/showpaste/templates/show_saved_item_min.html @@ -169,7 +169,7 @@ Domain - {{ crawler_metadata['domain'] }} + {{ crawler_metadata['domain'] }} Father diff --git a/var/www/modules/showpaste/templates/show_saved_paste.html b/var/www/modules/showpaste/templates/show_saved_paste.html index 3387e117..31d72436 100644 --- a/var/www/modules/showpaste/templates/show_saved_paste.html +++ b/var/www/modules/showpaste/templates/show_saved_paste.html @@ -393,7 +393,7 @@ {% for b64 in l_64 %}   {{ b64[1] }} - {{ b64[2] }} ({{ b64[4] }}) + {{ b64[2] }} ({{ b64[4] }}) {{ b64[3] }} {% if vt_enabled %} @@ -433,7 +433,7 @@ Domain - {{ crawler_metadata['domain'] }} + {{ crawler_metadata['domain'] }} Father diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index f3b8c7de..3e166063 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -6,25 +6,29 @@ note: The matching of credential against supplied credential is done using Levenshtein distance ''' +import json import redis import datetime import calendar import flask -from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect +from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, Response from Role_Manager import login_admin, login_analyst -from flask_login import login_required +from flask_login import login_required, current_user import re -import Paste from pprint import pprint import Levenshtein +# --------------------------------------------------------------- + +import Paste +import Term + # ============ VARIABLES ============ import Flask_config app = Flask_config.app -cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_term = Flask_config.r_serv_term r_serv_cred = Flask_config.r_serv_cred @@ -146,338 +150,6 @@ def save_tag_to_auto_push(list_tag): # ============ ROUTES ============ -@terms.route("/terms_management/") -@login_required -@login_analyst -def terms_management(): - per_paste = request.args.get('per_paste') - if per_paste == "1" or per_paste is None: - per_paste_text = "per_paste_" - per_paste = 1 - else: - per_paste_text = "" - per_paste = 0 - - today = datetime.datetime.now() - today = today.replace(hour=0, minute=0, second=0, microsecond=0) - today_timestamp = calendar.timegm(today.timetuple()) - - # Map tracking if notifications are enabled for a specific term - notificationEnabledDict = {} - - # Maps a specific term to the associated email addresses - notificationEMailTermMapping = {} - notificationTagsTermMapping = {} - - #Regex - trackReg_list = [] - trackReg_list_values = [] - trackReg_list_num_of_paste = [] - for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name): - - notificationEMailTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex) - notificationTagsTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_regex) - - if tracked_regex not in notificationEnabledDict: - notificationEnabledDict[tracked_regex] = False - - trackReg_list.append(tracked_regex) - value_range = Term_getValueOverRange(tracked_regex, today_timestamp, [1, 7, 31], per_paste=per_paste_text) - - term_date = r_serv_term.hget(TrackedRegexDate_Name, tracked_regex) - - set_paste_name = "regex_" + tracked_regex - trackReg_list_num_of_paste.append(r_serv_term.scard(set_paste_name)) - term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" - value_range.append(term_date) - trackReg_list_values.append(value_range) - - if tracked_regex in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name): - notificationEnabledDict[tracked_regex] = True - - #Set - trackSet_list = [] - trackSet_list_values = [] - trackSet_list_num_of_paste = [] - for tracked_set in r_serv_term.smembers(TrackedSetSet_Name): - tracked_set = tracked_set - - notificationEMailTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set) - notificationTagsTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_set) - - if tracked_set not in notificationEnabledDict: - notificationEnabledDict[tracked_set] = False - - trackSet_list.append(tracked_set) - value_range = Term_getValueOverRange(tracked_set, today_timestamp, [1, 7, 31], per_paste=per_paste_text) - - term_date = r_serv_term.hget(TrackedSetDate_Name, tracked_set) - - set_paste_name = "set_" + tracked_set - trackSet_list_num_of_paste.append(r_serv_term.scard(set_paste_name)) - term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" - value_range.append(term_date) - trackSet_list_values.append(value_range) - - if tracked_set in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name): - notificationEnabledDict[tracked_set] = True - - #Tracked terms - track_list = [] - track_list_values = [] - track_list_num_of_paste = [] - for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name): - - notificationEMailTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term) - notificationTagsTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_term) - - if tracked_term not in notificationEnabledDict: - notificationEnabledDict[tracked_term] = False - - track_list.append(tracked_term) - value_range = Term_getValueOverRange(tracked_term, today_timestamp, [1, 7, 31], per_paste=per_paste_text) - - term_date = r_serv_term.hget(TrackedTermsDate_Name, tracked_term) - - set_paste_name = "tracked_" + tracked_term - - track_list_num_of_paste.append( r_serv_term.scard(set_paste_name) ) - - term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" - value_range.append(term_date) - track_list_values.append(value_range) - - if tracked_term in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name): - notificationEnabledDict[tracked_term] = True - - #blacklist terms - black_list = [] - for blacked_term in r_serv_term.smembers(BlackListTermsSet_Name): - term_date = r_serv_term.hget(BlackListTermsDate_Name, blacked_term) - term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" - black_list.append([blacked_term, term_date]) - - return render_template("terms_management.html", - black_list=black_list, track_list=track_list, trackReg_list=trackReg_list, trackSet_list=trackSet_list, - track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste, - trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste, - trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste, - per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, bootstrap_label=bootstrap_label, - notificationEMailTermMapping=notificationEMailTermMapping, notificationTagsTermMapping=notificationTagsTermMapping) - - -@terms.route("/terms_management_query_paste/") -@login_required -@login_analyst -def terms_management_query_paste(): - term = request.args.get('term') - paste_info = [] - - # check if regex or not - if term.startswith('/') and term.endswith('/'): - set_paste_name = "regex_" + term - track_list_path = r_serv_term.smembers(set_paste_name) - elif term.startswith('\\') and term.endswith('\\'): - set_paste_name = "set_" + term - track_list_path = r_serv_term.smembers(set_paste_name) - else: - set_paste_name = "tracked_" + term - track_list_path = r_serv_term.smembers(set_paste_name) - - for path in track_list_path: - paste = Paste.Paste(path) - p_date = str(paste._get_p_date()) - p_date = p_date[0:4]+'/'+p_date[4:6]+'/'+p_date[6:8] - p_source = paste.p_source - p_size = paste.p_size - p_mime = paste.p_mime - p_lineinfo = paste.get_lines_info() - p_content = paste.get_p_content() - if p_content != 0: - p_content = p_content[0:400] - paste_info.append({"path": path, "date": p_date, "source": p_source, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content}) - - return jsonify(paste_info) - - -@terms.route("/terms_management_query/") -@login_required -@login_analyst -def terms_management_query(): - TrackedTermsDate_Name = "TrackedTermDate" - BlackListTermsDate_Name = "BlackListTermDate" - term = request.args.get('term') - section = request.args.get('section') - - today = datetime.datetime.now() - today = today.replace(hour=0, minute=0, second=0, microsecond=0) - today_timestamp = calendar.timegm(today.timetuple()) - value_range = Term_getValueOverRange(term, today_timestamp, [1, 7, 31]) - - if section == "followTerm": - term_date = r_serv_term.hget(TrackedTermsDate_Name, term) - elif section == "blacklistTerm": - term_date = r_serv_term.hget(BlackListTermsDate_Name, term) - - term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" - value_range.append(str(term_date)) - return jsonify(value_range) - - -@terms.route("/terms_management_action/", methods=['GET']) -@login_required -@login_analyst -def terms_management_action(): - today = datetime.datetime.now() - today = today.replace(microsecond=0) - today_timestamp = calendar.timegm(today.timetuple()) - - - section = request.args.get('section') - action = request.args.get('action') - term = request.args.get('term') - notificationEmailsParam = request.args.get('emailAddresses') - input_tags = request.args.get('tags') - - if action is None or term is None or notificationEmailsParam is None: - return "None" - else: - if section == "followTerm": - if action == "add": - - # Make a list of all passed email addresses - notificationEmails = notificationEmailsParam.split() - - validNotificationEmails = [] - # check for valid email addresses - for email in notificationEmails: - # Really basic validation: - # has exactly one @ sign, and at least one . in the part after the @ - if re.match(r"[^@]+@[^@]+\.[^@]+", email): - validNotificationEmails.append(email) - - # create tags list - list_tags = input_tags.split() - - # check if regex/set or simple term - #regex - if term.startswith('/') and term.endswith('/'): - r_serv_term.sadd(TrackedRegexSet_Name, term) - r_serv_term.hset(TrackedRegexDate_Name, term, today_timestamp) - # add all valid emails to the set - for email in validNotificationEmails: - r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email) - # enable notifications by default - r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term) - # add tags list - for tag in list_tags: - r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term, tag) - save_tag_to_auto_push(list_tags) - - #set - elif term.startswith('\\') and term.endswith('\\'): - tab_term = term[1:-1] - perc_finder = re.compile("\[[0-9]{1,3}\]").search(tab_term) - if perc_finder is not None: - match_percent = perc_finder.group(0)[1:-1] - set_to_add = term - else: - match_percent = DEFAULT_MATCH_PERCENT - set_to_add = "\\" + tab_term[:-1] + ", [{}]]\\".format(match_percent) - r_serv_term.sadd(TrackedSetSet_Name, set_to_add) - r_serv_term.hset(TrackedSetDate_Name, set_to_add, today_timestamp) - # add all valid emails to the set - for email in validNotificationEmails: - r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + set_to_add, email) - # enable notifications by default - r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, set_to_add) - # add tags list - for tag in list_tags: - r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + set_to_add, tag) - save_tag_to_auto_push(list_tags) - - #simple term - else: - r_serv_term.sadd(TrackedTermsSet_Name, term.lower()) - r_serv_term.hset(TrackedTermsDate_Name, term.lower(), today_timestamp) - # add all valid emails to the set - for email in validNotificationEmails: - r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email) - # enable notifications by default - r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower()) - # add tags list - for tag in list_tags: - r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term.lower(), tag) - save_tag_to_auto_push(list_tags) - - elif action == "toggleEMailNotification": - # get the current state - if term in r_serv_term.smembers(TrackedTermsNotificationEnabled_Name): - # remove it - r_serv_term.srem(TrackedTermsNotificationEnabled_Name, term.lower()) - else: - # add it - r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower()) - - #del action - else: - if term.startswith('/') and term.endswith('/'): - r_serv_term.srem(TrackedRegexSet_Name, term) - r_serv_term.hdel(TrackedRegexDate_Name, term) - elif term.startswith('\\') and term.endswith('\\'): - r_serv_term.srem(TrackedSetSet_Name, term) - r_serv_term.hdel(TrackedSetDate_Name, term) - else: - r_serv_term.srem(TrackedTermsSet_Name, term.lower()) - r_serv_term.hdel(TrackedTermsDate_Name, term.lower()) - - # delete the associated notification emails too - r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term) - # delete the associated tags set - r_serv_term.delete(TrackedTermsNotificationTagsPrefix_Name + term) - - elif section == "blacklistTerm": - if action == "add": - r_serv_term.sadd(BlackListTermsSet_Name, term.lower()) - r_serv_term.hset(BlackListTermsDate_Name, term, today_timestamp) - else: - r_serv_term.srem(BlackListTermsSet_Name, term.lower()) - else: - return "None" - - to_return = {} - to_return["section"] = section - to_return["action"] = action - to_return["term"] = term - return jsonify(to_return) - -@terms.route("/terms_management/delete_terms_tags", methods=['POST']) -@login_required -@login_analyst -def delete_terms_tags(): - term = request.form.get('term') - tags_to_delete = request.form.getlist('tags_to_delete') - - if term is not None and tags_to_delete is not None: - for tag in tags_to_delete: - r_serv_term.srem(TrackedTermsNotificationTagsPrefix_Name + term, tag) - return redirect(url_for('terms.terms_management')) - else: - return 'None args', 400 - -@terms.route("/terms_management/delete_terms_email", methods=['GET']) -@login_required -@login_analyst -def delete_terms_email(): - term = request.args.get('term') - email = request.args.get('email') - - if term is not None and email is not None: - r_serv_term.srem(TrackedTermsNotificationEmailsPrefix_Name + term, email) - return redirect(url_for('terms.terms_management')) - else: - return 'None args', 400 - @terms.route("/terms_plot_tool/") @login_required diff --git a/var/www/modules/terms/templates/header_terms.html b/var/www/modules/terms/templates/header_terms.html index a19290a5..90e0f6e6 100644 --- a/var/www/modules/terms/templates/header_terms.html +++ b/var/www/modules/terms/templates/header_terms.html @@ -1,7 +1,6 @@ -
  • Terms frequency +
  • Terms frequency
      -
    • Terms managements
    • Credentials seeker
    • Terms plot top
    • Terms plot tool
      • diff --git a/var/www/modules/trendingcharts/Flask_trendingcharts.py b/var/www/modules/trendingcharts/Flask_trendingcharts.py index a037e171..b2dfa68a 100644 --- a/var/www/modules/trendingcharts/Flask_trendingcharts.py +++ b/var/www/modules/trendingcharts/Flask_trendingcharts.py @@ -17,7 +17,7 @@ from flask_login import login_required import Flask_config app = Flask_config.app -cfg = Flask_config.cfg +config_loader = Flask_config.config_loader baseUrl = Flask_config.baseUrl r_serv_charts = Flask_config.r_serv_charts @@ -69,7 +69,7 @@ def progressionCharts(): @login_required @login_analyst def wordstrending(): - default_display = cfg.get("Flask", "default_display") + default_display = config_loader.get_config_str("Flask", "default_display") return render_template("Wordstrending.html", default_display = default_display) @@ -77,7 +77,7 @@ def wordstrending(): @login_required @login_analyst def protocolstrending(): - default_display = cfg.get("Flask", "default_display") + default_display = config_loader.get_config_str("Flask", "default_display") return render_template("Protocolstrending.html", default_display = default_display) @@ -85,7 +85,7 @@ def protocolstrending(): @login_required @login_analyst def trending(): - default_display = cfg.get("Flask", "default_display") + default_display = config_loader.get_config_str("Flask", "default_display") return render_template("Trending.html", default_display = default_display) diff --git a/var/www/modules/trendingmodules/Flask_trendingmodules.py b/var/www/modules/trendingmodules/Flask_trendingmodules.py index 80646ecb..816d8055 100644 --- a/var/www/modules/trendingmodules/Flask_trendingmodules.py +++ b/var/www/modules/trendingmodules/Flask_trendingmodules.py @@ -17,7 +17,6 @@ from flask_login import login_required import Flask_config app = Flask_config.app -cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_charts = Flask_config.r_serv_charts diff --git a/var/www/static/js/d3/graphlinesgroup.js b/var/www/static/js/d3/graphlinesgroup.js new file mode 100644 index 00000000..09881fd7 --- /dev/null +++ b/var/www/static/js/d3/graphlinesgroup.js @@ -0,0 +1,132 @@ +//Requirement: - D3v5 +// - jquery +// data input: var data = [{"name": "Name1","Data": [{"date": "20191101","value": "35"},{"date": "20191102","value": "12"}]}, +// {"name": "Name2","Data": [{"date": "20191101","value": "23"},{"date": "20191102","value": "34"}]} + +const multilines_group = (container_id, data, options) => { + + const defaults = { + style: { + stroke: "rgb(0, 0, 0)", + strokeWidth: 2, + strokeWidthHover: 4.5 + }, + margin: {top:5, right:150, bottom:40, left:40}, + width: 700, + height: 400 + }; + + options = $.extend(true, defaults, options); + + let width_graph = options.width - options.margin.left - options.margin.right; + let height_graph = options.height - options.margin.top - options.margin.bottom; + + let parseDate = d3.timeParse("%Y%m%d"); + + let x = d3.scaleTime() + .range([0, width_graph]); + + let y = d3.scaleLinear() + .range([height_graph, 0]); + + let color = d3.scaleOrdinal(d3.schemeCategory10); + color.domain(data.map(function (d) { return d.name; })); + + let xAxis = d3.axisBottom(x).tickFormat(d3.timeFormat("%m-%d")); + let yAxis = d3.axisLeft(y); + + let line = d3.line() + .x(function (d) { + return x(d.date); + }) + .y(function (d) { + return y(d.value); + }); + + data.forEach(function (kv) { + kv.Data.forEach(function (d) { + d.date = parseDate(d.date); + }); + }); + + let cities = data; + + let minX = d3.min(data, function (kv) { return d3.min(kv.Data, function (d) { return d.date; }) }); + let maxX = d3.max(data, function (kv) { return d3.max(kv.Data, function (d) { return d.date; }) }); + let minY = d3.min(data, function (kv) { return d3.min(kv.Data, function (d) { return d.value; }) }); + let maxY = d3.max(data, function (kv) { return d3.max(kv.Data, function (d) { return d.value; }) }); + + x.domain([minX, maxX]); + y.domain([minY, maxY]); + + let svg = d3.select("#"+container_id) + .append("svg") + .attr("width", options.width) + .attr("height", options.height) + .append("g") + .attr("transform", "translate("+options.margin.left+","+options.margin.top+")"); + + svg.append("g") + .attr("transform", "translate(0," + height_graph + ")") + .call(xAxis) + .selectAll("text") + .style("text-anchor", "end") + .attr("dx", "-.8em") + .attr("dy", ".15em") + .attr("transform", "rotate(-65)"); + + svg.append("g") + .call(yAxis) + .append("text") + .attr("transform", "rotate(-90)") + .attr("y", 6) + .attr("dy", ".71em") + .style("text-anchor", "end"); + + let name = svg.selectAll(".name") + .data(cities) + .enter().append("g") + + name.append("path") + .attr("d", function (d) { + return line(d.Data); + }) + .style("fill", "none") + .style("stroke-width", options.style.strokeWidth) + .style("stroke", function (d) { + return color(d.name); + }) + .on("mouseover", function(d) { + d3.select(this).style("stroke-width", options.style.strokeWidthHover); + }) + .on("mouseout", function(d) { + d3.select(this).style("stroke-width", options.style.strokeWidth) + .style("stroke", function (d) { + return color(d.name); + }) + }); + + name.append("text") + .datum(function (d) { + return { + name: d.name, + date: d.Data[d.Data.length - 1].date, + value: d.Data[d.Data.length - 1].value + }; + }) + .attr("transform", function (d) { + return "translate(" + x(d.date) + "," + y(d.value) + ")"; + }) + .attr("x", 3) + .attr("dy", ".35em") + .style("stroke", function (d) { + return color(d.name); + }) + .style("stroke-width", 0.5) + .text(function (d) { + return d.name; + }); + + return svg + +} diff --git a/var/www/static/js/d3/sparklines.js b/var/www/static/js/d3/sparklines.js new file mode 100644 index 00000000..f67add25 --- /dev/null +++ b/var/www/static/js/d3/sparklines.js @@ -0,0 +1,48 @@ +const sparkline = (container_id, data, options) => { + + const defaults = { + style: { + stroke: "rgb(0, 0, 0)", + strokeWidth: 2 + }, + margin: {top:3, right:3, bottom:3, left:3}, + width: 100, + height: 60 + }; + + options = $.extend(true, defaults, options); + + + let width_spark = options.width - options.margin.left - options.margin.right; + let height_spark = options.height - options.margin.top - options.margin.bottom; + + let maxX = data.length; + let maxY = d3.max(data, function(d) { return d } ); + + let x = d3.scaleLinear() + .range([0, width_spark]) + .domain([0,maxX]); + + let y = d3.scaleLinear() + .range([height_spark, 0]) + .domain([0,maxY]); + + let line = d3.line() + .x(function(d, i) {return x(i)}) + .y(function(d) {return y(d)}); + + let res = d3.select( "#"+container_id ).append('svg') + .attr('width', options.width) + .attr('height', options.height) + .append('g') + .attr("transform", "translate("+options.margin.left+","+options.margin.top+")") + .append('path') + .datum(data) + .attr('d', line) + .style("fill", "none") + .style("stroke", options.style.stroke) + .style("stroke-width", options.style.strokeWidth); + + return res + +} diff --git a/var/www/templates/change_password.html b/var/www/templates/change_password.html index 7560d8df..695db7c8 100644 --- a/var/www/templates/change_password.html +++ b/var/www/templates/change_password.html @@ -61,7 +61,7 @@ -
    diff --git a/var/www/templates/settings/menu_sidebar.html b/var/www/templates/settings/menu_sidebar.html index 7732bee5..ce63965f 100644 --- a/var/www/templates/settings/menu_sidebar.html +++ b/var/www/templates/settings/menu_sidebar.html @@ -28,7 +28,7 @@
  • - + Change Password