diff --git a/HOWTO.md b/HOWTO.md index 629e8fa8..c21a970f 100644 --- a/HOWTO.md +++ b/HOWTO.md @@ -27,7 +27,7 @@ Feed data to AIL: 4. Edit your configuration file ```bin/packages/config.cfg``` and modify the pystemonpath path accordingly -5. Launch pystemon-feeder ``` ./pystemon-feeder.py ``` +5. Launch pystemon-feeder ``` ./bin/feeder/pystemon-feeder.py ``` How to create a new module @@ -114,6 +114,9 @@ There are two types of installation. You can install a *local* or a *remote* Spl 2. *(Splash host)* To install and setup your tor proxy: - Install the tor proxy: ``sudo apt-get install tor -y`` (Not required if ``Splah host == AIL host`` - The tor proxy is installed by default in AIL) + - Allow Tor to bind to any interface or to the docker interface (by default binds to 127.0.0.1 only) in ``/etc/tor/torrc`` + ``SOCKSPort 0.0.0.0:9050`` or + ``SOCKSPort 172.17.0.1:9050`` - Add the following line ``SOCKSPolicy accept 172.17.0.0/16`` in ``/etc/tor/torrc`` (for a linux docker, the localhost IP is *172.17.0.1*; Should be adapted for other platform) - Restart the tor proxy: ``sudo service tor restart`` diff --git a/OVERVIEW.md b/OVERVIEW.md index effb387d..3d3a62ab 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -26,6 +26,24 @@ ARDB overview ARDB_DB * DB 1 - Curve * DB 2 - TermFreq + ----------------------------------------- TERM ---------------------------------------- + + SET - 'TrackedRegexSet' term + + HSET - 'TrackedRegexDate' tracked_regex today_timestamp + + SET - 'TrackedSetSet' set_to_add + + HSET - 'TrackedSetDate' set_to_add today_timestamp + + SET - 'TrackedSetTermSet' term + + HSET - 'TrackedTermDate' tracked_regex today_timestamp + + SET - 'TrackedNotificationEmails_'+term/set email + + SET - 'TrackedNotifications' term/set + * DB 3 - Trending * DB 4 - Sentiment * DB 5 - TermCred diff --git a/README.md b/README.md index fbcb5363..e8af8e5f 100644 --- a/README.md +++ b/README.md @@ -70,8 +70,6 @@ Type these command lines for a fully automated installation and start AIL framew git clone https://github.com/CIRCL/AIL-framework.git cd AIL-framework ./installing_deps.sh -cd var/www/ -./update_thirdparty.sh cd ~/AIL-framework/ . ./AILENV/bin/activate cd bin/ @@ -105,7 +103,7 @@ curl https://get.docker.com | /bin/bash 2. Type these commands to build the Docker image: ```bash -git clone https://github.com/CIRCL/ail-framework +git clone https://github.com/CIRCL/AIL-framework.git cd AIL-framework docker build -t ail-framework . ``` diff --git a/bin/Curve.py b/bin/Curve.py index 8e228039..c7083c54 100755 --- a/bin/Curve.py +++ b/bin/Curve.py @@ -48,6 +48,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7] top_termFreq_setName_month = ["TopTermFreq_set_month", 31] top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] +TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_" + # create direct link in mail full_paste_url = "/showsavedpaste/?paste=" @@ -71,6 +73,11 @@ def check_if_tracked_term(term, path): for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term): sendEmailNotification(email, 'Term', mail_body) + # tag paste + for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + term): + msg = '{};{}'.format(tag, path) + p.populate_set_out(msg, 'Tags') + def getValueOverRange(word, startDate, num_day): to_return = 0 diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 684af83b..549c0425 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -9,12 +9,30 @@ WHITE="\\033[0;02m" YELLOW="\\033[1;33m" CYAN="\\033[1;36m" -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; +# Getting CWD where bash script resides +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd |sed 's/bin//' )" +export AIL_HOME="${DIR}" +cd ${AIL_HOME} + +if [ -e "${DIR}/AILENV/bin/python" ]; then + echo "AIL-framework virtualenv seems to exist, good" + ENV_PY="${DIR}/AILENV/bin/python" +else + echo "Please make sure you have a AIL-framework environment, au revoir" + exit 1 +fi + +# redis-server is bundled during install +## [ ! -f "`which redis-server`" ] && echo "'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1 + +export AIL_BIN=${AIL_HOME}/bin/ +export AIL_FLASK=${AIL_HOME}/var/www/ +export AIL_REDIS=${AIL_HOME}/redis/src/ +export AIL_ARDB=${AIL_HOME}/ardb/src/ +export AIL_VENV=${AIL_HOME}/AILENV/ + +export PATH=$AIL_VENV/bin:$PATH export PATH=$AIL_HOME:$PATH export PATH=$AIL_REDIS:$PATH export PATH=$AIL_ARDB:$PATH @@ -91,9 +109,9 @@ function launching_logs { screen -dmS "Logging_AIL" sleep 0.1 echo -e $GREEN"\t* Launching logging process"$DEFAULT - screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Queuing -l ../logs/; read x' + screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Queuing -l ../logs/; read x" sleep 0.1 - screen -S "Logging_AIL" -X screen -t "LogScript" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Script -l ../logs/; read x' + screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Script -l ../logs/; read x" } function launching_queues { @@ -101,16 +119,16 @@ function launching_queues { sleep 0.1 echo -e $GREEN"\t* Launching all the queues"$DEFAULT - screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; python3 launch_queues.py; read x' + screen -S "Queue_AIL" -X screen -t "Queues" bash -c "cd ${AIL_BIN}; ${ENV_PY} launch_queues.py; read x" } function checking_configuration { bin_dir=${AIL_HOME}/bin echo -e "\t* Checking configuration" if [ "$1" == "automatic" ]; then - bash -c "python3 $bin_dir/Update-conf.py True" + bash -c "${ENV_PY} $bin_dir/Update-conf.py True" else - bash -c "python3 $bin_dir/Update-conf.py False" + bash -c "${ENV_PY} $bin_dir/Update-conf.py False" fi exitStatus=$? @@ -128,75 +146,75 @@ function launching_scripts { sleep 0.1 echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT - screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ./ModulesInformationV2.py -k 0 -c 1; read x' + screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ./Mixer.py; read x' + screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ./Global.py; read x' + screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Global.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ./Duplicates.py; read x' + screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ./Lines.py; read x' + screen -S "Script_AIL" -X screen -t "Lines" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Lines.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ./DomClassifier.py; read x' + screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./DomClassifier.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ./Categ.py; read x' + screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Categ.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ./Tokenize.py; read x' + screen -S "Script_AIL" -X screen -t "Tokenize" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tokenize.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ./CreditCards.py; read x' + screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ./BankAccount.py; read x' + screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ./Onion.py; read x' + screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Onion.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ./Mail.py; read x' + screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ./ApiKey.py; read x' + screen -S "Script_AIL" -X screen -t "ApiKey" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ApiKey.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ./Web.py; read x' + screen -S "Script_AIL" -X screen -t "Web" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Web.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ./Credential.py; read x' + screen -S "Script_AIL" -X screen -t "Credential" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Credential.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ./Curve.py; read x' + screen -S "Script_AIL" -X screen -t "Curve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Curve.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ./CurveManageTopSets.py; read x' + screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CurveManageTopSets.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./RegexForTermsFrequency.py; read x' + screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexForTermsFrequency.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./SetForTermsFrequency.py; read x' + screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SetForTermsFrequency.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ./Indexer.py; read x' + screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Keys.py; read x' + screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ./Decoder.py; read x' + screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Decoder.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ./Bitcoin.py; read x' + screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Bitcoin.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ./Phone.py; read x' + screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Phone.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ./Release.py; read x' + screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ./Cve.py; read x' + screen -S "Script_AIL" -X screen -t "Cve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cve.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ./WebStats.py; read x' + screen -S "Script_AIL" -X screen -t "WebStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./WebStats.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ./ModuleStats.py; read x' + screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModuleStats.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ./SQLInjectionDetection.py; read x' + screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SQLInjectionDetection.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ./LibInjection.py; read x' + screen -S "Script_AIL" -X screen -t "LibInjection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./LibInjection.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ./alertHandler.py; read x' + screen -S "Script_AIL" -X screen -t "alertHandler" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./alertHandler.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ./MISP_The_Hive_feeder.py; read x' + screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ./Tags.py; read x' + screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tags.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ./SentimentAnalysis.py; read x' + screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SentimentAnalysis.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ./submit_paste.py; read x' + screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./submit_paste.py; read x" } @@ -219,7 +237,7 @@ function launching_crawler { sleep 0.1 for ((i=first_port;i<=last_port;i++)); do - screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; ./Crawler.py onion '$i'; read x' + screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Crawler.py onion $i; read x" sleep 0.1 done @@ -342,7 +360,7 @@ function launch_flask { screen -dmS "Flask_AIL" sleep 0.1 echo -e $GREEN"\t* Launching Flask server"$DEFAULT - screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ./Flask_server.py; read x" + screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ${ENV_PY} ./Flask_server.py; read x" else echo -e $RED"\t* A Flask screen is already launched"$DEFAULT fi @@ -353,9 +371,9 @@ function launch_feeder { screen -dmS "Feeder_Pystemon" sleep 0.1 echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT - screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ./feeder/pystemon-feeder.py; read x' + screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./feeder/pystemon-feeder.py; read x" sleep 0.1 - screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; python2 pystemon.py; read x' + screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c "cd ${AIL_HOME}/../pystemon; ${ENV_PY} ./pystemon.py; read x" else echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT fi diff --git a/bin/MISP_The_Hive_feeder.py b/bin/MISP_The_Hive_feeder.py index 0a8f1791..07c121c9 100755 --- a/bin/MISP_The_Hive_feeder.py +++ b/bin/MISP_The_Hive_feeder.py @@ -54,7 +54,7 @@ from thehive4py.models import Case, CaseTask, CustomFieldHelper -def create_the_hive_alert(source, path, content, tag): +def create_the_hive_alert(source, path, tag): tags = list(r_serv_metadata.smembers('tag:'+path)) artifacts = [ @@ -63,7 +63,6 @@ def create_the_hive_alert(source, path, content, tag): ] l_tags = tag.split(',') - print(tag) # Prepare the sample Alert sourceRef = str(uuid.uuid4())[0:6] @@ -91,6 +90,41 @@ def create_the_hive_alert(source, path, content, tag): except: print('hive connection error') +def feeder(message, count=0): + + if flag_the_hive or flag_misp: + tag, path = message.split(';') + ## FIXME: remove it + if PASTES_FOLDER not in path: + path = os.path.join(PASTES_FOLDER, path) + try: + paste = Paste.Paste(path) + except FileNotFoundError: + if count < 10: + r_serv_db.zincrby('mess_not_saved_export', message, 1) + return 0 + else: + r_serv_db.zrem('mess_not_saved_export', message) + print('Error: {} do not exist, tag= {}'.format(path, tag)) + return 0 + + source = '/'.join(paste.p_path.split('/')[-6:]) + + if HiveApi != False: + if int(r_serv_db.get('hive:auto-alerts')) == 1: + whitelist_hive = r_serv_db.scard('whitelist_hive') + if r_serv_db.sismember('whitelist_hive', tag): + create_the_hive_alert(source, path, tag) + else: + print('hive, auto alerts creation disable') + if flag_misp: + if int(r_serv_db.get('misp:auto-events')) == 1: + if r_serv_db.sismember('whitelist_misp', tag): + misp_wrapper.pushToMISP(uuid_ail, path, tag) + else: + print('misp, auto events creation disable') + + if __name__ == "__main__": publisher.port = 6380 @@ -119,10 +153,18 @@ if __name__ == "__main__": db=cfg.getint("ARDB_Metadata", "db"), decode_responses=True) + # set sensor uuid uuid_ail = r_serv_db.get('ail:uuid') if uuid_ail is None: uuid_ail = r_serv_db.set('ail:uuid', uuid.uuid4() ) + # set default + if r_serv_db.get('hive:auto-alerts') is None: + r_serv_db.set('hive:auto-alerts', 0) + + if r_serv_db.get('misp:auto-events') is None: + r_serv_db.set('misp:auto-events', 0) + p = Process(config_section) # create MISP connection if flag_misp: @@ -167,36 +209,30 @@ if __name__ == "__main__": r_serv_db.set('ail:thehive', False) print('Not connected to The HIVE') + refresh_time = 3 + ## FIXME: remove it + PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + time_1 = time.time() + while True: # Get one message from the input queue message = p.get_from_set() if message is None: - publisher.debug("{} queue is empty, waiting 1s".format(config_section)) - time.sleep(1) - continue + + # handle not saved pastes + if int(time.time() - time_1) > refresh_time: + + num_queu = r_serv_db.zcard('mess_not_saved_export') + list_queu = r_serv_db.zrange('mess_not_saved_export', 0, -1, withscores=True) + + if num_queu and list_queu: + for i in range(0, num_queu): + feeder(list_queu[i][0],list_queu[i][1]) + + time_1 = time.time() + else: + publisher.debug("{} queue is empty, waiting 1s".format(config_section)) + time.sleep(1) else: - - if flag_the_hive or flag_misp: - tag, path = message.split(';') - paste = Paste.Paste(path) - source = '/'.join(paste.p_path.split('/')[-6:]) - - full_path = os.path.join(os.environ['AIL_HOME'], - p.config.get("Directories", "pastes"), path) - - - if HiveApi != False: - if int(r_serv_db.get('hive:auto-alerts')) == 1: - whitelist_hive = r_serv_db.scard('whitelist_hive') - if r_serv_db.sismember('whitelist_hive', tag): - create_the_hive_alert(source, path, full_path, tag) - - else: - print('hive, auto alerts creation disable') - if flag_misp: - if int(r_serv_db.get('misp:auto-events')) == 1: - if r_serv_db.sismember('whitelist_misp', tag): - misp_wrapper.pushToMISP(uuid_ail, path, tag) - else: - print('misp, auto events creation disable') + feeder(message) diff --git a/bin/Mixer.py b/bin/Mixer.py index e1656b8e..e7f9e6de 100755 --- a/bin/Mixer.py +++ b/bin/Mixer.py @@ -80,6 +80,7 @@ if __name__ == '__main__': # OTHER CONFIG # operation_mode = cfg.getint("Module_Mixer", "operation_mode") ttl_key = cfg.getint("Module_Mixer", "ttl_duplicate") + default_unnamed_feed_name = cfg.get("Module_Mixer", "default_unnamed_feed_name") # STATS # processed_paste = 0 @@ -106,7 +107,7 @@ if __name__ == '__main__': paste_name = complete_paste except ValueError as e: - feeder_name = "unnamed_feeder" + feeder_name = default_unnamed_feed_name paste_name = complete_paste # Processed paste diff --git a/bin/NotificationHelper.py b/bin/NotificationHelper.py index d8f7fe92..1bccd314 100755 --- a/bin/NotificationHelper.py +++ b/bin/NotificationHelper.py @@ -3,8 +3,10 @@ import argparse import configparser +import traceback import os import smtplib +from pubsublogger import publisher from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText @@ -15,6 +17,9 @@ This module allows the global configuration and management of notification setti # CONFIG # configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') +publisher.port = 6380 +publisher.channel = "Script" + # notifications enabled/disabled TrackedTermsNotificationEnabled_Name = "TrackedNotifications" @@ -22,7 +27,6 @@ TrackedTermsNotificationEnabled_Name = "TrackedNotifications" # Keys will be e.g. TrackedNotificationEmails TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_" - def sendEmailNotification(recipient, alert_name, content): if not os.path.exists(configfile): @@ -33,22 +37,12 @@ def sendEmailNotification(recipient, alert_name, content): cfg = configparser.ConfigParser() cfg.read(configfile) - sender = cfg.get("Notifications", "sender"), - sender_host = cfg.get("Notifications", "sender_host"), - sender_port = cfg.getint("Notifications", "sender_port"), - sender_pw = cfg.get("Notifications", "sender_pw"), - - if isinstance(sender, tuple): - sender = sender[0] - - if isinstance(sender_host, tuple): - sender_host = sender_host[0] - - if isinstance(sender_port, tuple): - sender_port = sender_port[0] - - if isinstance(sender_pw, tuple): - sender_pw = sender_pw[0] + sender = cfg.get("Notifications", "sender") + sender_host = cfg.get("Notifications", "sender_host") + sender_port = cfg.getint("Notifications", "sender_port") + sender_pw = cfg.get("Notifications", "sender_pw") + if sender_pw == 'None': + sender_pw = None # raise an exception if any of these is None if (sender is None or @@ -83,9 +77,9 @@ def sendEmailNotification(recipient, alert_name, content): smtp_server.quit() print('Send notification ' + alert_name + ' to '+recipient) - except Exception as e: - print(str(e)) - # raise e + except Exception as err: + traceback.print_tb(err.__traceback__) + publisher.warning(err) if __name__ == '__main__': parser = argparse.ArgumentParser(description='Test notification sender.') diff --git a/bin/RegexForTermsFrequency.py b/bin/RegexForTermsFrequency.py index ecca8e4d..0db7f2ee 100755 --- a/bin/RegexForTermsFrequency.py +++ b/bin/RegexForTermsFrequency.py @@ -9,35 +9,45 @@ supplied in the term webpage. import redis import time from pubsublogger import publisher -from packages import lib_words from packages import Paste -import os -from os import environ -import datetime import calendar import re +import signal +import time from Helper import Process - # Email notifications from NotificationHelper import * + +class TimeoutException(Exception): + pass + + +def timeout_handler(signum, frame): + raise TimeoutException + +signal.signal(signal.SIGALRM, timeout_handler) + # Config Variables -DICO_REFRESH_TIME = 60 #s +DICO_REFRESH_TIME = 60 # s BlackListTermsSet_Name = "BlackListSetTermSet" TrackedTermsSet_Name = "TrackedSetTermSet" TrackedRegexSet_Name = "TrackedRegexSet" -top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set +top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set oneDay = 60*60*24 top_termFreq_setName_day = ["TopTermFreq_set_day_", 1] top_termFreq_setName_week = ["TopTermFreq_set_week", 7] top_termFreq_setName_month = ["TopTermFreq_set_month", 31] -top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] +top_termFreq_set_array = [top_termFreq_setName_day, top_termFreq_setName_week, top_termFreq_setName_month] + +TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_" # create direct link in mail full_paste_url = "/showsavedpaste/?paste=" + def refresh_dicos(): dico_regex = {} dico_regexname_to_redis = {} @@ -53,6 +63,7 @@ if __name__ == "__main__": config_section = 'RegexForTermsFrequency' p = Process(config_section) + max_execution_time = p.config.getint(config_section, "max_execution_time") # REDIS # server_term = redis.StrictRedis( @@ -67,7 +78,7 @@ if __name__ == "__main__": # create direct link in mail full_paste_url = p.config.get("Notifications", "ail_domain") + full_paste_url - #compile the regex + # compile the regex dico_refresh_cooldown = time.time() dico_regex, dico_regexname_to_redis = refresh_dicos() @@ -87,13 +98,22 @@ if __name__ == "__main__": timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0)) curr_set = top_termFreq_setName_day[0] + str(timestamp) - content = Paste.Paste(filename).get_p_content() + paste = Paste.Paste(filename) + content = paste.get_p_content() - #iterate the word with the regex + # iterate the word with the regex for regex_str, compiled_regex in dico_regex.items(): - matched = compiled_regex.search(content) - if matched is not None: #there is a match + signal.alarm(max_execution_time) + try: + matched = compiled_regex.search(content) + except TimeoutException: + print ("{0} processing timeout".format(paste.p_path)) + continue + else: + signal.alarm(0) + + if matched is not None: # there is a match print('regex matched {}'.format(regex_str)) matched = matched.group(0) regex_str_complete = "/" + regex_str + "/" @@ -104,20 +124,25 @@ if __name__ == "__main__": # create mail body mail_body = ("AIL Framework,\n" - "New occurrence for regex: " + regex_str + "\n" - ''+full_paste_url + filename) + "New occurrence for regex: " + regex_str + "\n" + ''+full_paste_url + filename) # Send to every associated email adress for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + regex_str_complete): sendEmailNotification(email, 'Term', mail_body) + # tag paste + for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + regex_str_complete): + msg = '{};{}'.format(tag, filename) + p.populate_set_out(msg, 'Tags') + set_name = 'regex_' + dico_regexname_to_redis[regex_str] new_to_the_set = server_term.sadd(set_name, filename) new_to_the_set = True if new_to_the_set == 1 else False - #consider the num of occurence of this term + # consider the num of occurence of this term regex_value = int(server_term.hincrby(timestamp, dico_regexname_to_redis[regex_str], int(1))) - #1 term per paste + # 1 term per paste if new_to_the_set: regex_value_perPaste = int(server_term.hincrby("per_paste_" + str(timestamp), dico_regexname_to_redis[regex_str], int(1))) server_term.zincrby("per_paste_" + curr_set, dico_regexname_to_redis[regex_str], float(1)) diff --git a/bin/SetForTermsFrequency.py b/bin/SetForTermsFrequency.py index 78de9b08..19ed7210 100755 --- a/bin/SetForTermsFrequency.py +++ b/bin/SetForTermsFrequency.py @@ -34,6 +34,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7] top_termFreq_setName_month = ["TopTermFreq_set_month", 31] top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] +TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_" + # create direct link in mail full_paste_url = "/showsavedpaste/?paste=" @@ -121,6 +123,11 @@ if __name__ == "__main__": for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + dico_setname_to_redis[str(the_set)]): sendEmailNotification(email, 'Term', mail_body) + # tag paste + for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + dico_setname_to_redis[str(the_set)]): + msg = '{};{}'.format(tag, filename) + p.populate_set_out(msg, 'Tags') + print(the_set, "matched in", filename) set_name = 'set_' + dico_setname_to_redis[the_set] new_to_the_set = server_term.sadd(set_name, filename) diff --git a/bin/feeder/pystemon-feeder.py b/bin/feeder/pystemon-feeder.py index b6680ee9..280849ba 100755 --- a/bin/feeder/pystemon-feeder.py +++ b/bin/feeder/pystemon-feeder.py @@ -61,10 +61,10 @@ topic = '102' while True: time.sleep(base_sleeptime + sleep_inc) paste = r.lpop("pastes") - print(paste) if paste is None: continue try: + print(paste) with open(pystemonpath+paste, 'rb') as f: #.read() messagedata = f.read() path_to_send = pastes_directory+paste diff --git a/bin/feeder/test-zmq.py b/bin/feeder/test-zmq.py index f6f28aa1..110c5de2 100644 --- a/bin/feeder/test-zmq.py +++ b/bin/feeder/test-zmq.py @@ -20,7 +20,7 @@ socket.connect ("tcp://crf.circl.lu:%s" % port) # 102 Full pastes in raw base64(gz) topicfilter = "102" -socket.setsockopt(zmq.SUBSCRIBE, topicfilter) +socket.setsockopt_string(zmq.SUBSCRIBE, topicfilter) while True: message = socket.recv() diff --git a/bin/packages/Paste.py b/bin/packages/Paste.py index 524a7665..d02a92f5 100755 --- a/bin/packages/Paste.py +++ b/bin/packages/Paste.py @@ -82,6 +82,13 @@ class Paste(object): db=cfg.getint("ARDB_Metadata", "db"), decode_responses=True) + PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + if PASTES_FOLDER not in p_path: + self.p_rel_path = p_path + p_path = os.path.join(PASTES_FOLDER, p_path) + else: + self.p_rel_path = None + self.p_path = p_path self.p_name = os.path.basename(self.p_path) self.p_size = round(os.path.getsize(self.p_path)/1024.0, 2) @@ -280,6 +287,8 @@ class Paste(object): def _get_p_duplicate(self): self.p_duplicate = self.store_metadata.smembers('dup:'+self.p_path) + if self.p_rel_path is not None: + self.p_duplicate.union( self.store_metadata.smembers('dup:'+self.p_rel_path) ) if self.p_duplicate is not None: return list(self.p_duplicate) else: diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index b459f011..b5980766 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -27,6 +27,7 @@ ail_domain = http://localhost:7000 sender = sender@example.com sender_host = smtp.example.com sender_port = 1337 +sender_pw = None # optional for using with authenticated SMTP over SSL # sender_pw = securepassword @@ -98,6 +99,10 @@ threshold_stucked_module=600 operation_mode = 3 #Define the time that a paste will be considerate duplicate. in seconds (1day = 86400) ttl_duplicate = 86400 +default_unnamed_feed_name = unnamed_feeder + +[RegexForTermsFrequency] +max_execution_time = 60 ##### Redis ##### [Redis_Cache] @@ -201,6 +206,9 @@ dns = 8.8.8.8 [Mail] dns = 8.8.8.8 +[Web] +dns = 149.13.33.69 + # Indexer configuration [Indexer] type = whoosh diff --git a/bin/packages/lib_refine.py b/bin/packages/lib_refine.py index 5d2af0a9..32f56900 100644 --- a/bin/packages/lib_refine.py +++ b/bin/packages/lib_refine.py @@ -1,6 +1,8 @@ #!/usr/bin/python3 import re +import os +import configparser import dns.resolver from pubsublogger import publisher @@ -101,11 +103,20 @@ def checking_MX_record(r_serv, adress_set, addr_dns): def checking_A_record(r_serv, domains_set): + configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + cfg = configparser.ConfigParser() + cfg.read(configfile) + dns_server = cfg.get("Web", "dns") + score = 0 num = len(domains_set) WalidA = set([]) resolver = dns.resolver.Resolver() - resolver.nameservers = ['149.13.33.69'] + resolver.nameservers = [dns_server] resolver.timeout = 5 resolver.lifetime = 2 diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index deb5a069..0dc40448 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -32,13 +32,15 @@ publish = Redis_Words [Curve] subscribe = Redis_Words -publish = Redis_CurveManageTopSets +publish = Redis_CurveManageTopSets,Redis_Tags [RegexForTermsFrequency] subscribe = Redis_Global +publish = Redis_Tags [SetForTermsFrequency] subscribe = Redis_Global +publish = Redis_Tags [CurveManageTopSets] subscribe = Redis_CurveManageTopSets diff --git a/pip3_packages_requirement.txt b/pip3_packages_requirement.txt index dd447d5c..12ef09ff 100644 --- a/pip3_packages_requirement.txt +++ b/pip3_packages_requirement.txt @@ -2,7 +2,7 @@ pymisp thehive4py -redis +redis==2.10.6 #filemagic conflict with magic crcmod mmh3 diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index ea6fd6ed..7cc802f0 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -102,7 +102,6 @@ r_serv_onion = redis.StrictRedis( db=cfg.getint("ARDB_Onion", "db"), decode_responses=True) - sys.path.append('../../configs/keys') # MISP # try: diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 16930ef8..cc38de77 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -506,6 +506,8 @@ def edit_tag_export(): status_misp = [] status_hive = [] + infoleak_tags = Taxonomies().get('infoleak').machinetags() + is_infoleak_tag = [] for tag in list_export_tags: if r_serv_db.sismember('whitelist_misp', tag): @@ -519,6 +521,11 @@ def edit_tag_export(): else: status_hive.append(False) + if tag in infoleak_tags: + is_infoleak_tag.append(True) + else: + is_infoleak_tag.append(False) + if misp_auto_events is not None: if int(misp_auto_events) == 1: misp_active = True @@ -543,6 +550,7 @@ def edit_tag_export(): misp_active=misp_active, hive_active=hive_active, list_export_tags=list_export_tags, + is_infoleak_tag=is_infoleak_tag, status_misp=status_misp, status_hive=status_hive, nb_tags_whitelist_misp=nb_tags_whitelist_misp, @@ -594,5 +602,37 @@ def disable_hive_auto_alert(): r_serv_db.set('hive:auto-alerts', 0) return edit_tag_export() +@PasteSubmit.route("/PasteSubmit/add_push_tag") +def add_push_tag(): + tag = request.args.get('tag') + if tag is not None: + + #limit tag length + if len(tag) > 49: + tag = tag[0:48] + + r_serv_db.sadd('list_export_tags', tag) + + to_return = {} + to_return["tag"] = tag + return jsonify(to_return) + else: + return 'None args', 400 + +@PasteSubmit.route("/PasteSubmit/delete_push_tag") +def delete_push_tag(): + tag = request.args.get('tag') + + infoleak_tags = Taxonomies().get('infoleak').machinetags() + if tag not in infoleak_tags and r_serv_db.sismember('list_export_tags', tag): + r_serv_db.srem('list_export_tags', tag) + r_serv_db.srem('whitelist_misp', tag) + r_serv_db.srem('whitelist_hive', tag) + to_return = {} + to_return["tag"] = tag + return jsonify(to_return) + else: + return 'this tag can\'t be removed', 400 + # ========= REGISTRATION ========= app.register_blueprint(PasteSubmit, url_prefix=baseUrl) diff --git a/var/www/modules/PasteSubmit/templates/edit_tag_export.html b/var/www/modules/PasteSubmit/templates/edit_tag_export.html index 04a506d6..94980787 100644 --- a/var/www/modules/PasteSubmit/templates/edit_tag_export.html +++ b/var/www/modules/PasteSubmit/templates/edit_tag_export.html @@ -37,6 +37,9 @@ background: #d91f2d; color: #fff; } + .mouse_pointer{ + cursor: pointer; + } @@ -169,7 +172,14 @@ {% endif %} - {{ tag }} + + {{ tag }} + {% if not is_infoleak_tag[loop.index0] %} +
+ +
+ {% endif %} + {% endfor %} @@ -209,7 +219,14 @@ {% endif %} - {{ tag }} + + {{ tag }} + {% if not is_infoleak_tag[loop.index0] %} +
+ +
+ {% endif %} + {% endfor %} @@ -232,6 +249,42 @@ +
+ + + + +
+ @@ -277,6 +330,25 @@ $(document).ready(function(){ } ); } + + function delete_push_tag(tag){ + //var row_tr = $(this).closest("tr"); + $.get("{{ url_for('PasteSubmit.delete_push_tag') }}", { tag: tag }, function(data, status){ + if(status == "success") { + //row_tr.remove(); + window.location.reload(false); + } + }); + } + + function add_custom_tag(){ + $.get("{{ url_for('PasteSubmit.add_push_tag') }}", { tag: document.getElementById('new_custom_tag').value }, function(data, status){ + if(status == "success") { + //row_tr.remove(); + window.location.reload(false); + } + }); + } diff --git a/var/www/modules/Tags/Flask_Tags.py b/var/www/modules/Tags/Flask_Tags.py index e79d56fc..bbc918ed 100644 --- a/var/www/modules/Tags/Flask_Tags.py +++ b/var/www/modules/Tags/Flask_Tags.py @@ -28,6 +28,7 @@ r_serv_statistics = Flask_config.r_serv_statistics max_preview_char = Flask_config.max_preview_char max_preview_modal = Flask_config.max_preview_modal bootstrap_label = Flask_config.bootstrap_label +PASTES_FOLDER = Flask_config.PASTES_FOLDER Tags = Blueprint('Tags', __name__, template_folder='templates') diff --git a/var/www/modules/search/Flask_search.py b/var/www/modules/search/Flask_search.py index 5b458589..7f6cd724 100644 --- a/var/www/modules/search/Flask_search.py +++ b/var/www/modules/search/Flask_search.py @@ -16,6 +16,8 @@ from whoosh import index from whoosh.fields import Schema, TEXT, ID from whoosh.qparser import QueryParser +import time + # ============ VARIABLES ============ import Flask_config @@ -55,8 +57,8 @@ def get_index_list(selected_index=""): if os.path.isdir(os.path.join(baseindexpath, dirs)): value = dirs name = to_iso_date(dirs) + " - " + \ - str(get_dir_size(dirs) / (1000*1000)) + " Mb " + \ - "(" + str(get_item_count(dirs)) + " Items" + ")" + str(get_dir_size(dirs) / (1000*1000)) + " Mb " #+ \ + #"(" + str(get_item_count(dirs))''' + " Items" + ")" flag = dirs==selected_index.split('/')[-1] if dirs == "old_index": temp = [value, name, flag] @@ -66,6 +68,7 @@ def get_index_list(selected_index=""): index_list.sort(reverse=True, key=lambda x: x[0]) if len(temp) != 0: index_list.append(temp) + return index_list def get_dir_size(directory): @@ -108,6 +111,7 @@ def search(): else: selected_index = os.path.join(baseindexpath, index_name) + ''' temporary disabled # Search filename for path in r_serv_pasteName.smembers(q[0]): r.append(path) @@ -119,13 +123,14 @@ def search(): curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] paste_date.append(curr_date) paste_size.append(paste._get_p_size()) + ''' # Search full line schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) ix = index.open_dir(selected_index) with ix.searcher() as searcher: - query = QueryParser("content", ix.schema).parse(" ".join(q)) + query = QueryParser("content", ix.schema).parse("".join(q)) results = searcher.search_page(query, 1, pagelen=num_elem_to_get) for x in results: r.append(x.items()[0][1]) @@ -159,15 +164,18 @@ def search(): results = searcher.search(query) num_res = len(results) + index_list = get_index_list() + index_min = 1 - index_max = len(get_index_list()) + index_max = len(index_list) + return render_template("search.html", r=r, c=c, query=request.form['query'], paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal, num_res=num_res, index_min=index_min, index_max=index_max, bootstrap_label=bootstrap_label, paste_tags=paste_tags, - index_list=get_index_list(selected_index) + index_list=index_list ) @@ -212,6 +220,7 @@ def get_more_search_result(): p_tags = r_serv_metadata.smembers('tag:'+path) l_tags = [] for tag in p_tags: + complete_tag = tag tag = tag.split('=') if len(tag) > 1: if tag[1] != '': @@ -223,7 +232,7 @@ def get_more_search_result(): else: tag = tag[0] - l_tags.append(tag) + l_tags.append( (tag, complete_tag) ) list_tags.append(l_tags) to_return = {} diff --git a/var/www/modules/search/templates/search.html b/var/www/modules/search/templates/search.html index adc1b555..5754a0ee 100644 --- a/var/www/modules/search/templates/search.html +++ b/var/www/modules/search/templates/search.html @@ -201,7 +201,9 @@ var curr_preview = data.preview_array[i].replace(/\"/g, "\'"); var tag = "" for(j=0; j" + data.list_tags[j] + "" + tag = tag + "" + + "" + data.list_tags[i][j][0] + + "" + "" } search_table.row.add( [ init_num_of_elements_in_table+((offset))+i+1, diff --git a/var/www/modules/showpaste/Flask_showpaste.py b/var/www/modules/showpaste/Flask_showpaste.py index 4912e7b0..188af759 100644 --- a/var/www/modules/showpaste/Flask_showpaste.py +++ b/var/www/modules/showpaste/Flask_showpaste.py @@ -41,7 +41,9 @@ showsavedpastes = Blueprint('showsavedpastes', __name__, template_folder='templa # ============ FUNCTIONS ============ def showpaste(content_range, requested_path): + relative_path = None if PASTES_FOLDER not in requested_path: + relative_path = requested_path requested_path = os.path.join(PASTES_FOLDER, requested_path) # remove old full path #requested_path = requested_path.replace(PASTES_FOLDER, '') @@ -122,6 +124,8 @@ def showpaste(content_range, requested_path): active_taxonomies = r_serv_tags.smembers('active_taxonomies') l_tags = r_serv_metadata.smembers('tag:'+requested_path) + if relative_path is not None: + l_tags.union( r_serv_metadata.smembers('tag:'+relative_path) ) #active galaxies active_galaxies = r_serv_tags.smembers('active_galaxies') @@ -224,7 +228,6 @@ def showpaste(content_range, requested_path): @showsavedpastes.route("/showsavedpaste/") #completely shows the paste in a new tab def showsavedpaste(): requested_path = request.args.get('paste', '') - print(requested_path) return showpaste(0, requested_path) @showsavedpastes.route("/showsavedrawpaste/") #shows raw diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index b0794593..86d67af8 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -10,7 +10,7 @@ import redis import datetime import calendar import flask -from flask import Flask, render_template, jsonify, request, Blueprint +from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect import re import Paste from pprint import pprint @@ -24,6 +24,8 @@ cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_term = Flask_config.r_serv_term r_serv_cred = Flask_config.r_serv_cred +r_serv_db = Flask_config.r_serv_db +bootstrap_label = Flask_config.bootstrap_label terms = Blueprint('terms', __name__, template_folder='templates') @@ -51,6 +53,7 @@ TrackedTermsNotificationEnabled_Name = "TrackedNotifications" # same value as in `bin/NotificationHelper.py` # Keys will be e.g. TrackedNotificationEmails_ TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_" +TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_" '''CRED''' REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+' @@ -130,6 +133,12 @@ def mixUserName(supplied, extensive=False): filtered_usernames.append(usr) return filtered_usernames +def save_tag_to_auto_push(list_tag): + for tag in set(list_tag): + #limit tag length + if len(tag) > 49: + tag = tag[0:48] + r_serv_db.sadd('list_export_tags', tag) # ============ ROUTES ============ @@ -152,6 +161,7 @@ def terms_management(): # Maps a specific term to the associated email addresses notificationEMailTermMapping = {} + notificationTagsTermMapping = {} #Regex trackReg_list = [] @@ -159,7 +169,8 @@ def terms_management(): trackReg_list_num_of_paste = [] for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name): - notificationEMailTermMapping[tracked_regex] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)) ) + notificationEMailTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex) + notificationTagsTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_regex) if tracked_regex not in notificationEnabledDict: notificationEnabledDict[tracked_regex] = False @@ -185,8 +196,8 @@ def terms_management(): for tracked_set in r_serv_term.smembers(TrackedSetSet_Name): tracked_set = tracked_set - notificationEMailTermMapping[tracked_set] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)) ) - + notificationEMailTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set) + notificationTagsTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_set) if tracked_set not in notificationEnabledDict: notificationEnabledDict[tracked_set] = False @@ -211,7 +222,8 @@ def terms_management(): track_list_num_of_paste = [] for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name): - notificationEMailTermMapping[tracked_term] = "\n".join( r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term)) + notificationEMailTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term) + notificationTagsTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_term) if tracked_term not in notificationEnabledDict: notificationEnabledDict[tracked_term] = False @@ -244,7 +256,8 @@ def terms_management(): track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste, trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste, trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste, - per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, notificationEMailTermMapping=notificationEMailTermMapping) + per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, bootstrap_label=bootstrap_label, + notificationEMailTermMapping=notificationEMailTermMapping, notificationTagsTermMapping=notificationTagsTermMapping) @terms.route("/terms_management_query_paste/") @@ -313,6 +326,7 @@ def terms_management_action(): action = request.args.get('action') term = request.args.get('term') notificationEmailsParam = request.args.get('emailAddresses') + input_tags = request.args.get('tags') if action is None or term is None or notificationEmailsParam is None: return "None" @@ -320,11 +334,8 @@ def terms_management_action(): if section == "followTerm": if action == "add": - # Strip all whitespace - notificationEmailsParam = "".join(notificationEmailsParam.split()) - # Make a list of all passed email addresses - notificationEmails = notificationEmailsParam.split(",") + notificationEmails = notificationEmailsParam.split() validNotificationEmails = [] # check for valid email addresses @@ -334,6 +345,8 @@ def terms_management_action(): if re.match(r"[^@]+@[^@]+\.[^@]+", email): validNotificationEmails.append(email) + # create tags list + list_tags = input_tags.split() # check if regex/set or simple term #regex @@ -345,6 +358,10 @@ def terms_management_action(): r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email) # enable notifications by default r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term) + # add tags list + for tag in list_tags: + r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term, tag) + save_tag_to_auto_push(list_tags) #set elif term.startswith('\\') and term.endswith('\\'): @@ -363,6 +380,10 @@ def terms_management_action(): r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + set_to_add, email) # enable notifications by default r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, set_to_add) + # add tags list + for tag in list_tags: + r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + set_to_add, tag) + save_tag_to_auto_push(list_tags) #simple term else: @@ -373,6 +394,10 @@ def terms_management_action(): r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email) # enable notifications by default r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower()) + # add tags list + for tag in list_tags: + r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term.lower(), tag) + save_tag_to_auto_push(list_tags) elif action == "toggleEMailNotification": # get the current state @@ -397,6 +422,8 @@ def terms_management_action(): # delete the associated notification emails too r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term) + # delete the associated tags set + r_serv_term.delete(TrackedTermsNotificationTagsPrefix_Name + term) elif section == "blacklistTerm": if action == "add": @@ -413,6 +440,28 @@ def terms_management_action(): to_return["term"] = term return jsonify(to_return) +@terms.route("/terms_management/delete_terms_tags", methods=['POST']) +def delete_terms_tags(): + term = request.form.get('term') + tags_to_delete = request.form.getlist('tags_to_delete') + + if term is not None and tags_to_delete is not None: + for tag in tags_to_delete: + r_serv_term.srem(TrackedTermsNotificationTagsPrefix_Name + term, tag) + return redirect(url_for('terms.terms_management')) + else: + return 'None args', 400 + +@terms.route("/terms_management/delete_terms_email", methods=['GET']) +def delete_terms_email(): + term = request.args.get('term') + email = request.args.get('email') + + if term is not None and email is not None: + r_serv_term.srem(TrackedTermsNotificationEmailsPrefix_Name + term, email) + return redirect(url_for('terms.terms_management')) + else: + return 'None args', 400 @terms.route("/terms_plot_tool/") diff --git a/var/www/modules/terms/templates/terms_management.html b/var/www/modules/terms/templates/terms_management.html index 0efda575..a2ecd906 100644 --- a/var/www/modules/terms/templates/terms_management.html +++ b/var/www/modules/terms/templates/terms_management.html @@ -36,6 +36,12 @@ white-space:pre-wrap; word-wrap:break-word; } + .mouse_pointer{ + cursor: pointer; + } + .lb-md { + font-size: 16px; + } @@ -74,6 +80,7 @@
+ {% set uniq_id = namespace(modal_id=0)%}