Merge pull request #208 from CIRCL/tags

Tags
This commit is contained in:
Alexandre Dulaunoy 2018-06-20 10:11:45 +02:00 committed by GitHub
commit 5d2de7c52d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
41 changed files with 2423 additions and 100 deletions

3
.gitignore vendored
View file

@ -17,6 +17,7 @@ BASE64
DATA_ARDB
indexdir/
logs/
old/
# Webstuff
var/www/static/
@ -26,9 +27,11 @@ var/www/static/
!var/www/static/js/plot-graph.js
!var/www/static/js/trendingchart.js
var/www/templates/header.html
var/www/submitted
# Local config
bin/packages/config.cfg
configs/keys
# installed files
nltk_data/

View file

@ -36,6 +36,9 @@ Features
* Detect Bitcoin address and Bitcoin private keys
* Detect private keys and certificate
* Tagging system with [MISP Galaxy](https://github.com/MISP/misp-galaxy) and [MISP Taxonomies](https://github.com/MISP/misp-taxonomies) tags
* UI paste submission
* Create events on [MISP](https://github.com/MISP/MISP) and cases on [The Hive](https://github.com/TheHive-Project/TheHive)
* Automatic paste export on [MISP](https://github.com/MISP/MISP) (events) and [The Hive](https://github.com/TheHive-Project/TheHive) (alerts) on selected tags
Installation
------------
@ -154,6 +157,16 @@ Tagging system
![Tags](./doc/screenshots/tags.png?raw=true "AIL framework tags")
MISP and The Hive, automatic events and alerts creation
--------
![paste_submit](./doc/screenshots/tag_auto_export.png?raw=true "AIL framework MISP and Hive auto export")
Paste submission
--------
![paste_submit](./doc/screenshots/paste_submit.png?raw=true "AIL framework paste submission")
Sentiment analysis
------------------

View file

@ -57,7 +57,6 @@ if __name__ == '__main__':
while True:
message = p.get_from_set()
#print(message)
# Recovering the streamed message informations.
if message is not None:
splitted = message.split()

View file

@ -160,9 +160,13 @@ function launching_scripts {
sleep 0.1
screen -S "Script_AIL" -X screen -t "alertHandler" bash -c './alertHandler.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c './MISP_The_Hive_feeder.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Tags" bash -c './Tags.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c './submit_paste.py; read x'
}

200
bin/MISP_The_Hive_feeder.py Executable file
View file

@ -0,0 +1,200 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
module
====================
This module send tagged pastes to MISP or THE HIVE Project
"""
import redis
import sys
import os
import time
import json
import configparser
from pubsublogger import publisher
from Helper import Process
from packages import Paste
import ailleakObject
import uuid
from pymisp import PyMISP
sys.path.append('../configs/keys')
# import MISP KEYS
try:
from mispKEYS import misp_url, misp_key, misp_verifycert
flag_misp = True
except:
print('Misp keys not present')
flag_misp = False
# import The Hive Keys
try:
from theHiveKEYS import the_hive_url, the_hive_key, the_hive_verifycert
if the_hive_url == '':
flag_the_hive = False
else:
flag_the_hive = True
except:
print('The HIVE keys not present')
flag_the_hive = False
from thehive4py.api import TheHiveApi
import thehive4py.exceptions
from thehive4py.models import Alert, AlertArtifact
from thehive4py.models import Case, CaseTask, CustomFieldHelper
def create_the_hive_alert(source, path, content, tag):
tags = list(r_serv_metadata.smembers('tag:'+path))
artifacts = [
AlertArtifact( dataType='uuid-ail', data=r_serv_db.get('ail:uuid') ),
AlertArtifact( dataType='file', data=path, tags=tags )
]
l_tags = tag.split(',')
print(tag)
# Prepare the sample Alert
sourceRef = str(uuid.uuid4())[0:6]
alert = Alert(title='AIL Leak',
tlp=3,
tags=l_tags,
description='infoleak',
type='ail',
source=source,
sourceRef=sourceRef,
artifacts=artifacts)
# Create the Alert
id = None
try:
response = HiveApi.create_alert(alert)
if response.status_code == 201:
#print(json.dumps(response.json(), indent=4, sort_keys=True))
print('Alert Created')
print('')
id = response.json()['id']
else:
print('ko: {}/{}'.format(response.status_code, response.text))
return 0
except:
print('hive connection error')
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
config_section = 'misp_the_hive_feeder'
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \
Did you set environment variables? \
Or activate the virtualenv.')
cfg = configparser.ConfigParser()
cfg.read(configfile)
r_serv_db = redis.StrictRedis(
host=cfg.get("ARDB_DB", "host"),
port=cfg.getint("ARDB_DB", "port"),
db=cfg.getint("ARDB_DB", "db"),
decode_responses=True)
r_serv_metadata = redis.StrictRedis(
host=cfg.get("ARDB_Metadata", "host"),
port=cfg.getint("ARDB_Metadata", "port"),
db=cfg.getint("ARDB_Metadata", "db"),
decode_responses=True)
uuid_ail = r_serv_db.get('ail:uuid')
if uuid_ail is None:
uuid_ail = r_serv_db.set('ail:uuid', uuid.uuid4() )
config_section = 'misp_the_hive_feeder'
p = Process(config_section)
# create MISP connection
if flag_misp:
try:
pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
except:
flag_misp = False
r_serv_db.set('ail:misp', False)
print('Not connected to MISP')
if flag_misp:
try:
misp_wrapper = ailleakObject.ObjectWrapper(pymisp)
r_serv_db.set('ail:misp', True)
print('Connected to MISP:', misp_url)
except e:
flag_misp = False
r_serv_db.set('ail:misp', False)
print(e)
print('Not connected to MISP')
# create The HIVE connection
if flag_the_hive:
try:
HiveApi = TheHiveApi(the_hive_url, the_hive_key, cert = the_hive_verifycert)
r_serv_db.set('ail:thehive', True)
except:
HiveApi = False
flag_the_hive = False
r_serv_db.set('ail:thehive', False)
print('Not connected to The HIVE')
if HiveApi != False and flag_the_hive:
try:
HiveApi.get_alert(0)
print('Connected to The HIVE:', the_hive_url)
except thehive4py.exceptions.AlertException:
HiveApi = False
flag_the_hive = False
print('Not connected to The HIVE')
while True:
# Get one message from the input queue
message = p.get_from_set()
if message is None:
publisher.debug("{} queue is empty, waiting 1s".format(config_section))
time.sleep(1)
continue
else:
if flag_the_hive or flag_misp:
tag, path = message.split(';')
paste = Paste.Paste(path)
source = '/'.join(paste.p_path.split('/')[-6:])
full_path = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "pastes"), path)
if HiveApi != False:
if int(r_serv_db.get('hive:auto-alerts')) == 1:
whitelist_hive = r_serv_db.scard('whitelist_hive')
if r_serv_db.sismember('whitelist_hive', tag):
create_the_hive_alert(source, path, full_path, tag)
else:
print('hive, auto alerts creation disable')
if flag_misp:
if int(r_serv_db.get('misp:auto-events')) == 1:
if r_serv_db.sismember('whitelist_misp', tag):
misp_wrapper.pushToMISP(uuid_ail, path, tag)
else:
print('misp, auto events creation disable')

View file

@ -66,3 +66,5 @@ if __name__ == '__main__':
print("new paste: {}".format(path))
print(" tagged: {}".format(tag))
server_metadata.sadd('tag:'+path, tag)
p.populate_set_out(message, 'MISP_The_Hive_feeder')

View file

@ -8,12 +8,11 @@ import datetime
import json
from io import BytesIO
class AilleakObject(AbstractMISPObjectGenerator):
def __init__(self, moduleName, p_source, p_date, p_content, p_duplicate, p_duplicate_number):
class AilLeakObject(AbstractMISPObjectGenerator):
def __init__(self, uuid_ail, p_source, p_date, p_content, p_duplicate, p_duplicate_number):
super(AbstractMISPObjectGenerator, self).__init__('ail-leak')
self._moduleName = moduleName
self._p_source = p_source.split('/')[-5:]
self._p_source = '/'.join(self._p_source)[:-3] # -3 removes .gz
self._uuid = uuid_ail
self._p_source = p_source
self._p_date = p_date
self._p_content = p_content
self._p_duplicate = p_duplicate
@ -21,14 +20,15 @@ class AilleakObject(AbstractMISPObjectGenerator):
self.generate_attributes()
def generate_attributes(self):
self.add_attribute('type', value=self._moduleName)
self.add_attribute('origin', value=self._p_source, type='text')
self.add_attribute('last-seen', value=self._p_date)
self.add_attribute('last-seen', value=self._p_date, type='datetime')
if self._p_duplicate_number > 0:
self.add_attribute('duplicate', value=self._p_duplicate, type='text')
self.add_attribute('duplicate_number', value=self._p_duplicate_number, type='counter')
self._pseudofile = BytesIO(self._p_content)
self.add_attribute('raw-data', value=self._p_source, data=self._pseudofile, type="attachment")
self._pseudofile = BytesIO(self._p_content.encode())
res = self.add_attribute('raw-data', value=self._p_source, data=self._pseudofile, type="attachment")# , ShadowAttribute=self.p_tag)
#res.add_shadow_attributes(tag)
self.add_attribute('sensor', value=self._uuid, type="text")
class ObjectWrapper:
def __init__(self, pymisp):
@ -38,30 +38,40 @@ class ObjectWrapper:
cfg = configparser.ConfigParser()
cfg.read('./packages/config.cfg')
self.maxDuplicateToPushToMISP = cfg.getint("ailleakObject", "maxDuplicateToPushToMISP")
self.attribute_to_tag = None
def add_new_object(self, moduleName, path):
self.moduleName = moduleName
def add_new_object(self, uuid_ail, path, p_source, tag):
self.uuid_ail = uuid_ail
self.path = path
self.p_source = p_source
self.paste = Paste.Paste(path)
self.p_date = self.date_to_str(self.paste.p_date)
self.p_source = self.paste.p_path
self.p_content = self.paste.get_p_content()
self.p_tag = tag
temp = self.paste._get_p_duplicate()
#beautifier
temp = json.loads(temp)
self.p_duplicate_number = len(temp) if len(temp) >= 0 else 0
if not temp:
temp = ''
p_duplicate_number = len(temp) if len(temp) >= 0 else 0
to_ret = ""
for dup in temp[:self.maxDuplicateToPushToMISP]:
for dup in temp[:10]:
dup = dup.replace('\'','\"').replace('(','[').replace(')',']')
dup = json.loads(dup)
algo = dup[0]
path = dup[1].split('/')[-5:]
path = dup[1].split('/')[-6:]
path = '/'.join(path)[:-3] # -3 removes .gz
if algo == 'tlsh':
perc = 100 - int(dup[2])
else:
perc = dup[2]
to_ret += "{}: {} [{}%]\n".format(path, algo, perc)
self.p_duplicate = to_ret
p_duplicate = to_ret
self.mispObject = AilleakObject(self.moduleName, self.p_source, self.p_date, self.p_content, self.p_duplicate, self.p_duplicate_number)
self.mispObject = AilLeakObject(self.uuid_ail, self.p_source, self.p_date, self.p_content, p_duplicate, p_duplicate_number)
def date_to_str(self, date):
return "{0}-{1}-{2}".format(date.year, date.month, date.day)
@ -108,14 +118,30 @@ class ObjectWrapper:
event = self.pymisp.new_event(distribution, threat,
analysis, info, date,
published, orgc_id, org_id, sharing_group_id)
eventUuid = event['Event']['uuid']
self.pymisp.tag(eventUuid, 'infoleak:output-format="ail-daily"')
return event
# Publish object to MISP
def pushToMISP(self):
def pushToMISP(self, uuid_ail, path, tag):
self._p_source = path.split('/')[-5:]
self._p_source = '/'.join(self._p_source)[:-3]
if self.currentID_date != datetime.date.today(): #refresh id
self.eventID_to_push = self.get_daily_event_id()
mispTYPE = 'ail-leak'
# paste object already exist
if self.paste_object_exist(self.eventID_to_push, self._p_source):
# add new tag
self.tag(self.attribute_to_tag, tag)
print(self._p_source + ' tagged: ' + tag)
#create object
else:
self.add_new_object(uuid_ail, path, self._p_source, tag)
try:
templateID = [x['ObjectTemplate']['id'] for x in self.pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0]
except IndexError:
@ -125,4 +151,24 @@ class ObjectWrapper:
if 'errors' in r:
print(r)
else:
print('Pushed:', self.moduleName, '->', self.p_source)
# tag new object
self.set_attribute_to_tag_uuid(self.eventID_to_push, self._p_source)
self.tag(self.attribute_to_tag, tag)
print('Pushed:', tag, '->', self._p_source)
def paste_object_exist(self, eventId, source):
res = self.pymisp.search(controller='attributes', eventid=eventId, values=source)
# object already exist
if res['response']:
self.attribute_to_tag = res['response']['Attribute'][0]['uuid']
return True
# new object
else:
return False
def set_attribute_to_tag_uuid(self, eventId, source):
res = self.pymisp.search(controller='attributes', eventid=eventId, values=source)
self.attribute_to_tag = res['response']['Attribute'][0]['uuid']
def tag(self, uuid, tag):
self.pymisp.tag(uuid, tag)

View file

@ -20,15 +20,9 @@ from packages import Paste
from pubsublogger import publisher
from Helper import Process
from pymisp import PyMISP
import ailleakObject
import sys
sys.path.append('../')
try:
from mispKEYS import misp_url, misp_key, misp_verifycert
flag_misp = True
except:
print('Misp keys not present')
flag_misp = False
if __name__ == "__main__":
@ -38,16 +32,6 @@ if __name__ == "__main__":
config_section = 'alertHandler'
p = Process(config_section)
if flag_misp:
try:
pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
print('Connected to MISP:', misp_url)
except:
flag_misp = False
print('Not connected to MISP')
if flag_misp:
wrapper = ailleakObject.ObjectWrapper(pymisp)
# port generated automatically depending on the date
curYear = datetime.now().year
@ -77,12 +61,3 @@ if __name__ == "__main__":
server.sadd(key, p_path)
publisher.info('Saved warning paste {}'.format(p_path))
# Create MISP AIL-leak object and push it
if flag_misp:
allowed_modules = ['credential', 'phone', 'creditcards']
if module_name in allowed_modules:
wrapper.add_new_object(module_name, p_path)
wrapper.pushToMISP()
else:
print('not pushing to MISP:', module_name, p_path)

View file

@ -76,7 +76,7 @@ class Paste(object):
port=cfg.getint("Redis_Data_Merging", "port"),
db=cfg.getint("Redis_Data_Merging", "db"),
decode_responses=True)
self.store_duplicate = redis.StrictRedis(
self.store_metadata = redis.StrictRedis(
host=cfg.get("ARDB_Metadata", "host"),
port=cfg.getint("ARDB_Metadata", "port"),
db=cfg.getint("ARDB_Metadata", "db"),
@ -105,6 +105,7 @@ class Paste(object):
self.p_max_length_line = None
self.array_line_above_threshold = None
self.p_duplicate = None
self.p_tags = None
def get_p_content(self):
"""
@ -277,12 +278,19 @@ class Paste(object):
return False, var
def _get_p_duplicate(self):
self.p_duplicate = self.store_duplicate.smembers('dup:'+self.p_path)
self.p_duplicate = self.store_metadata.smembers('dup:'+self.p_path)
if self.p_duplicate is not None:
return list(self.p_duplicate)
else:
return '[]'
def _get_p_tags(self):
self.p_tags = self.store_metadata.smembers('tag:'+path, tag)
if self.self.p_tags is not None:
return list(self.p_tags)
else:
return '[]'
def save_all_attributes_redis(self, key=None):
"""
Saving all the attributes in a "Redis-like" Database (Redis, LevelDB)
@ -333,7 +341,7 @@ class Paste(object):
Save an attribute as a field
"""
for tuple in value:
self.store_duplicate.sadd('dup:'+self.p_path, tuple)
self.store_metadata.sadd('dup:'+self.p_path, tuple)
def save_others_pastes_attribute_duplicate(self, list_value):
"""
@ -341,7 +349,7 @@ class Paste(object):
"""
for hash_type, path, percent, date in list_value:
to_add = (hash_type, self.p_path, percent, date)
self.store_duplicate.sadd('dup:'+path,to_add)
self.store_metadata.sadd('dup:'+path,to_add)
def _get_from_redis(self, r_serv):
ans = {}

View file

@ -92,6 +92,11 @@ host = localhost
port = 6380
db = 0
[Redis_Log_submit]
host = localhost
port = 6380
db = 1
[Redis_Queues]
host = localhost
port = 6381
@ -157,6 +162,11 @@ host = localhost
port = 6382
db = 7
[ARDB_Statistics]
host = localhost
port = 6382
db = 8
[Url]
cc_critical = DE

View file

@ -82,6 +82,10 @@ subscribe = Redis_alertHandler
[Tags]
subscribe = Redis_Tags
publish = Redis_Tags_feed
[misp_the_hive_feeder]
subscribe = Redis_Tags_feed
#[send_to_queue]
#subscribe = Redis_Cve
@ -120,3 +124,6 @@ publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags
[Bitcoin]
subscribe = Redis_Global
publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags
[submit_paste]
publish = Redis_Mixer

274
bin/submit_paste.py Executable file
View file

@ -0,0 +1,274 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import configparser
import os
import sys
import gzip
import io
import redis
import base64
import datetime
import time
from sflock.main import unpack
import sflock
from Helper import Process
from pubsublogger import publisher
def create_paste(uuid, paste_content, ltags, ltagsgalaxies, name):
now = datetime.datetime.now()
save_path = 'submitted/' + now.strftime("%Y") + '/' + now.strftime("%m") + '/' + now.strftime("%d") + '/' + name + '.gz'
full_path = filename = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "pastes"), save_path)
if os.path.isfile(full_path):
addError(uuid, 'File: ' + save_path + ' already exist in submitted pastes')
return 1
gzipencoded = gzip.compress(paste_content)
gzip64encoded = base64.standard_b64encode(gzipencoded).decode()
# send paste to Global module
relay_message = "{0} {1}".format(save_path, gzip64encoded)
p.populate_set_out(relay_message, 'Mixer')
# add tags
add_tags(ltags, ltagsgalaxies, full_path)
r_serv_log_submit.incr(uuid + ':nb_end')
r_serv_log_submit.incr(uuid + ':nb_sucess')
if r_serv_log_submit.get(uuid + ':nb_end') == r_serv_log_submit.get(uuid + ':nb_total'):
r_serv_log_submit.set(uuid + ':end', 1)
print(' {} send to Global'.format(save_path))
r_serv_log_submit.sadd(uuid + ':paste_submit_link', full_path)
return 0
def addError(uuid, errorMessage):
print(errorMessage)
error = r_serv_log_submit.get(uuid + ':error')
if error != None:
r_serv_log_submit.set(uuid + ':error', error + '<br></br>' + errorMessage)
r_serv_log_submit.incr(uuid + ':nb_end')
def abord_file_submission(uuid, errorMessage):
addError(uuid, errorMessage)
r_serv_log_submit.set(uuid + ':end', 1)
remove_submit_uuid(uuid)
def remove_submit_uuid(uuid):
# save temp value on disk
r_serv_db.delete(uuid + ':ltags')
r_serv_db.delete(uuid + ':ltagsgalaxies')
r_serv_db.delete(uuid + ':paste_content')
r_serv_db.delete(uuid + ':isfile')
r_serv_db.delete(uuid + ':password')
r_serv_log_submit.expire(uuid + ':end', expire_time)
r_serv_log_submit.expire(uuid + ':processing', expire_time)
r_serv_log_submit.expire(uuid + ':nb_total', expire_time)
r_serv_log_submit.expire(uuid + ':nb_sucess', expire_time)
r_serv_log_submit.expire(uuid + ':nb_end', expire_time)
r_serv_log_submit.expire(uuid + ':error', expire_time)
r_serv_log_submit.srem(uuid + ':paste_submit_link', '')
r_serv_log_submit.expire(uuid + ':paste_submit_link', expire_time)
# delete uuid
r_serv_db.srem('submitted:uuid', uuid)
print('{} all file submitted'.format(uuid))
def add_tags(tags, tagsgalaxies, path):
list_tag = tags.split(',')
list_tag_galaxies = tagsgalaxies.split(',')
if list_tag != ['']:
for tag in list_tag:
#add tag
r_serv_metadata.sadd('tag:'+path, tag)
r_serv_tags.sadd(tag, path)
#add new tag in list of all used tags
r_serv_tags.sadd('list_tags', tag)
if list_tag_galaxies != ['']:
for tag in list_tag_galaxies:
#add tag
r_serv_metadata.sadd('tag:'+path, tag)
r_serv_tags.sadd(tag, path)
#add new tag in list of all used tags
r_serv_tags.sadd('list_tags', tag)
def verify_extention_filename(filename):
if not '.' in filename:
return True
else:
file_type = filename.rsplit('.', 1)[1]
#txt file
if file_type in ALLOWED_EXTENSIONS:
return True
else:
return False
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \
Did you set environment variables? \
Or activate the virtualenv.')
cfg = configparser.ConfigParser()
cfg.read(configfile)
r_serv_db = redis.StrictRedis(
host=cfg.get("ARDB_DB", "host"),
port=cfg.getint("ARDB_DB", "port"),
db=cfg.getint("ARDB_DB", "db"),
decode_responses=True)
r_serv_log_submit = redis.StrictRedis(
host=cfg.get("Redis_Log_submit", "host"),
port=cfg.getint("Redis_Log_submit", "port"),
db=cfg.getint("Redis_Log_submit", "db"),
decode_responses=True)
r_serv_tags = redis.StrictRedis(
host=cfg.get("ARDB_Tags", "host"),
port=cfg.getint("ARDB_Tags", "port"),
db=cfg.getint("ARDB_Tags", "db"),
decode_responses=True)
r_serv_metadata = redis.StrictRedis(
host=cfg.get("ARDB_Metadata", "host"),
port=cfg.getint("ARDB_Metadata", "port"),
db=cfg.getint("ARDB_Metadata", "db"),
decode_responses=True)
expire_time = 120
MAX_FILE_SIZE = 1000000000
ALLOWED_EXTENSIONS = ['txt', 'sh', 'pdf']
config_section = 'submit_paste'
p = Process(config_section)
while True:
# paste submitted
if r_serv_db.scard('submitted:uuid') > 0:
uuid = r_serv_db.srandmember('submitted:uuid')
# get temp value save on disk
ltags = r_serv_db.get(uuid + ':ltags')
ltagsgalaxies = r_serv_db.get(uuid + ':ltagsgalaxies')
paste_content = r_serv_db.get(uuid + ':paste_content')
isfile = r_serv_db.get(uuid + ':isfile')
password = r_serv_db.get(uuid + ':password')
# needed if redis is restarted
r_serv_log_submit.set(uuid + ':end', 0)
r_serv_log_submit.set(uuid + ':processing', 0)
r_serv_log_submit.set(uuid + ':nb_total', -1)
r_serv_log_submit.set(uuid + ':nb_end', 0)
r_serv_log_submit.set(uuid + ':nb_sucess', 0)
r_serv_log_submit.set(uuid + ':error', 'error:')
r_serv_log_submit.sadd(uuid + ':paste_submit_link', '')
r_serv_log_submit.set(uuid + ':processing', 1)
if isfile == 'True':
file_full_path = paste_content
if not os.path.exists(file_full_path):
abord_file_submission(uuid, "Server Error, the archive can't be found")
continue
#verify file lengh
if os.stat(file_full_path).st_size > MAX_FILE_SIZE:
abord_file_submission(uuid, 'File :{} too large'.format(file_full_path))
else:
filename = file_full_path.split('/')[-1]
if not '.' in filename:
# read file
try:
with open(file_full_path,'r') as f:
content = f.read()
except:
abord_file_submission(uuid, "file error")
continue
r_serv_log_submit.set(uuid + ':nb_total', 1)
create_paste(uuid, content, ltags, ltagsgalaxies, uuid)
remove_submit_uuid(uuid)
else:
file_type = filename.rsplit('.', 1)[1]
#txt file
if file_type in ALLOWED_EXTENSIONS:
with open(file_full_path,'r') as f:
content = f.read()
r_serv_log_submit.set(uuid + ':nb_total', 1)
create_paste(uuid, content.encode(), ltags, ltagsgalaxies, uuid)
remove_submit_uuid(uuid)
#compressed file
else:
#decompress file
try:
if password == '':
files = unpack(file_full_path.encode())
#print(files.children)
else:
try:
files = unpack(file_full_path.encode(), password=password.encode())
#print(files.children)
except sflock.exception.IncorrectUsageException:
abord_file_submission(uuid, "Wrong Password")
continue
except:
abord_file_submission(uuid, "file decompression error")
continue
print('unpacking {} file'.format(files.unpacker))
if(not files.children):
abord_file_submission(uuid, "Empty compressed file")
continue
# set number of files to submit
r_serv_log_submit.set(uuid + ':nb_total', len(files.children))
n = 1
for child in files.children:
if verify_extention_filename(child.filename.decode()):
create_paste(uuid, child.contents, ltags, ltagsgalaxies, uuid+'_'+ str(n) )
n = n + 1
else:
print('bad extention')
addError(uuid, 'Bad file extension: {}'.format(child.filename.decode()) )
except FileNotFoundError:
print('file not found')
addError(uuid, 'File not found: {}'.format(file_full_path), uuid )
remove_submit_uuid(uuid)
# textarea input paste
else:
r_serv_log_submit.set(uuid + ':nb_total', 1)
create_paste(uuid, paste_content.encode(), ltags, ltagsgalaxies, uuid)
remove_submit_uuid(uuid)
time.sleep(0.5)
# wait for paste
else:
publisher.debug("Script submit_paste is Idling 10s")
time.sleep(3)

View file

@ -0,0 +1,6 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
misp_url = 'http://10.1.0.143'
misp_key = 'c5VRXJahYVux3rkPULbbILTNmAwlXU8Eas9zvl36' # The MISP auth key can be found on the MISP web interface under the automation section
misp_verifycert = True

View file

@ -0,0 +1,6 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
the_hive_url = 'http://10.1.0.145:9000'
the_hive_key = 'KfiPGz3Bi8E5styWbC2eU4WiTKDGdf22' # The Hive auth key can be found on the The Hive web interface under the User Management
the_hive_verifycert = True

Binary file not shown.

After

Width:  |  Height:  |  Size: 28 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 82 KiB

View file

@ -33,6 +33,9 @@ sudo pip install nose
sudo apt-get install libfuzzy-dev -y
sudo apt-get install build-essential libffi-dev automake autoconf libtool -y
# sflock, gz requirement
sudo apt-get install p7zip-full -y
# REDIS #
test ! -d redis/ && git clone https://github.com/antirez/redis.git
pushd redis/

View file

@ -1,6 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
misp_url = ''
misp_key = '' # The MISP auth key can be found on the MISP web interface under the automation section
misp_verifycert = True

View file

@ -1,5 +1,7 @@
pymisp
thehive4py
redis
#filemagic conflict with magic
crcmod
@ -56,6 +58,9 @@ pycountry
# To fetch Onion urls
PySocks
# decompress files
sflock
#ASN lookup requirements
#https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/adns-python/adns-python-1.2.1.tar.gz
https://github.com/trolldbois/python3-adns/archive/master.zip

View file

@ -7,7 +7,7 @@ import json
import datetime
import time
import calendar
from flask import Flask, render_template, jsonify, request
from flask import Flask, render_template, jsonify, request, Request
import flask
import importlib
import os
@ -28,6 +28,7 @@ cfg = Flask_config.cfg
Flask_config.app = Flask(__name__, static_url_path='/static/')
app = Flask_config.app
app.config['MAX_CONTENT_LENGTH'] = 900 * 1024 * 1024
# ========= HEADER GENERATION ========
@ -134,6 +135,19 @@ for tag in taxonomies.get('gdpr').machinetags():
for tag in taxonomies.get('fpf').machinetags():
r_serv_tags.sadd('active_tag_fpf', tag)
# ========== INITIAL tags auto export ============
r_serv_db = redis.StrictRedis(
host=cfg.get("ARDB_DB", "host"),
port=cfg.getint("ARDB_DB", "port"),
db=cfg.getint("ARDB_DB", "db"),
decode_responses=True)
infoleak_tags = taxonomies.get('infoleak').machinetags()
infoleak_automatic_tags = []
for tag in taxonomies.get('infoleak').machinetags():
if tag.split('=')[0][:] == 'infoleak:automatic-detection':
r_serv_db.sadd('list_export_tags', tag)
r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"')
# ============ MAIN ============
if __name__ == "__main__":

View file

@ -7,9 +7,11 @@
import configparser
import redis
import os
import sys
# FLASK #
app = None
#secret_key = 'ail-super-secret_key01C'
# CONFIG #
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
@ -21,7 +23,6 @@ if not os.path.exists(configfile):
cfg = configparser.ConfigParser()
cfg.read(configfile)
# REDIS #
r_serv = redis.StrictRedis(
host=cfg.get("Redis_Queues", "host"),
@ -35,6 +36,12 @@ r_serv_log = redis.StrictRedis(
db=cfg.getint("Redis_Log", "db"),
decode_responses=True)
r_serv_log_submit = redis.StrictRedis(
host=cfg.get("Redis_Log_submit", "host"),
port=cfg.getint("Redis_Log_submit", "port"),
db=cfg.getint("Redis_Log_submit", "db"),
decode_responses=True)
r_serv_charts = redis.StrictRedis(
host=cfg.get("ARDB_Trending", "host"),
port=cfg.getint("ARDB_Trending", "port"),
@ -77,6 +84,56 @@ r_serv_metadata = redis.StrictRedis(
db=cfg.getint("ARDB_Metadata", "db"),
decode_responses=True)
r_serv_db = redis.StrictRedis(
host=cfg.get("ARDB_DB", "host"),
port=cfg.getint("ARDB_DB", "port"),
db=cfg.getint("ARDB_DB", "db"),
decode_responses=True)
r_serv_statistics = redis.StrictRedis(
host=cfg.get("ARDB_Statistics", "host"),
port=cfg.getint("ARDB_Statistics", "port"),
db=cfg.getint("ARDB_Statistics", "db"),
decode_responses=True)
sys.path.append('../../configs/keys')
# MISP #
try:
from pymisp import PyMISP
from mispKEYS import misp_url, misp_key, misp_verifycert
pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
misp_event_url = misp_url + '/events/view/'
print('Misp connected')
except:
print('Misp not connected')
pymisp = False
misp_event_url = '#'
# The Hive #
try:
from thehive4py.api import TheHiveApi
import thehive4py.exceptions
from theHiveKEYS import the_hive_url, the_hive_key, the_hive_verifycert
if the_hive_url == '':
HiveApi = False
hive_case_url = '#'
print('The HIVE not connected')
else:
HiveApi = TheHiveApi(the_hive_url, the_hive_key, cert=the_hive_verifycert)
hive_case_url = the_hive_url+'/index.html#/case/id_here/details'
except:
print('The HIVE not connected')
HiveApi = False
hive_case_url = '#'
if HiveApi != False:
try:
HiveApi.get_alert(0)
print('The Hive connected')
except thehive4py.exceptions.AlertException:
HiveApi = False
print('The Hive not connected')
# VARIABLES #
max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
@ -84,3 +141,5 @@ max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number
DiffMaxLineLength = int(cfg.get("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')

View file

@ -0,0 +1,591 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Flask functions and routes for the trending modules page
'''
import redis
from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect
import unicodedata
import string
import subprocess
import os
import sys
import datetime
import uuid
from io import BytesIO
from Date import Date
import json
import Paste
from pytaxonomies import Taxonomies
from pymispgalaxies import Galaxies, Clusters
try:
from pymisp.mispevent import MISPObject
flag_misp = True
except:
flag_misp = False
try:
from thehive4py.models import Case, CaseTask, CustomFieldHelper, CaseObservable
flag_hive = True
except:
flag_hive = False
# ============ VARIABLES ============
import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
r_serv_tags = Flask_config.r_serv_tags
r_serv_metadata = Flask_config.r_serv_metadata
r_serv_db = Flask_config.r_serv_db
r_serv_log_submit = Flask_config.r_serv_log_submit
pymisp = Flask_config.pymisp
if pymisp is False:
flag_misp = False
HiveApi = Flask_config.HiveApi
if HiveApi is False:
flag_hive = False
PasteSubmit = Blueprint('PasteSubmit', __name__, template_folder='templates')
valid_filename_chars = "-_ %s%s" % (string.ascii_letters, string.digits)
ALLOWED_EXTENSIONS = set(['txt', 'sh', 'pdf', 'zip', 'gz', 'tar.gz'])
UPLOAD_FOLDER = Flask_config.UPLOAD_FOLDER
misp_event_url = Flask_config.misp_event_url
hive_case_url = Flask_config.hive_case_url
# ============ FUNCTIONS ============
def one():
return 1
def allowed_file(filename):
if not '.' in filename:
return True
else:
return filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
def clean_filename(filename, whitelist=valid_filename_chars, replace=' '):
# replace characters
for r in replace:
filename = filename.replace(r,'_')
# keep only valid ascii chars
cleaned_filename = unicodedata.normalize('NFKD', filename).encode('ASCII', 'ignore').decode()
# keep only whitelisted chars
return ''.join(c for c in cleaned_filename if c in whitelist)
def launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password, isfile = False):
# save temp value on disk
r_serv_db.set(UUID + ':ltags', ltags)
r_serv_db.set(UUID + ':ltagsgalaxies', ltagsgalaxies)
r_serv_db.set(UUID + ':paste_content', paste_content)
r_serv_db.set(UUID + ':password', password)
r_serv_db.set(UUID + ':isfile', isfile)
r_serv_log_submit.set(UUID + ':end', 0)
r_serv_log_submit.set(UUID + ':processing', 0)
r_serv_log_submit.set(UUID + ':nb_total', -1)
r_serv_log_submit.set(UUID + ':nb_end', 0)
r_serv_log_submit.set(UUID + ':nb_sucess', 0)
r_serv_log_submit.set(UUID + ':error', 'error:')
r_serv_log_submit.sadd(UUID + ':paste_submit_link', '')
# save UUID on disk
r_serv_db.sadd('submitted:uuid', UUID)
def addTagsVerification(tags, tagsgalaxies):
list_tag = tags.split(',')
list_tag_galaxies = tagsgalaxies.split(',')
taxonomies = Taxonomies()
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
active_galaxies = r_serv_tags.smembers('active_galaxies')
if list_tag != ['']:
for tag in list_tag:
# verify input
tax = tag.split(':')[0]
if tax in active_taxonomies:
if tag in r_serv_tags.smembers('active_tag_' + tax):
pass
else:
return False
else:
return False
if list_tag_galaxies != ['']:
for tag in list_tag_galaxies:
# verify input
gal = tag.split(':')[1]
gal = gal.split('=')[0]
if gal in active_galaxies:
if tag in r_serv_tags.smembers('active_tag_galaxies_' + gal):
pass
else:
return False
else:
return False
return True
def date_to_str(date):
return "{0}-{1}-{2}".format(date.year, date.month, date.day)
def misp_create_event(distribution, threat_level_id, analysis, info, l_tags, publish, path):
paste = Paste.Paste(path)
source = path.split('/')[-6:]
source = '/'.join(source)[:-3]
ail_uuid = r_serv_db.get('ail:uuid')
pseudofile = BytesIO(paste.get_p_content().encode())
temp = paste._get_p_duplicate()
#beautifier
if not temp:
temp = ''
p_duplicate_number = len(temp) if len(temp) >= 0 else 0
to_ret = ""
for dup in temp[:10]:
dup = dup.replace('\'','\"').replace('(','[').replace(')',']')
dup = json.loads(dup)
algo = dup[0]
path = dup[1].split('/')[-6:]
path = '/'.join(path)[:-3] # -3 removes .gz
if algo == 'tlsh':
perc = 100 - int(dup[2])
else:
perc = dup[2]
to_ret += "{}: {} [{}%]\n".format(path, algo, perc)
p_duplicate = to_ret
today = datetime.date.today()
# [0-3]
if publish == 'True':
published = True
else:
published = False
org_id = None
orgc_id = None
sharing_group_id = None
date = today
event = pymisp.new_event(distribution, threat_level_id,
analysis, info, date,
published, orgc_id, org_id, sharing_group_id)
eventUuid = event['Event']['uuid']
eventid = event['Event']['id']
r_serv_metadata.set('misp_events:'+path, eventid)
# add tags
for tag in l_tags:
pymisp.tag(eventUuid, tag)
# create attributes
obj_name = 'ail-leak'
leak_obj = MISPObject(obj_name)
leak_obj.add_attribute('sensor', value=ail_uuid, type="text")
leak_obj.add_attribute('origin', value=source, type='text')
leak_obj.add_attribute('last-seen', value=date_to_str(paste.p_date), type='datetime')
leak_obj.add_attribute('raw-data', value=source, data=pseudofile, type="attachment")
if p_duplicate_number > 0:
leak_obj.add_attribute('duplicate', value=p_duplicate, type='text')
leak_obj.add_attribute('duplicate_number', value=p_duplicate_number, type='counter')
try:
templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == obj_name][0]
except IndexError:
valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()])
print ("Template for type {} not found! Valid types are: {%s}".format(obj_name, valid_types))
r = pymisp.add_object(eventid, templateID, leak_obj)
if 'errors' in r:
print(r)
return False
else:
event_url = misp_event_url + eventid
return event_url
def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title, l_tags, path):
ail_uuid = r_serv_db.get('ail:uuid')
source = path.split('/')[-6:]
source = '/'.join(source)[:-3]
# get paste date
var = path.split('/')
last_seen = "{0}-{1}-{2}".format(var[-4], var[-3], var[-2])
case = Case(title=hive_case_title,
tlp=hive_tlp,
severity=threat_level,
flag=False,
tags=l_tags,
description='hive_description')
# Create the case
id = None
response = HiveApi.create_case(case)
if response.status_code == 201:
id = response.json()['id']
observ_sensor = CaseObservable(dataType="other", data=[ail_uuid], message="sensor")
observ_file = CaseObservable(dataType="file", data=[path], tags=l_tags)
observ_source = CaseObservable(dataType="other", data=[source], message="source")
observ_last_seen = CaseObservable(dataType="other", data=[last_seen], message="last-seen")
res = HiveApi.create_case_observable(id,observ_sensor)
if res.status_code != 201:
print('ko: {}/{}'.format(res.status_code, res.text))
res = HiveApi.create_case_observable(id, observ_source)
if res.status_code != 201:
print('ko: {}/{}'.format(res.status_code, res.text))
res = HiveApi.create_case_observable(id, observ_file)
if res.status_code != 201:
print('ko: {}/{}'.format(res.status_code, res.text))
res = HiveApi.create_case_observable(id, observ_last_seen)
if res.status_code != 201:
print('ko: {}/{}'.format(res.status_code, res.text))
r_serv_metadata.set('hive_cases:'+path, id)
return hive_case_url.replace('id_here', id)
else:
print('ko: {}/{}'.format(response.status_code, response.text))
return False
# ============= ROUTES ==============
@PasteSubmit.route("/PasteSubmit/", methods=['GET'])
def PasteSubmit_page():
#active taxonomies
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
#active galaxies
active_galaxies = r_serv_tags.smembers('active_galaxies')
return render_template("PasteSubmit.html",
active_taxonomies = active_taxonomies,
active_galaxies = active_galaxies)
@PasteSubmit.route("/PasteSubmit/submit", methods=['POST'])
def submit():
#paste_name = request.form['paste_name']
password = request.form['password']
ltags = request.form['tags_taxonomies']
ltagsgalaxies = request.form['tags_galaxies']
paste_content = request.form['paste_content']
submitted_tag = 'infoleak:submission="manual"'
if ltags or ltagsgalaxies:
if not addTagsVerification(ltags, ltagsgalaxies):
content = 'INVALID TAGS'
return content, 400
# add submitted tags
if(ltags != ''):
ltags = ltags + ',' + submitted_tag
else:
ltags = submitted_tag
if 'file' in request.files:
file = request.files['file']
if file:
if file and allowed_file(file.filename):
# get UUID
UUID = str(uuid.uuid4())
'''if paste_name:
# clean file name
UUID = clean_filename(paste_name)'''
# create submitted dir
if not os.path.exists(UPLOAD_FOLDER):
os.makedirs(UPLOAD_FOLDER)
if not '.' in file.filename:
full_path = os.path.join(UPLOAD_FOLDER, UUID)
else:
if file.filename[-6:] == 'tar.gz':
file_type = 'tar.gz'
else:
file_type = file.filename.rsplit('.', 1)[1]
name = UUID + '.' + file_type
full_path = os.path.join(UPLOAD_FOLDER, name)
#Flask verify the file size
file.save(full_path)
paste_content = full_path
launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password ,True)
return render_template("submiting.html",
UUID = UUID)
else:
content = 'wrong file type, allowed_extensions: sh, pdf, zip, gz, tar.gz or remove the extension'
return content, 400
elif paste_content != '':
if sys.getsizeof(paste_content) < 900000:
# get id
UUID = str(uuid.uuid4())
#if paste_name:
# clean file name
#id = clean_filename(paste_name)
launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password)
return render_template("submiting.html",
UUID = UUID)
else:
content = 'size error'
return content, 400
content = 'submit aborded'
return content, 400
return PasteSubmit_page()
@PasteSubmit.route("/PasteSubmit/submit_status", methods=['GET'])
def submit_status():
UUID = request.args.get('UUID')
if UUID:
end = r_serv_log_submit.get(UUID + ':end')
nb_total = r_serv_log_submit.get(UUID + ':nb_total')
nb_end = r_serv_log_submit.get(UUID + ':nb_end')
error = r_serv_log_submit.get(UUID + ':error')
processing = r_serv_log_submit.get(UUID + ':processing')
nb_sucess = r_serv_log_submit.get(UUID + ':nb_sucess')
paste_submit_link = list(r_serv_log_submit.smembers(UUID + ':paste_submit_link'))
if (end != None) and (nb_total != None) and (nb_end != None) and (error != None) and (processing != None) and (paste_submit_link != None):
link = ''
if paste_submit_link:
for paste in paste_submit_link:
url = url_for('showsavedpastes.showsavedpaste') + '?paste=' + paste
link += '<a target="_blank" href="' + url + '" class="list-group-item">' + paste +'</a>'
if nb_total == '-1':
in_progress = nb_sucess + ' / '
else:
in_progress = nb_sucess + ' / ' + nb_total
if int(nb_total) != 0:
prog = int(int(nb_end) * 100 / int(nb_total))
else:
prog = 0
if error == 'error:':
isError = False
else:
isError = True
if end == '0':
end = False
else:
end = True
if processing == '0':
processing = False
else:
processing = True
return jsonify(end=end,
in_progress=in_progress,
prog=prog,
link=link,
processing=processing,
isError=isError,
error=error)
else:
# FIXME TODO
print(end)
print(nb_total)
print(nb_end)
print(error)
print(processing)
print(nb_sucess)
return 'to do'
else:
return 'INVALID UUID'
@PasteSubmit.route("/PasteSubmit/create_misp_event", methods=['POST'])
def create_misp_event():
distribution = int(request.form['misp_data[Event][distribution]'])
threat_level_id = int(request.form['misp_data[Event][threat_level_id]'])
analysis = int(request.form['misp_data[Event][analysis]'])
info = request.form['misp_data[Event][info]']
path = request.form['paste']
publish = request.form.get('misp_publish')
#verify input
if (0 <= distribution <= 3) and (1 <= threat_level_id <= 4) and (0 <= analysis <= 2):
l_tags = list(r_serv_metadata.smembers('tag:'+path))
event = misp_create_event(distribution, threat_level_id, analysis, info, l_tags, publish, path)
if event != False:
return redirect(event)
else:
return 'error, event creation'
return 'error0'
@PasteSubmit.route("/PasteSubmit/create_hive_case", methods=['POST'])
def create_hive_case():
hive_tlp = int(request.form['hive_tlp'])
threat_level = int(request.form['threat_level_hive'])
hive_description = request.form['hive_description']
hive_case_title = request.form['hive_case_title']
path = request.form['paste']
#verify input
if (0 <= hive_tlp <= 3) and (1 <= threat_level <= 4):
l_tags = list(r_serv_metadata.smembers('tag:'+path))
case = hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title, l_tags, path)
if case != False:
return redirect(case)
else:
return 'error'
return 'error'
@PasteSubmit.route("/PasteSubmit/edit_tag_export")
def edit_tag_export():
misp_auto_events = r_serv_db.get('misp:auto-events')
hive_auto_alerts = r_serv_db.get('hive:auto-alerts')
whitelist_misp = r_serv_db.scard('whitelist_misp')
whitelist_hive = r_serv_db.scard('whitelist_hive')
list_export_tags = list(r_serv_db.smembers('list_export_tags'))
status_misp = []
status_hive = []
for tag in list_export_tags:
if r_serv_db.sismember('whitelist_misp', tag):
status_misp.append(True)
else:
status_misp.append(False)
# empty whitelist
if whitelist_hive == 0:
for tag in list_export_tags:
status_hive.append(True)
else:
for tag in list_export_tags:
if r_serv_db.sismember('whitelist_hive', tag):
status_hive.append(True)
else:
status_hive.append(False)
if (misp_auto_events is not None) and (hive_auto_alerts is not None):
if int(misp_auto_events) == 1:
misp_active = True
else:
misp_active = False
if int(hive_auto_alerts) == 1:
hive_active = True
else:
hive_active = False
else:
misp_active = False
hive_active = False
nb_tags = str(r_serv_db.scard('list_export_tags'))
nb_tags_whitelist_misp = str(r_serv_db.scard('whitelist_misp')) + ' / ' + nb_tags
nb_tags_whitelist_hive = str(r_serv_db.scard('whitelist_hive')) + ' / ' + nb_tags
return render_template("edit_tag_export.html",
misp_active=misp_active,
hive_active=hive_active,
list_export_tags=list_export_tags,
status_misp=status_misp,
status_hive=status_hive,
nb_tags_whitelist_misp=nb_tags_whitelist_misp,
nb_tags_whitelist_hive=nb_tags_whitelist_hive,
flag_misp=flag_misp,
flag_hive=flag_hive)
@PasteSubmit.route("/PasteSubmit/tag_export_edited", methods=['POST'])
def tag_export_edited():
tag_enabled_misp = request.form.getlist('tag_enabled_misp')
tag_enabled_hive = request.form.getlist('tag_enabled_hive')
list_export_tags = list(r_serv_db.smembers('list_export_tags'))
r_serv_db.delete('whitelist_misp')
r_serv_db.delete('whitelist_hive')
for tag in tag_enabled_misp:
if r_serv_db.sismember('list_export_tags', tag):
r_serv_db.sadd('whitelist_misp', tag)
else:
return 'invalid input'
for tag in tag_enabled_hive:
if r_serv_db.sismember('list_export_tags', tag):
r_serv_db.sadd('whitelist_hive', tag)
else:
return 'invalid input'
return redirect(url_for('PasteSubmit.edit_tag_export'))
@PasteSubmit.route("/PasteSubmit/enable_misp_auto_event")
def enable_misp_auto_event():
r_serv_db.set('misp:auto-events', 1)
return edit_tag_export()
@PasteSubmit.route("/PasteSubmit/disable_misp_auto_event")
def disable_misp_auto_event():
r_serv_db.set('misp:auto-events', 0)
return edit_tag_export()
@PasteSubmit.route("/PasteSubmit/enable_hive_auto_alert")
def enable_hive_auto_alert():
r_serv_db.set('hive:auto-alerts', 1)
return edit_tag_export()
@PasteSubmit.route("/PasteSubmit/disable_hive_auto_alert")
def disable_hive_auto_alert():
r_serv_db.set('hive:auto-alerts', 0)
return edit_tag_export()
# ========= REGISTRATION =========
app.register_blueprint(PasteSubmit)

View file

@ -0,0 +1,200 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.pie.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="/static//js/bootstrap.min.js"></script>
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
</head>
<body>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<!-- /.col-lg-12 -->
<form action="/PasteSubmit/submit" id="pasteSubmitForm" method="post" enctype=multipart/form-data onsubmit="submitPaste()">
<input type="hidden" id="tags_taxonomies" name="tags_taxonomies" value="test">
<input type="hidden" id="tags_galaxies" name="tags_galaxies" value="test">
<div class="panel panel-primary pull-left" style="width: 40%;">
<div class="panel-heading">Files submission
</div>
<div class="panel-body">
<div class="form-group">
<label for="file">Submit a file </label>
<input type="file" class="form-control-file" id="file" name="file">
</div>
<div class="form-group">
<label for="paste_name">Archive Password</label>
<input type="password" class="form-control" id="password" name="password" placeholder="Optionnal">
</div>
</div>
</div>
<div class="panel panel-info pull-right" style="width: auto;">
<div class="panel-heading"> Tags :
<ul class="list-group">
<li class="list-group-item">
<div class="form-group input-group" >
<input id="ltags" style="width:450%;" type="text">
</div>
<div class="btn-group btn-block">
<button type="button" class="btn btn-primary dropdown-toggle btn-block" data-toggle="dropdown">Taxonomie Selection
<i class="fa fa-chevron-down"></i>
</button>
<ul class="dropdown-menu scrollable-menu" role="menu">
<li><a href="#" id="all-tags-taxonomies">All Tags <i class="fa fa-tags"></i></a></li>
<li role="separator" class="divider"></li>
{% for taxo in active_taxonomies %}
<li><a href="#" id="{{ taxo }}-id{{ loop.index0 }}">{{ taxo }}</a></li>
{% endfor %}
</ul>
</div>
</li>
<li class="list-group-item">
<div class="form-group input-group" >
<input id="ltagsgalaxies" style="width:450%;" type="text">
</div>
<div class="btn-group btn-block">
<button type="button" class="btn btn-primary dropdown-toggle btn-block" data-toggle="dropdown">Galaxy Selection
<i class="fa fa-chevron-down"></i>
</button>
<ul class="dropdown-menu scrollable-menu" role="menu">
<li><a href="#" id="all-tags-galaxies">All Tags <i class="fa fa-tags"></i></a></li>
<li role="separator" class="divider"></li>
{% for galaxy in active_galaxies %}
<li><a href="#" id="{{ galaxy }}-idgalax{{ loop.index0 }}">{{ galaxy }}</a></li>
{% endfor %}
</ul>
</div>
</li>
</ul>
</div>
</div>
<div>
<div class="form-group">
<textarea class="form-control" id="paste_content" name="paste_content" rows="25"></textarea>
</div>
<div class="form-group">
<button class="btn btn-primary " name="submit" type="submit">Submit this paste</button>
</div>
</form>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
</div>
</body>
<script>
var ltags
var ltagsgalaxies
$(document).ready(function(){
$.getJSON('/Tags/get_all_tags_taxonomies',
function(data) {
ltags = $('#ltags').tagSuggest({
data: data,
maxDropHeight: 200,
name: 'ltags'
});
});
$.getJSON('/Tags/get_all_tags_galaxies',
function(data) {
ltagsgalaxies = $('#ltagsgalaxies').tagSuggest({
data: data,
maxDropHeight: 200,
name: 'ltagsgalaxies'
});
});
activePage = "page-PasteSubmit"
$("#"+activePage).addClass("active");
$('#modalsubmit').modal({backdrop: 'static'})
});
</script>
<script>
function submitPaste(){
document.getElementById("tags_taxonomies").value = ltags.getValue();
document.getElementById("tags_galaxies").value = ltagsgalaxies.getValue();
}
</script>
<script>
jQuery("#all-tags-taxonomies").click(function(e){
//change input tags list
$.getJSON('/Tags/get_all_tags_taxonomies',
function(data) {
ltags.setData(data)
});
});
</script>
<script>
jQuery("#all-tags-galaxies").click(function(e){
$.getJSON('/Tags/get_all_tags_galaxies',
function(data) {
ltagsgalaxies.setData(data)
});
});
{% for taxo in active_taxonomies %}
jQuery("#{{ taxo }}-id{{ loop.index0 }}").click(function(e){
$.getJSON('/Tags/get_tags_taxonomie?taxonomie={{ taxo }}',
function(data) {
ltags.setData(data)
});
});
{% endfor %}
</script>
<script>
{% for galaxy in active_galaxies %}
jQuery("#{{ galaxy }}-idgalax{{ loop.index0 }}").click(function(e){
$.getJSON('/Tags/get_tags_galaxy?galaxy={{ galaxy }}',
function(data) {
ltagsgalaxies.setData(data)
});
});
{% endfor %}
</script>
</html>

View file

@ -0,0 +1,281 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="/static//css/dataTables.bootstrap.css" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="/static//js/bootstrap.min.js"></script>
<script src="/static//js/jquery.dataTables.min.js"></script>
<script src="/static//js/dataTables.bootstrap.js"></script>
<style>
.tooltip-inner {
text-align: left;
height: 200%;
width: 200%;
max-width: 500px;
max-height: 500px;
font-size: 13px;
}
xmp {
white-space:pre-wrap;
word-wrap:break-word;
}
.test thead{
background: #d91f2d;
color: #fff;
}
</style>
</head>
<body>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="container">
<div class="row">
<div class="col-md-6">
<div class="panel panel-primary">
<div class="panel-heading"> MISP Auto Event Creation
{% if misp_active %}
<span class="label label-success pull-right"> Enabled</span>
{% endif %}
{% if not misp_active %}
<span class="label label-danger pull-right"> Disabled</span>
{% endif %}
</div>
<div class="panel-body">
<div class="text-center">
<img id="misp-logo" src="{{url_for('static', filename='image/misp-logo.png') }}" class="text-center" >
</div>
<br>
{% if flag_misp %}
{% if misp_active %}
<a href="{{ url_for('PasteSubmit.disable_misp_auto_event') }}" class="btn btn-danger pull-right">
<i class="fa fa-times fa"></i> Disable Event Creation
</a>
{% endif %}
{% if not misp_active %}
<a href="{{ url_for('PasteSubmit.enable_misp_auto_event') }}" class="btn btn-success pull-right">
<i class="fa fa-check-square-o fa"></i> Enable Event Creation
</a>
{% endif %}
{% endif %}
{% if not flag_misp %}
<button class="btn btn-danger pull-right disabled">
<i class="fa fa-times fa"></i> MISP is not connected
</button>
{% endif %}
</div>
</div>
</div>
<div class="col-md-6">
<div class="panel panel-primary" style="min-width: 500px;">
<div class="panel-heading">The hive auto export
{% if hive_active %}
<span class="label label-success pull-right"> Enabled</span>
{% endif %}
{% if not hive_active %}
<span class="label label-danger pull-right"> Disabled</span>
{% endif %}
</div>
<div class="panel-body">
<div class="text-center">
<img id="misp-logo" src="{{url_for('static', filename='image/thehive-logo.png') }}" width="500">
</div>
{% if flag_hive %}
{% if hive_active %}
<a href="{{ url_for('PasteSubmit.disable_hive_auto_alert') }}" class="btn btn-danger pull-right">
<i class="fa fa-times fa"></i> Disable Alert Creation
</a>
{% endif %}
{% if not hive_active %}
<a href="{{ url_for('PasteSubmit.enable_hive_auto_alert') }}" class="btn btn-success pull-right">
<i class="fa fa-check-square-o fa"></i> Enable Alert Creation
</a>
{% endif %}
{% endif %}
{% if not flag_hive %}
<button class="btn btn-danger pull-right disabled">
<i class="fa fa-times fa"></i> The Hive is not connected
</button>
{% endif %}
</div>
</div>
</div>
</div>
</div>
<form action="/PasteSubmit/tag_export_edited" id="checkboxForm" method='post'>
<div class="container">
<div class="row">
<div class="col-md-6">
<div class="panel panel-info">
<div class="panel-heading">
Metadata :
<span class="pull-right">&nbsp;&nbsp;&nbsp;</span>
<span class="badge pull-right">{{ nb_tags_whitelist_misp }}</span>
<ul class="list-group"><li class="list-group-item">
<table class="test table table-striped table-bordered table-hover table-responsive " id="myTable_misp">
<thead>
<tr>
<th>Whitelist</th>
<th>Tag</th>
</tr>
</thead>
<tbody>
{% for tag in list_export_tags %}
<tr>
<td>
{% if status_misp[loop.index0] %}
<div style="display:none;">Enabled</div>
<input type="checkbox" value="{{ tag }}" name="tag_enabled_misp" checked>
{% endif %}
{% if not status_misp[loop.index0] %}
<div style="display:none;">Disabled</div>
<input type="checkbox" value="{{ tag }}" name="tag_enabled_misp" >
{% endif %}
</td>
<td>{{ tag }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</li></ul>
</div></div>
</div>
<div class="col-md-6">
<div class="panel panel-info">
<div class="panel-heading">
Metadata :
<span class="pull-right">&nbsp;&nbsp;&nbsp;</span>
<span class="badge pull-right">{{ nb_tags_whitelist_hive }}</span>
<ul class="list-group"><li class="list-group-item">
<table class="test table table-striped table-bordered table-hover table-responsive " id="myTable_hive">
<thead>
<tr>
<th>Whitelist</th>
<th>Tag</th>
</tr>
</thead>
<tbody>
{% for tag in list_export_tags %}
<tr>
<td>
{% if status_hive[loop.index0] %}
<div style="display:none;">Enabled</div>
<input type="checkbox" value="{{ tag }}" name="tag_enabled_hive" checked>
{% endif %}
{% if not status_hive[loop.index0] %}
<div style="display:none;">Disabled</div>
<input type="checkbox" value="{{ tag }}" name="tag_enabled_hive" >
{% endif %}
</td>
<td>{{ tag }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</li></ul>
</div></div>
</div>
</div>
</div>
<div class="input-group-btn text-center">
<button class="btn btn-primary btn-lg" onclick="submitActiveTags()">
<i class="fa fa-check-square-o fa"></i>
Update Tags
</button>
</div>
</form>
</div>
<!-- /#page-wrapper -->
</body>
<script>
var table_misp
var table_hive
$(document).ready(function(){
table_misp = $('#myTable_misp').DataTable(
{
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 1, "asc" ]]
}
);
table_hive = $('#myTable_hive').DataTable(
{
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 1, "asc" ]]
}
);
});
</script>
<script>
function submitActiveTags(){
table_misp.destroy()
table_misp = $('#myTable_misp').DataTable(
{
"iDisplayLength": -1,
}
);
table_hive.destroy()
table_hive = $('#myTable_hive').DataTable(
{
"iDisplayLength": -1,
}
);
}
</script>
</html>

View file

@ -0,0 +1 @@
<li id='page-PasteSubmit'><a href="{{ url_for('PasteSubmit.PasteSubmit_page') }}"><i class="glyphicon glyphicon-new-window white"></i> PasteSubmit </a></li>

View file

@ -0,0 +1,360 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.pie.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="/static//js/bootstrap.min.js"></script>
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
<style>
.glyphicon-refresh-animate {
-animation: spin .7s infinite linear;
-webkit-animation: spin2 .7s infinite linear;
}
@-webkit-keyframes spin2 {
from { -webkit-transform: rotate(0deg);}
to { -webkit-transform: rotate(360deg);}
}
@keyframes spin {
from { transform: scale(1) rotate(0deg);}
to { transform: scale(1) rotate(360deg);}
}
</style>
</head>
<body>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<!-- /.col-lg-12 -->
<form action="/PasteSubmit/submit" id="pasteSubmitForm" method="post" enctype=multipart/form-data onsubmit="submitPaste()">
<input type="hidden" id="tags_taxonomies" name="tags_taxonomies" value="test">
<input type="hidden" id="tags_galaxies" name="tags_galaxies" value="test">
<div class="panel panel-primary pull-left" style="width: 40%;">
<div class="panel-heading">Files submission
</div>
<div class="panel-body">
<div class="form-group">
<label for="file">Submit a file</label>
<input type="file" class="form-control-file" id="file" name="file">
</div>
<div class="form-group">
<label for="paste_name">Archive Password</label>
<input type="password" class="form-control" id="password" name="password" placeholder="Optionnal">
</div>
</div>
</div>
<div class="panel panel-info pull-right" style="width: auto;">
<div class="panel-heading"> Tags :
<ul class="list-group">
<li class="list-group-item">
<div class="form-group input-group" >
<input id="ltags" style="width:450%;" type="text">
</div>
<div class="btn-group btn-block">
<button type="button" class="btn btn-primary dropdown-toggle btn-block" data-toggle="dropdown">Taxonomie Selection
<i class="fa fa-chevron-down"></i>
</button>
<ul class="dropdown-menu scrollable-menu" role="menu">
<li><a href="#" id="all-tags-taxonomies">All Tags <i class="fa fa-tags"></i></a></li>
<li role="separator" class="divider"></li>
{% for taxo in active_taxonomies %}
<li><a href="#" id="{{ taxo }}-id{{ loop.index0 }}">{{ taxo }}</a></li>
{% endfor %}
</ul>
</div>
</li>
<li class="list-group-item">
<div class="form-group input-group" >
<input id="ltagsgalaxies" style="width:450%;" type="text">
</div>
<div class="btn-group btn-block">
<button type="button" class="btn btn-primary dropdown-toggle btn-block" data-toggle="dropdown">Galaxy Selection
<i class="fa fa-chevron-down"></i>
</button>
<ul class="dropdown-menu scrollable-menu" role="menu">
<li><a href="#" id="all-tags-galaxies">All Tags <i class="fa fa-tags"></i></a></li>
<li role="separator" class="divider"></li>
{% for galaxy in active_galaxies %}
<li><a href="#" id="{{ galaxy }}-idgalax{{ loop.index0 }}">{{ galaxy }}</a></li>
{% endfor %}
</ul>
</div>
</li>
</ul>
</div>
</div>
<div>
<div class="form-group">
<textarea class="form-control" id="paste_content" name="paste_content" rows="25"></textarea>
</div>
<div class="form-group">
<button class="btn btn-primary " name="submit" type="submit">Submit this paste</button>
</div>
</form>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
</div>
<div id="modalsubmit" class="modal fade" role="dialog">
<div class="modal-dialog modal-lg">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div class="modal-header" style="border-bottom: 4px solid #cccccc; background-color: #cccccc; color: #ffffff;">
<p class="heading"><h1>Submitting Pastes ...</h1></p>
</div>
<div class="modal-body">
<br></br>
<div class="progress hidden" id="pr">
<div class="progress-bar progress-bar-striped active" role="progressbar"
aria-valuenow="0" aria-valuemin="0" aria-valuemax="100" style="width:0%" id="progress-bar">
0%
</div>
</div>
<span class="btn btn-info hidden" id="files_submitted_btn">
Files Submitted <span class="badge badge-light" id="files_submitted">1 / 1</span>
</span>
<button class="btn btn-lg btn-warning" style="margin: auto;" id="loading_button">
<span class="glyphicon glyphicon-refresh glyphicon-refresh-animate">
</span> &nbsp;&nbsp;&nbsp;&nbsp;Loading&nbsp;.&nbsp;.&nbsp;.
</button>
<br></br>
<div class="panel panel-danger hidden" id="panel_error">
<div class="panel-heading">ERROR</div>
<div class="panel-body">
<p id="error_message">error</p>
</div>
</div>
<div class="list-group hidden" id="submit_pastes">
<li class="list-group-item active">Submitted pastes</li>
</div>
</div>
<div class="modal-footer">
<button class="btn btn-success btn-tags hidden " data-dismiss="modal" id="success_submit_button">
<span class="label-icon">Success </span>
<i class="fa fa-check"></i>
</button>
<button class="btn btn-danger btn-tags hidden" data-dismiss="modal" id="error_submit_button">
<span class="label-icon">ERROR </span>
<i class="fa fa-times"></i>
</button>
<a class="btn btn-tags pull-left hidden" href="{{ url_for('Tags.get_tagged_paste') }}?ltags=submitted" target="_blank" id="submit_result">
<span class="label-icon">Submitted Pastes </span>
<i class="fa fa-paper-plane fa-2x"></i>
</a>
</div>
</div>
</div>
</div>
</body>
<script>
var ltags
var ltagsgalaxies
$(document).ready(function(){
$.getJSON('/Tags/get_all_tags_taxonomies',
function(data) {
ltags = $('#ltags').tagSuggest({
data: data,
maxDropHeight: 200,
name: 'ltags'
});
});
$.getJSON('/Tags/get_all_tags_galaxies',
function(data) {
ltagsgalaxies = $('#ltagsgalaxies').tagSuggest({
data: data,
maxDropHeight: 200,
name: 'ltagsgalaxies'
});
});
activePage = "page-PasteSubmit"
$("#"+activePage).addClass("active");
$('#modalsubmit').modal({backdrop: 'static'})
});
</script>
<script>
function submitPaste(){
document.getElementById("tags_taxonomies").value = ltags.getValue();
document.getElementById("tags_galaxies").value = ltagsgalaxies.getValue();
}
</script>
<script>
jQuery("#all-tags-taxonomies").click(function(e){
//change input tags list
$.getJSON('/Tags/get_all_tags_taxonomies',
function(data) {
ltags.setData(data)
});
});
</script>
<script>
jQuery("#all-tags-galaxies").click(function(e){
$.getJSON('/Tags/get_all_tags_galaxies',
function(data) {
ltagsgalaxies.setData(data)
});
});
{% for taxo in active_taxonomies %}
jQuery("#{{ taxo }}-id{{ loop.index0 }}").click(function(e){
$.getJSON('/Tags/get_tags_taxonomie?taxonomie={{ taxo }}',
function(data) {
ltags.setData(data)
});
});
{% endfor %}
</script>
<script>
{% for galaxy in active_galaxies %}
jQuery("#{{ galaxy }}-idgalax{{ loop.index0 }}").click(function(e){
$.getJSON('/Tags/get_tags_galaxy?galaxy={{ galaxy }}',
function(data) {
ltagsgalaxies.setData(data)
});
});
{% endfor %}
</script>
<script type="text/javascript">
var i = 0;
var err = 0;
$(".progress-bar").css("width", i + "%").text(i + " %");
function makeProgress(){
$.getJSON('/PasteSubmit/submit_status?UUID={{ UUID }}',
function(data) {
var end = data.end;
var prog = data.prog;
var in_progress = data.in_progress;
var processing = data.processing;
var isError = data.isError;
var error = data.error;
if(processing){
$("#loading_button").addClass('hidden');
$("#pr").removeClass('hidden');
$("#files_submitted_btn").removeClass('hidden');
}
if(i < 100){
$(".progress-bar").css("width", prog + "%").text(prog + " %");
}
if(!end){
document.getElementById('files_submitted').innerHTML = in_progress;
//error handler
if(isError){
document.getElementById('error_message').innerHTML = error;
$("#panel_error").removeClass('hidden');
$("#error_submit_button").removeClass('hidden');
err = err + 1;
}
// Wait for sometime before running this script again
if(err < 100){
setTimeout("makeProgress()", 100);
}
} else {
document.getElementById('files_submitted').innerHTML = in_progress;
$("#progress-bar").removeClass('active');
$("#submit_result").removeClass('hidden');
if(isError){
document.getElementById('error_message').innerHTML = error;
$("#panel_error").removeClass('hidden');
$("#error_submit_button").removeClass('hidden');
} else {
$("#success_submit_button").removeClass('hidden');
}
if(in_progress == '0 / '){
document.getElementById('files_submitted').innerHTML = '0 / 0'
$(".progress-bar").css("width", "100%").text("100 %");
$("#progress-bar").addClass('progress-bar-danger')
} else {
var link = data.link;
$("#submit_pastes").append(link);
$("#submit_pastes").removeClass('hidden');
}
}
}
);
}
makeProgress();
</script>
</html>

View file

@ -9,6 +9,7 @@ from flask import Flask, render_template, jsonify, request, Blueprint, redirect,
import json
from datetime import datetime
import ssdeep
import Paste
@ -22,6 +23,7 @@ app = Flask_config.app
cfg = Flask_config.cfg
r_serv_tags = Flask_config.r_serv_tags
r_serv_metadata = Flask_config.r_serv_metadata
r_serv_statistics = Flask_config.r_serv_statistics
max_preview_char = Flask_config.max_preview_char
max_preview_modal = Flask_config.max_preview_modal
bootstrap_label = Flask_config.bootstrap_label
@ -64,6 +66,7 @@ def get_tags_with_synonyms(tag):
else:
return {'name':tag,'id':tag}
# ============= ROUTES ==============
@Tags.route("/Tags/", methods=['GET'])
@ -296,6 +299,26 @@ def confirm_tag():
return 'incompatible tag'
@Tags.route("/Tags/tag_validation")
def tag_validation():
path = request.args.get('paste')
tag = request.args.get('tag')
status = request.args.get('status')
if (status == 'fp' or status == 'tp') and r_serv_tags.sismember('list_tags', tag):
if status == 'tp':
r_serv_statistics.sadd('tp:'+tag, path)
r_serv_statistics.srem('fp:'+tag, path)
else:
r_serv_statistics.sadd('fp:'+tag, path)
r_serv_statistics.srem('tp:'+tag, path)
return redirect(url_for('showsavedpastes.showsavedpaste', paste=path))
else:
return 'input error'
@Tags.route("/Tags/addTags")
def addTags():

View file

@ -65,6 +65,15 @@
</a>
</div>
<div>
<br></br>
<a class="btn btn-tags" href="{{ url_for('PasteSubmit.edit_tag_export') }}" target="_blank">
<i class="fa fa-cogs fa-2x"></i>
<br></br>
<span class="label-icon">MISP and Hive, auto push</span>
</a>
</div>
</div>
<!-- /#page-wrapper -->
@ -93,7 +102,6 @@
window.location.replace("{{ url_for('Tags.get_tagged_paste') }}?ltags=" + data);
}
function emptyTags() {
console.log('b');
ltags.clear();
}
</script>

View file

@ -144,6 +144,15 @@
</a>
</div>
<div>
<br></br>
<a class="btn btn-tags" href="{{ url_for('PasteSubmit.edit_tag_export') }}" target="_blank">
<i class="fa fa-cogs fa-2x"></i>
<br></br>
<span class="label-icon">MISP and Hive, auto push</span>
</a>
</div>
</div>
</body>

View file

@ -36,7 +36,7 @@
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
<nav class="navbar navbar-inverse navbar-static-top nav">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">

View file

@ -122,7 +122,6 @@ def search():
# Search full line
schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
print(selected_index)
ix = index.open_dir(selected_index)
with ix.searcher() as searcher:
query = QueryParser("content", ix.schema).parse(" ".join(q))

View file

@ -66,7 +66,7 @@
</br>
<div class="panel panel-default">
<div class="panel-heading">
<i class="glyphicon glyphicon-search"></i> <b id="numberOfRes">{{ r|length }}</b> Results for "<strong>{{ query }}</strong>
<i class="glyphicon glyphicon-search"></i> <b id="numberOfRes">{{ r|length }}</b> Results for "<strong>{{ query }}</strong>"
<div class="pull-right">
</div>

View file

@ -9,6 +9,7 @@ import json
import flask
from flask import Flask, render_template, jsonify, request, Blueprint, make_response
import difflib
import ssdeep
import Paste
@ -20,10 +21,13 @@ cfg = Flask_config.cfg
r_serv_pasteName = Flask_config.r_serv_pasteName
r_serv_metadata = Flask_config.r_serv_metadata
r_serv_tags = Flask_config.r_serv_tags
r_serv_statistics = Flask_config.r_serv_statistics
max_preview_char = Flask_config.max_preview_char
max_preview_modal = Flask_config.max_preview_modal
DiffMaxLineLength = Flask_config.DiffMaxLineLength
bootstrap_label = Flask_config.bootstrap_label
misp_event_url = Flask_config.misp_event_url
hive_case_url = Flask_config.hive_case_url
showsavedpastes = Blueprint('showsavedpastes', __name__, template_folder='templates')
@ -110,11 +114,50 @@ def showpaste(content_range):
for tag in l_tags:
if(tag[9:28] == 'automatic-detection'):
list_tags.append( (tag, True) )
automatic = True
else:
list_tags.append( (tag, False) )
automatic = False
return render_template("show_saved_paste.html", date=p_date, bootstrap_label=bootstrap_label, active_taxonomies=active_taxonomies, active_galaxies=active_galaxies, list_tags=list_tags, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list)
tag_hash = ssdeep.hash(tag)
if r_serv_statistics.sismember('tp:'+tag, requested_path):
tag_status_tp = True
else:
tag_status_tp = False
if r_serv_statistics.sismember('fp:'+tag, requested_path):
tag_status_fp = True
else:
tag_status_fp = False
list_tags.append( (tag, automatic, tag_status_tp, tag_status_fp) )
if Flask_config.pymisp is False:
misp = False
else:
misp = True
if Flask_config.HiveApi is False:
hive = False
else:
hive = True
misp_event = r_serv_metadata.get('misp_events:' + requested_path)
if misp_event is None:
misp_eventid = False
misp_url = ''
else:
misp_eventid = True
misp_url = misp_event_url + misp_event
hive_case = r_serv_metadata.get('hive_cases:' + requested_path)
if hive_case is None:
hive_caseid = False
hive_url = ''
else:
hive_caseid = True
hive_url = hive_case_url.replace('id_here', hive_case)
return render_template("show_saved_paste.html", date=p_date, bootstrap_label=bootstrap_label, active_taxonomies=active_taxonomies, active_galaxies=active_galaxies, list_tags=list_tags, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list,
misp=misp, hive=hive, misp_eventid=misp_eventid, misp_url=misp_url, hive_caseid=hive_caseid, hive_url=hive_url)
# ============ ROUTES ============

View file

@ -122,6 +122,28 @@
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }}" >{{ tag[0] }}</span>
</div>
<div class="modal-footer center">
{% if not tag[2] %}
<a href="{{ url_for('Tags.tag_validation') }}?paste={{ request.args.get('paste') }}&tag={{ tag[0] }}&status=tp" class="btn btn-success pull-left" data-toggle="tooltip" title="Good Detection">
<span class="glyphicon glyphicon-thumbs-up "></span>
</a>
{% endif %}
{% if tag[2] %}
<button class="btn btn-success pull-left disabled" data-toggle="tooltip" title="Good Detection">
<span class="glyphicon glyphicon-thumbs-up "></span>
</button>
{% endif %}
{% if not tag[3] %}
<a href="{{ url_for('Tags.tag_validation') }}?paste={{ request.args.get('paste') }}&tag={{ tag[0] }}&status=fp" class="btn btn-danger pull-left" data-toggle="tooltip" title="Bad Detection">
<span class="glyphicon glyphicon-thumbs-down "></span>
</a>
{% endif %}
{% if tag[3] %}
<button class="btn btn-danger pull-left disabled" data-toggle="tooltip" title="Bad Detection">
<span class="glyphicon glyphicon-thumbs-down "></span>
</button>
{% endif %}
{% if tag[1] %}
<a href="{{ url_for('Tags.confirm_tag') }}?paste={{ request.args.get('paste') }}&tag={{ tag[0] }}" class="btn btn-primary">
<span class="glyphicon glyphicon-ok "></span> Confirm this Tag
@ -170,8 +192,158 @@
</tbody>
</table>
<div>
{% if misp %}
<button class="btn btn-default btn-lg" data-toggle="modal" data-target="#myModal_MISP">
Create
<img id="misp-logo" src="{{url_for('static', filename='image/misp-logo.png') }}" height="32">
Event
</button>
<!-- Modal MISP-->
<div class="modal fade" id="myModal_MISP" role="dialog">
<div class="modal-dialog">
<div class="modal-content">
<div class="modal-header text-center">
<img id="misp-logo" src="{{url_for('static', filename='image/misp-logo.png') }}" >
</div>
<div class="modal-body">
<form method="post" action="/PasteSubmit/create_misp_event"target="_blank">
<div class="input select required">
<label for="EventDistribution">Distribution</label>
<select name="misp_data[Event][distribution]" id="EventDistribution" required="required">
<option value="0" selected="selected">Your organisation only</option>
<option value="1">This community only</option>
<option value="2">Connected communities</option>
<option value="3">All communities</option>
</select>
</div>
<div class="input clear required">
<label for="EventThreatLevelId">Threat Level</label>
<select name="misp_data[Event][threat_level_id]" id="EventThreatLevelId" required="required">
<option value="1">High</option>
<option value="2" selected="selected">Medium</option>
<option value="3">Low</option>
<option value="4">Undefined</option>
</select>
</div>
<div class="input select required">
<label for="EventAnalysis">Analysis</label>
<select name="misp_data[Event][analysis]" id="EventAnalysis" required="required">
<option value="0">Initial</option>
<option value="1">Ongoing</option>
<option value="2">Completed</option>
</select>
</div>
<div class="clear required">
<label for="EventInfo">Event Info</label>
<input name="misp_data[Event][info]" class="form-control span6" placeholder="Quick Event Description or Tracking Info" type="text" id="EventInfo" required="required"/>
</div>
<div class="clear">
<label for="EventInfo">Publish Event</label>
<input type="checkbox" value="True" id="misp_publish" name="misp_publish" >
</div>
<input type="hidden" id="paste" name="paste" value="{{ request.args.get('paste') }}">
{% if misp_eventid %}
<br>
<div class="list-group" id="misp_event">
<li class="list-group-item active">MISP Events already Created</li>
<a target="_blank" href="{{ misp_url }}" class="list-group-item">{{ misp_url }}</a></div>
{% endif %}
</div>
<div class="modal-footer center">
<button class="btn btn-primary">
<span class="glyphicon glyphicon-ok "></span> Create Event
</button>
</form>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
{% endif %}
{% if hive %}
<button class="btn btn-primary btn-lg" data-toggle="modal" data-target="#myModal_hive">
Create
<img id="thehive-icon" src="{{url_for('static', filename='image/thehive_icon.png') }}">
Case
</button>
<!-- Modal HIVE-->
<div class="modal fade" id="myModal_hive" role="dialog">
<div class="modal-dialog">
<div class="modal-content">
<div class="modal-header text-center">
<img id="misp-logo" src="{{url_for('static', filename='image/thehive-logo.png') }}" width="500" >
</div>
<div class="modal-body">
<form method="post" action="/PasteSubmit/create_hive_case" target="_blank">
<div class="input clear required">
<label for="EventThreatLevelId">Threat Level</label>
<select name="threat_level_hive" id="EventThreatLevelId" required="required">
<option value="1">High</option>
<option value="2" selected="selected">Medium</option>
<option value="3">Low</option>
</select>
</div>
<div class="input select required">
<label for="TLP">TLP</label>
<select name="hive_tlp" id="hive_tlp" required="required" class="selectpicker">
<option value="0">White</option>
<option value="1">Green</option>
<option value="2" selected="selected">Amber</option>
<option value="3">Red</option>
</select>
</div>
<div class="clear required">
<label for="hive_case_title">Title</label>
<input name="hive_case_title" class="form-control span6" placeholder="Title" type="text" id="hive_case_title" required="required"/>
</div>
<div class="clear required">
<label for="hive_description">Description</label>
<input name="hive_description" class="form-control span6" placeholder="Quick Case Description" type="text" id="hive_description" required="required"/>
</div>
<input type="hidden" id="paste" name="paste" value="{{ request.args.get('paste') }}">
{% if hive_caseid %}
<br>
<div class="list-group" id="misp_event">
<li class="list-group-item active">The Hive Case already Created</li>
<a target="_blank" href="{{ hive_url }}" class="list-group-item">{{ hive_url }}</a></div>
{% endif %}
</div>
<div class="modal-footer center">
<button href="" class="btn btn-primary">
<span class="glyphicon glyphicon-ok "></span> Create Case
</button>
</form>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
{% endif %}
</div>
</div>
<div class="panel-body" id="panel-body">
{% if duplicate_list|length == 0 %}
<h3> No Duplicate </h3>
{% else %}

Binary file not shown.

After

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 12 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.6 KiB

View file

@ -1,11 +1,13 @@
<div class="navbar-header">
<ul class="nav navbar-nav">
<div class="nav navbar-nav">
<!--dashboard-->
<!--PasteSubmit-->
<!--Tags-->
<!--terms-->
<!--browsepastes-->
<!--trendingcharts-->
<!--trendingmodules-->
<!--browsepastes-->
<!--sentiment-->
<!--terms-->
<!--insert here-->
</ul>
</div>
</div>

View file

@ -1,5 +1,5 @@
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
<div class="navbar navbar-inverse navbar-static-top nav">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
@ -15,5 +15,5 @@
<a href="{{ url_for('dashboard.index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
</div>
</div>

View file

@ -70,3 +70,9 @@ source ./../../AILENV/bin/activate
#Update MISP Taxonomies and Galaxies
python3 -m pip install git+https://github.com/MISP/PyTaxonomies
python3 -m pip install git+https://github.com/MISP/PyMISPGalaxies
#Update PyMISP
python3 -m pip install git+https://github.com/MISP/PyMISP
#Update the Hive
python3 -m pip install git+https://github.com/TheHive-Project/TheHive4py