diff --git a/bin/DB_KVROCKS_MIGRATION.py b/bin/DB_KVROCKS_MIGRATION.py index 5e0f0169..351e582b 100755 --- a/bin/DB_KVROCKS_MIGRATION.py +++ b/bin/DB_KVROCKS_MIGRATION.py @@ -919,11 +919,11 @@ if __name__ == '__main__': # user_migration() #tags_migration() # items_migration() - crawler_migration() + # crawler_migration() # domain_migration() # TO TEST ########################### # decodeds_migration() # screenshots_migration() - # subtypes_obj_migration() + subtypes_obj_migration() # ail_2_ail_migration() # trackers_migration() # investigations_migration() diff --git a/bin/Mixer.py b/bin/Mixer.py index 9f170277..b15b4d62 100755 --- a/bin/Mixer.py +++ b/bin/Mixer.py @@ -85,9 +85,10 @@ if __name__ == '__main__': message = p.get_from_set() if message is not None: + print(message) splitted = message.split() if len(splitted) == 2: - complete_paste, gzip64encoded = splitted + complete_paste, gzip64encoded = splitted # NEW: source, item_id, gzip64 source if len==3 ??? try: #feeder_name = ( complete_paste.replace("archive/","") ).split("/")[0] diff --git a/bin/export/Export.py b/bin/export/Export.py index a3bd7305..7b39acd4 100755 --- a/bin/export/Export.py +++ b/bin/export/Export.py @@ -10,17 +10,10 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from lib.ConfigLoader import ConfigLoader -from lib.objects.Items import Item -from lib.ail_core import get_ail_uuid -from lib.Investigations import Investigation -from lib.objects import ail_objects ## LOAD CONFIG ## config_loader = ConfigLoader() -r_cache = config_loader.get_redis_conn("Redis_Cache") -r_db = config_loader.get_db_conn("Kvrocks_DB") -r_serv_db = config_loader.get_redis_conn("ARDB_DB") ###################################### r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") ###################################### config_loader = None ## -- ## @@ -39,154 +32,6 @@ from pymisp import MISPEvent, MISPObject, PyMISP # THE HIVE ################################## -HIVE_CLIENT = None -try: - from theHiveKEYS import the_hive_url, the_hive_key, the_hive_verifycert - - HIVE_URL = the_hive_url - HIVE_KEY = the_hive_key - HIVE_VERIFY_CERT = the_hive_verifycert -except: - HIVE_URL = None - HIVE_KEY = None - HIVE_VERIFY_CERT = None - - -def get_hive_client(): - global HIVE_CLIENT - try: - HIVE_CLIENT = TheHiveApi(HIVE_URL, HIVE_KEY, cert=HIVE_VERIFY_CERT) - except: - HIVE_CLIENT = None - return HIVE_CLIENT - - -def is_hive_connected(): - try: - # print(hive_client.health()) - HIVE_CLIENT.get_alert(0) - return True - except thehive4py.exceptions.AlertException: - return False - - -HIVE_CLIENT = get_hive_client() - -def sanitize_threat_level_hive(threat_level): - try: - int(threat_level) - if 1 <= threat_level <= 3: - return threat_level - else: - return 2 - except: - return 2 - -def sanitize_tlp_hive(tlp): - try: - int(tlp) - if 0 <= tlp <= 3: - return tlp - else: - return 2 - except: - return 2 - -def create_thehive_alert(item_id, tag_trigger): - item = Item(item_id) - meta = item.get_meta() - # TheHive expects a file - content = item.get_raw_content(decompress=True) - - # remove .gz from submitted path to TheHive because we've decompressed it - if item_id.endswith(".gz"): - item_id = item_id[:-3] - # add .txt it's easier to open when downloaded from TheHive - item_id = f'{item_id}.txt' - - artifacts = [ - AlertArtifact(dataType='other', message='uuid-ail', data=(get_ail_uuid())), - AlertArtifact(dataType='file', data=(content, item_id), tags=meta['tags']) - ] - - # Prepare the sample Alert - sourceRef = str(uuid.uuid4())[0:6] - alert = Alert(title='AIL Leak', - tlp=3, - tags=meta['tags'], - description='AIL Leak, triggered by {}'.format(tag_trigger), - type='ail', - source=meta['source'], # Use item ID ? - sourceRef=sourceRef, - artifacts=artifacts) - - # Create the Alert - alert_id = None - try: - response = HIVE_CLIENT.create_alert(alert) - if response.status_code == 201: - # print(json.dumps(response.json(), indent=4, sort_keys=True)) - print('Alert Created') - print(response.json()) - alert_id = response.json()['id'] - else: - print(f'ko: {response.status_code}/{response.text}') - return 0 - except: - print('hive connection error') - print(alert_id) - - -# TODO SAVE CASE URL ???????????????????????? -def create_thehive_case(item_id, title=None, tlp=2, threat_level=2, description=None): - item = Item(item_id) - ail_uuid = get_ail_uuid() - - if not title: - title = f'AIL Case {item.id}' - if not description: - description = f'AIL {ail_uuid} Case' - date = item.get_date() - date = f'{date[0:4]}-{date[4:6]}-{date[6:8]}' - tags = item.get_tags(r_list=True) - - case = Case(title=title, - tlp=tlp, - severity=threat_level, - flag=False, - tags=tags, - description=description) - - # Create Case - response = get_hive_client().create_case(case) - if response.status_code == 201: - case_id = response.json()['id'] - - observables = [ - CaseObservable(dataType="other", data=[ail_uuid], message="uuid-ail"), - CaseObservable(dataType="file", data=item.get_filename(), tags=tags), - CaseObservable(dataType="other", data=[item.get_source()], message="source"), - CaseObservable(dataType="other", data=[date], message="last-seen") - ] - - for observable in observables: - resp = HIVE_CLIENT.create_case_observable(case_id, observable) - if resp.status_code != 201: - print(f'error observable creation: {resp.status_code}/{resp.text}') - # print(case_id) - # return HIVE_URL /thehive/cases/~37040/details - return case_id - - # r_serv_metadata.set('hive_cases:'+path, id) - else: - print(f'ko: {response.status_code}/{response.text}') - return None - - -def get_case_url(case_id): - return f'{HIVE_URL}/cases/{case_id}/details' - - # TODO def get_item_hive_cases(item_id): hive_case = r_serv_metadata.get('hive_cases:{}'.format(item_id)) @@ -195,11 +40,7 @@ def get_item_hive_cases(item_id): return hive_case -################################## -# MISP -################################## -#####################################################################3 ########################################################### # # set default diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 45b7d059..b90c3cae 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -112,7 +112,7 @@ class Tracker: return r_tracker.smembers(f'tracker:sources:{self.uuid}') def get_tracker(self): - return r_serv_tracker.hget(f'tracker:{self.uuid}', 'tracked') + return r_tracker.hget(f'tracker:{self.uuid}', 'tracked') def get_type(self): return r_tracker.hget(f'tracker:{self.uuid}', 'type') diff --git a/bin/lib/Users.py b/bin/lib/Users.py index e29b1a45..2c231396 100755 --- a/bin/lib/Users.py +++ b/bin/lib/Users.py @@ -131,7 +131,7 @@ def create_user(user_id, password=None, chg_passwd=True, role=None): r_serv_db.hset('ail:users:all', user_id, password_hash) if chg_passwd: - r_serv_db.hset(f'ail:user:metadata:{user_id}', 'change_passwd', True) + r_serv_db.hset(f'ail:user:metadata:{user_id}', 'change_passwd', 'True') # create user token generate_new_token(user_id) diff --git a/var/www/blueprints/root.py b/var/www/blueprints/root.py index 36cf7bec..b56e9efd 100644 --- a/var/www/blueprints/root.py +++ b/var/www/blueprints/root.py @@ -85,7 +85,7 @@ def login(): # login failed else: # set brute force protection - #logger.warning("Login failed, ip={}, username={}".format(current_ip, username)) + # logger.warning("Login failed, ip={}, username={}".format(current_ip, username)) r_cache.incr('failed_login_ip:{}'.format(current_ip)) r_cache.expire('failed_login_ip:{}'.format(current_ip), 300) r_cache.incr('failed_login_user_id:{}'.format(username)) diff --git a/var/www/create_default_user.py b/var/www/create_default_user.py index 9a1af059..bc0c921d 100755 --- a/var/www/create_default_user.py +++ b/var/www/create_default_user.py @@ -18,7 +18,7 @@ if __name__ == "__main__": user_id = 'admin@admin.test' password = Users.gen_password() - create_user(user_id, password=password, role='admin') + Users.create_user(user_id, password=password, role='admin') token = Users.get_default_admin_token() default_passwd_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD')