From d269a5b31b9535f4f275e6b886299063beedf2b0 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 30 Apr 2020 15:46:38 +0200 Subject: [PATCH] chg: [import er url_extract] add item twitter parent --- .../ail_json_importer/Ail_feeder_twitter.py | 4 ++-- .../ail_json_importer/Ail_feeder_url_extract.py | 16 ++++++++++++---- bin/import/ail_json_importer/Default_json.py | 5 ++++- bin/import/importer.py | 2 +- bin/lib/item_basic.py | 8 ++++++++ bin/packages/Item.py | 7 ++++++- var/www/modules/showpaste/Flask_showpaste.py | 8 +++++++- .../showpaste/templates/show_saved_item_min.html | 6 ++++++ .../showpaste/templates/show_saved_paste.html | 6 ++++++ 9 files changed, 52 insertions(+), 10 deletions(-) diff --git a/bin/import/ail_json_importer/Ail_feeder_twitter.py b/bin/import/ail_json_importer/Ail_feeder_twitter.py index 831b56c3..61ca4ec4 100755 --- a/bin/import/ail_json_importer/Ail_feeder_twitter.py +++ b/bin/import/ail_json_importer/Ail_feeder_twitter.py @@ -34,8 +34,8 @@ class Ail_feeder_twitter(Default_json): item_id = str(self.json_item['meta']['twitter:tweet_id']) return os.path.join('twitter', item_date, item_id) + '.gz' - # # TODO: - def process_json_meta(self, process): + # # TODO: + def process_json_meta(self, process, item_id): ''' Process JSON meta filed. ''' diff --git a/bin/import/ail_json_importer/Ail_feeder_url_extract.py b/bin/import/ail_json_importer/Ail_feeder_url_extract.py index 9eb3324b..8e117d07 100755 --- a/bin/import/ail_json_importer/Ail_feeder_url_extract.py +++ b/bin/import/ail_json_importer/Ail_feeder_url_extract.py @@ -12,8 +12,8 @@ import json import sys import datetime -# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) -# import item_basic +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) +import item_basic sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import', 'ail_json_importer')) from Default_json import Default_json @@ -39,8 +39,16 @@ class Ail_feeder_twitter(Default_json): return os.path.join('urlextract', item_date, item_id) + '.gz' # # TODO: - def process_json_meta(self, process): + def process_json_meta(self, process, item_id): ''' Process JSON meta filed. ''' - return None + json_meta = self.get_json_meta() + + # # TODO: change me + parent_type = 'twitter' + item_parent = json_meta['parent:twitter:tweet_id'] + parent_date = datetime.date.today().strftime("%Y/%m/%d") + item_parent = os.path.join('twitter', parent_date, item_parent) + '.gz' + + item_basic.add_item_parent(item_parent, item_id) diff --git a/bin/import/ail_json_importer/Default_json.py b/bin/import/ail_json_importer/Default_json.py index a7ceb820..00e384ef 100755 --- a/bin/import/ail_json_importer/Default_json.py +++ b/bin/import/ail_json_importer/Default_json.py @@ -44,6 +44,9 @@ class Default_json(object): ''' return self.json_item + def get_json_meta(self): + return self.json_item['meta'] + def get_feeder_uuid(self): pass @@ -62,7 +65,7 @@ class Default_json(object): return os.path.join(self.get_feeder_name(), item_date, str(uuid.uuid4())) + '.gz' ## OVERWRITE ME ## - def process_json_meta(self, process): + def process_json_meta(self, process, item_id): ''' Process JSON meta filed. ''' diff --git a/bin/import/importer.py b/bin/import/importer.py index 760e99d5..2ea4c73f 100755 --- a/bin/import/importer.py +++ b/bin/import/importer.py @@ -81,7 +81,7 @@ def get_json_source(json_item): def process_json(importer_obj, process): item_id = importer_obj.get_item_id() if 'meta' in importer_obj.get_json_file(): - importer_obj.process_json_meta(process) + importer_obj.process_json_meta(process, item_id) # send data to queue send_item_to_ail_queue(item_id, importer_obj.get_item_gzip64encoded_content(), importer_obj.get_feeder_name(), process) diff --git a/bin/lib/item_basic.py b/bin/lib/item_basic.py index 756c55ef..35674301 100755 --- a/bin/lib/item_basic.py +++ b/bin/lib/item_basic.py @@ -23,3 +23,11 @@ def exist_item(item_id): def get_item_filepath(item_id): filename = os.path.join(PASTES_FOLDER, item_id) return os.path.realpath(filename) + +def add_item_parent(item_parent, item_id): + if not exist_item(item_parent): + return False + else: + r_serv_metadata.hset('paste_metadata:{}'.format(item_id), 'father', item_parent) + r_serv_metadata.sadd('paste_children:{}'.format(item_parent), item_id) + return True diff --git a/bin/packages/Item.py b/bin/packages/Item.py index a8cc5a51..ad9a41ad 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -284,9 +284,15 @@ def get_domain(item_id): item_id = item_id[-1] return item_id[:-36] +def get_item_parent(item_id): + return r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'father') + def get_item_children(item_id): return list(r_serv_metadata.smembers('paste_children:{}'.format(item_id))) +def add_item_parent(item_parent, item_id): + return item_basic.add_item_parent(item_parent, item_id) + def get_item_link(item_id): return r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'real_link') @@ -306,7 +312,6 @@ def get_item_har_name(item_id): def get_item_har(har_path): pass - def get_item_filename(item_id): # Creating the full filepath filename = os.path.join(PASTES_FOLDER, item_id) diff --git a/var/www/modules/showpaste/Flask_showpaste.py b/var/www/modules/showpaste/Flask_showpaste.py index 52525843..479d9cb2 100644 --- a/var/www/modules/showpaste/Flask_showpaste.py +++ b/var/www/modules/showpaste/Flask_showpaste.py @@ -22,6 +22,7 @@ import requests sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) import Tag +import Item sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import Domain @@ -229,6 +230,8 @@ def showpaste(content_range, requested_path): else: crawler_metadata['get_metadata'] = False + item_parent = Item.get_item_parent(requested_path) + if Flask_config.pymisp is False: misp = False else: @@ -256,7 +259,7 @@ def showpaste(content_range, requested_path): hive_url = hive_case_url.replace('id_here', hive_case) return render_template("show_saved_paste.html", date=p_date, bootstrap_label=bootstrap_label, active_taxonomies=active_taxonomies, active_galaxies=active_galaxies, list_tags=list_tags, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list, - crawler_metadata=crawler_metadata, tags_safe=tags_safe, + crawler_metadata=crawler_metadata, tags_safe=tags_safe, item_parent=item_parent, l_64=l_64, vt_enabled=vt_enabled, misp=misp, hive=hive, misp_eventid=misp_eventid, misp_url=misp_url, hive_caseid=hive_caseid, hive_url=hive_url) def get_item_basic_info(item): @@ -373,6 +376,8 @@ def show_item_min(requested_path , content_range=0): else: crawler_metadata['get_metadata'] = False + item_parent = Item.get_item_parent(requested_path) + misp_event = r_serv_metadata.get('misp_events:' + requested_path) if misp_event is None: misp_eventid = False @@ -391,6 +396,7 @@ def show_item_min(requested_path , content_range=0): return render_template("show_saved_item_min.html", bootstrap_label=bootstrap_label, content=item_content, item_basic_info=item_basic_info, item_info=item_info, + item_parent=item_parent, initsize=len(item_content), hashtype_list = p_hashtype_list, crawler_metadata=crawler_metadata, diff --git a/var/www/modules/showpaste/templates/show_saved_item_min.html b/var/www/modules/showpaste/templates/show_saved_item_min.html index 3d3da3fc..94327771 100644 --- a/var/www/modules/showpaste/templates/show_saved_item_min.html +++ b/var/www/modules/showpaste/templates/show_saved_item_min.html @@ -71,6 +71,12 @@ + {% if item_parent %} +
+ {{item_parent}} +
+ {% endif %} + {% if misp_eventid %}
  • MISP Events already Created
    • diff --git a/var/www/modules/showpaste/templates/show_saved_paste.html b/var/www/modules/showpaste/templates/show_saved_paste.html index 6af60e7b..e7993694 100644 --- a/var/www/modules/showpaste/templates/show_saved_paste.html +++ b/var/www/modules/showpaste/templates/show_saved_paste.html @@ -198,6 +198,12 @@ + {% if item_parent %} +
    + {{item_parent}} +
    + {% endif %} +
    {% with obj_type='item', obj_id=request.args.get('paste'), obj_lvl=0%} {% include 'import_export/block_add_user_object_to_export.html' %}