From 1f5d9960783755912d941019db752d8cceab1a24 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 26 Mar 2020 10:00:40 +0100 Subject: [PATCH 1/5] fix: [update thirdparty] update taxonomies --- var/www/update_thirdparty.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/update_thirdparty.sh b/var/www/update_thirdparty.sh index 667266e2..53b4e6bf 100755 --- a/var/www/update_thirdparty.sh +++ b/var/www/update_thirdparty.sh @@ -112,7 +112,7 @@ then fi #Update MISP Taxonomies and Galaxies -python3 -m pip install git+https://github.com/MISP/PyTaxonomies +python3 -m pip install git+https://github.com/MISP/PyTaxonomies --upgrade python3 -m pip install git+https://github.com/MISP/PyMISPGalaxies --upgrade #Update PyMISP From 1f08467777e650539fda7c2c07643096c18887b6 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 26 Mar 2020 11:34:29 +0100 Subject: [PATCH 2/5] chg: [domain explorer UI] add shortcut button: misp export + correlation --- var/www/templates/domains/card_img_domain.html | 5 ++++- .../block_add_user_object_to_export_small.html | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 var/www/templates/import_export/block_add_user_object_to_export_small.html diff --git a/var/www/templates/domains/card_img_domain.html b/var/www/templates/domains/card_img_domain.html index e2a02341..eb7e8371 100644 --- a/var/www/templates/domains/card_img_domain.html +++ b/var/www/templates/domains/card_img_domain.html @@ -8,7 +8,7 @@
-
+
{{dict_domain["id"]}} @@ -39,6 +39,9 @@ {% endfor %}
+ {% with obj_type='domain', obj_id=dict_domain["id"], obj_lvl=0%} + {% include 'import_export/block_add_user_object_to_export_small.html' %} + {% endwith %} {% if loop.index0 % 4 == 3 %} diff --git a/var/www/templates/import_export/block_add_user_object_to_export_small.html b/var/www/templates/import_export/block_add_user_object_to_export_small.html new file mode 100644 index 00000000..f18561e7 --- /dev/null +++ b/var/www/templates/import_export/block_add_user_object_to_export_small.html @@ -0,0 +1,12 @@ +
+
+ + + +
+
+ + + +
+
From 720ffa1c9c4df4c94a3d3455035c66ee3ef77fa7 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 26 Mar 2020 17:03:57 +0100 Subject: [PATCH 3/5] chg: [MISP export] export domain as domain-crawled object --- bin/export/MispExport.py | 14 +++++++------- bin/export/MispImport.py | 2 +- bin/lib/Domain.py | 33 +++++++++++++++++++++++++++++---- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/bin/export/MispExport.py b/bin/export/MispExport.py index b35099f6..fe1fc304 100755 --- a/bin/export/MispExport.py +++ b/bin/export/MispExport.py @@ -75,23 +75,23 @@ def export_ail_item(item_id): tag_misp_object_attributes(l_obj_attr, dict_metadata['tags']) return obj -# # TODO: create domain-port-history object def export_domain(domain): domain_obj = Domain.Domain(domain) dict_metadata = domain_obj.get_domain_metadata(tags=True) - dict_metadata['ports'] = ['80', '223', '443'] # create domain-ip obj - obj = MISPObject('domain-ip', standalone=True) + obj = MISPObject('domain-crawled', standalone=True) obj.first_seen = dict_metadata['first_seen'] obj.last_seen = dict_metadata['last_check'] l_obj_attr = [] - l_obj_attr.append( obj.add_attribute('first-seen', value=dict_metadata['first_seen']) ) - l_obj_attr.append( obj.add_attribute('last-seen', value=dict_metadata['last_check']) ) l_obj_attr.append( obj.add_attribute('domain', value=domain) ) - for port in dict_metadata['ports']: - l_obj_attr.append( obj.add_attribute('port', value=port) ) + dict_all_url = Domain.get_domain_all_url(domain, domain_obj.get_domain_type()) + for crawled_url in dict_all_url: + attribute = obj.add_attribute('url', value=crawled_url) + attribute.first_seen = str(dict_all_url[crawled_url]['first_seen']) + attribute.last_seen = str(dict_all_url[crawled_url]['last_seen']) + l_obj_attr.append( attribute ) # add tags if dict_metadata['tags']: diff --git a/bin/export/MispImport.py b/bin/export/MispImport.py index d371de85..d62a486f 100755 --- a/bin/export/MispImport.py +++ b/bin/export/MispImport.py @@ -190,7 +190,7 @@ def unpack_file(map_uuid_global_id, misp_obj): def get_misp_import_fct(map_uuid_global_id, misp_obj): if misp_obj.name == 'ail-leak': unpack_item_obj(map_uuid_global_id, misp_obj) - elif misp_obj.name == 'domain-ip': + elif misp_obj.name == 'domain-crawled': pass elif misp_obj.name == 'pgp-meta': unpack_obj_pgp(map_uuid_global_id, misp_obj) diff --git a/bin/lib/Domain.py b/bin/lib/Domain.py index 8f2500e6..b8ba49e3 100755 --- a/bin/lib/Domain.py +++ b/bin/lib/Domain.py @@ -327,6 +327,28 @@ def get_all_domain_up_by_type(domain_type): else: return ({"status": "error", "reason": "Invalid domain type"}, 400) +def get_domain_all_url(domain, domain_type, domain_ports=None): + if not domain_ports: + domain_ports = get_domain_all_ports(domain, domain_type) + all_url = {} + for port in domain_ports: + for dict_history in get_domain_history_with_status(domain, domain_type, port, add_root_item=True): + if dict_history['status']: # domain UP + crawled_items = get_domain_items(domain, dict_history['root_item']) + for item_id in crawled_items: + item_url = Item.get_item_link(item_id) + item_date = int(Item.get_item_date(item_id)) + if item_url: + if item_url not in all_url: + all_url[item_url] = {'first_seen': item_date,'last_seen': item_date} + else: # update first_seen / last_seen + if item_date < all_url[item_url]['first_seen']: + all_url[item_url]['first_seen'] = item_date + if item_date > all_url[item_url]['last_seen']: + all_url[item_url]['last_seen'] = item_date + return all_url + + def get_domain_items(domain, root_item_id): dom_item = get_domain_item_children(domain, root_item_id) dom_item.append(root_item_id) @@ -605,7 +627,7 @@ def get_domain_history(domain, domain_type, port): # TODO: add date_range: from ''' return r_serv_onion.zrange('crawler_history_{}:{}:{}'.format(domain_type, domain, port), 0, -1, withscores=True) -def get_domain_history_with_status(domain, domain_type, port): # TODO: add date_range: from to + nb_elem +def get_domain_history_with_status(domain, domain_type, port, add_root_item=False): # TODO: add date_range: from to + nb_elem ''' Retun . @@ -619,14 +641,17 @@ def get_domain_history_with_status(domain, domain_type, port): # TODO: add date_ history = get_domain_history(domain, domain_type, port) for root_item, epoch_val in history: epoch_val = int(epoch_val) # force int + dict_history = {"epoch": epoch_val, "date": time.strftime('%Y/%m/%d - %H:%M.%S', time.gmtime(epoch_val))} # domain down, root_item==epoch_val try: int(root_item) - status = False + dict_history['status'] = False # domain up, root_item=str except ValueError: - status = True - l_history.append({"epoch": epoch_val, "date": time.strftime('%Y/%m/%d - %H:%M.%S', time.gmtime(epoch_val)), "status": status}) + dict_history['status'] = True + if add_root_item: + dict_history['root_item'] = root_item + l_history.append(dict_history) return l_history def verify_if_domain_exist(domain): From bdbbcc64c04ba79c4f40f0442233ecdaeeaab982 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 26 Mar 2020 18:06:45 +0100 Subject: [PATCH 4/5] fix: [MISP export UI] fix input: name + value overwrite --- var/www/blueprints/import_export.py | 8 +------- var/www/templates/import_export/export_object.html | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/var/www/blueprints/import_export.py b/var/www/blueprints/import_export.py index 3d13fcbb..ddb3b534 100644 --- a/var/www/blueprints/import_export.py +++ b/var/www/blueprints/import_export.py @@ -137,14 +137,10 @@ def export_object_file(): dict_misp_event_export = None if l_obj_invalid: - for obj_dict in l_obj_to_export: - obj_dict['uuid'] = str(uuid.uuid4()) - obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(obj_dict['type'], obj_dict.get('subtype', None)) - # get user saved obj to export # # TODO: # performance l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id) - for obj_dict in l_obj_invalid: + for obj_dict in l_obj_invalid: # set uuid input obj_dict['uuid'] = str(uuid.uuid4()) obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(obj_dict['type'], obj_dict.get('subtype', None)) @@ -152,7 +148,6 @@ def export_object_file(): l_obj_invalid=l_obj_invalid, dict_misp_event_export=dict_misp_event_export) else: if export_to_misp and MispExport.ping_misp(): - l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id) event = MispExport.create_list_of_objs_to_export(l_obj_to_export, r_type='event') event_metadata = MispExport.create_misp_event(event, distribution=dict_misp_event_export.get('export_to_misp', None), @@ -167,7 +162,6 @@ def export_object_file(): l_obj_invalid=[], dict_misp_event_export=[]) else: # get user saved obj to export # # TODO: # performance - l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id) json_export = MispExport.create_list_of_objs_to_export(l_obj_to_export) export_filename = MispExport.get_export_filename(json_export) json_export = MispExport.create_in_memory_file(json_export.to_json()) diff --git a/var/www/templates/import_export/export_object.html b/var/www/templates/import_export/export_object.html index 9481671a..ed65850c 100644 --- a/var/www/templates/import_export/export_object.html +++ b/var/www/templates/import_export/export_object.html @@ -73,7 +73,7 @@ {% for obj_dict in l_obj_to_export %} - {% with obj_type=obj_dict['type'], obj_id=obj_dict['id'], obj_lvl=obj_dict['lvl'], input_uuid=obj_dict['uuid'], obj_error=False%} + {% with obj_type=obj_dict['type'], obj_id=obj_dict['id'], obj_lvl=obj_dict['lvl'], input_uuid=obj_dict, obj_error=False%} {% include 'import_export/block_to_export_input.html' %} {% endwith %} {% endfor %} From e3e543ba8b965c7c98bc4945d5db4a0919eedf60 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 30 Mar 2020 09:42:14 +0200 Subject: [PATCH 5/5] chg: [Crawler] default docker memory usage --- bin/torcrawler/launch_splash_crawler.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/torcrawler/launch_splash_crawler.sh b/bin/torcrawler/launch_splash_crawler.sh index 5c7f21ee..dcbe7bee 100755 --- a/bin/torcrawler/launch_splash_crawler.sh +++ b/bin/torcrawler/launch_splash_crawler.sh @@ -62,7 +62,7 @@ sleep 0.1 for ((i=0;i<=$((${n} - 1));i++)); do port_number=$((${p} + $i)) - screen -S "Docker_Splash" -X screen -t "docker_splash:$port_number" bash -c 'sudo docker run -d -p '$port_number':8050 --restart=always --cpus=1 --memory=4.5G -v '$f':/etc/splash/proxy-profiles/ --net="bridge" scrapinghub/splash --maxrss '$u'; read x' + screen -S "Docker_Splash" -X screen -t "docker_splash:$port_number" bash -c 'sudo docker run -d -p '$port_number':8050 --restart=always --cpus=1 --memory=2G -v '$f':/etc/splash/proxy-profiles/ --net="bridge" scrapinghub/splash --maxrss '$u'; read x' sleep 0.1 printf "$GREEN Splash server launched on port $port_number$DEFAULT\n" done