From b4f49f9e814586f828c57fa23be5c3a0756a00d1 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 16 Dec 2019 15:55:50 +0100 Subject: [PATCH] fix: [UI showDomain] fix down domain history --- bin/lib/Domain.py | 21 ++-- var/www/blueprints/crawler_splash.py | 5 +- .../correlation/show_correlation.html | 2 +- .../crawler/crawler_splash/showDomain.html | 105 +++++++++--------- 4 files changed, 69 insertions(+), 64 deletions(-) diff --git a/bin/lib/Domain.py b/bin/lib/Domain.py index 4073f751..54fd678a 100755 --- a/bin/lib/Domain.py +++ b/bin/lib/Domain.py @@ -283,15 +283,16 @@ def get_domain_items_crawled(domain, domain_type, port, epoch=None, items_link=F item_crawled['epoch'] = item_root['epoch'] item_crawled['date'] = time.strftime('%Y/%m/%d - %H:%M.%S', time.gmtime(item_root['epoch'])) item_crawled['items'] = [] - for item in get_domain_items(domain, item_root['root_item']): - dict_item = {"id": item} - if items_link: - dict_item['link'] = Item.get_item_link(item) - if item_screenshot: - dict_item['screenshot'] = Item.get_item_screenshot(item) - if item_tag: - dict_item['tags'] = Tag.get_item_tags_minimal(item) - item_crawled['items'].append(dict_item) + if item_root['root_item'] != str(item_root['epoch']): + for item in get_domain_items(domain, item_root['root_item']): + dict_item = {"id": item} + if items_link: + dict_item['link'] = Item.get_item_link(item) + if item_screenshot: + dict_item['screenshot'] = Item.get_item_screenshot(item) + if item_tag: + dict_item['tags'] = Tag.get_item_tags_minimal(item) + item_crawled['items'].append(dict_item) return item_crawled def get_link_tree(): @@ -540,7 +541,7 @@ class Domain(object): def __init__(self, domain, port=None): self.domain = str(domain) self.type = get_domain_type(domain) - if self.is_domain_up(): + if self.domain_was_up(): self.current_port = sanathyse_port(port, self.domain, self.type) def get_domain_name(self): diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index 989b8d51..bb728a65 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -61,7 +61,7 @@ def showDomain(): dict_domain = domain.get_domain_metadata() dict_domain['domain'] = domain_name - if domain.is_domain_up(): + if domain.domain_was_up(): dict_domain = {**dict_domain, **domain.get_domain_correlation()} dict_domain['correlation_nb'] = Domain.get_domain_total_nb_correlation(dict_domain) dict_domain['origin_item'] = domain.get_domain_last_origin() @@ -69,7 +69,8 @@ def showDomain(): dict_domain['tags_safe'] = Tag.is_tags_safe(dict_domain['tags']) dict_domain['history'] = domain.get_domain_history_with_status() dict_domain['crawler_history'] = domain.get_domain_items_crawled(items_link=True, epoch=epoch, item_screenshot=True, item_tag=True) # # TODO: handle multiple port - dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items']) + if dict_domain['crawler_history']['items']: + dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items']) return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label, modal_add_tags=Tag.get_modal_add_tags(dict_domain['domain'], tag_type="domain")) diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 4f7086d6..dcff5c82 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -449,7 +449,7 @@ if (d.popover) { } else { desc = desc + "Red" } - desc = desc + ";\"> -
Crawled Items
+
Crawled Items {% if not dict_domain["crawler_history"]["items"] %}: DOWN{% endif %}
- - - - - - - - - {% for item in dict_domain["crawler_history"]["items"] %} - - - - - {% endfor %} - - -
Crawled Pastes
- -
{{ item["link"] }}
- -
- {% for tag in item["tags"] %} - - {{ tag["min_tag"] }} - - {% endfor %} -
- 		- {%if item["screenshot"]%} - - + {% if dict_domain["crawler_history"]["items"] %} + + + + + + + + + {% for item in dict_domain["crawler_history"]["items"] %} + + + + + {% endfor %} + + +
Crawled Pastes
+ +
{{ item["link"] }}
+ +
+ {% for tag in item["tags"] %} + + {{ tag["min_tag"] }} + + {% endfor %} +
+ 		+ {%if item["screenshot"]%} + + + + {%endif%} +
+ +
+ + - {%endif%} -
- -
- - - -
+ + + {%endif%} {%endif%} {% if dict_domain["history"] %} @@ -456,17 +458,18 @@ img.addEventListener("error", img_error); var draw_img = false; {%if "crawler_history" in dict_domain%} - {%if dict_domain['crawler_history']['random_item']['screenshot']%} - var screenshot = "{{dict_domain['crawler_history']['random_item']['screenshot']}}"; - var selected_icon = $("#"+screenshot.replace(/\//g, "")); - selected_icon.addClass("icon_selected"); - selected_icon.removeClass("icon_img"); + {%if "random_item" in dict_domain['crawler_history']%} + {%if dict_domain['crawler_history']['random_item']['screenshot']%} + var screenshot = "{{dict_domain['crawler_history']['random_item']['screenshot']}}"; + var selected_icon = $("#"+screenshot.replace(/\//g, "")); + selected_icon.addClass("icon_selected"); + selected_icon.removeClass("icon_img"); - - $("#screenshot_link").attr("href", screenshot_href + "{{dict_domain['crawler_history']['random_item']['id']}}"); - $("#screenshot_link").text("{{dict_domain['crawler_history']['random_item']['link']}}"); - {%else%} - var screenshot = ""; + $("#screenshot_link").attr("href", screenshot_href + "{{dict_domain['crawler_history']['random_item']['id']}}"); + $("#screenshot_link").text("{{dict_domain['crawler_history']['random_item']['link']}}"); + {%else%} + var screenshot = ""; + {%endif%} {%endif%} {%else%} var screenshot = "";