2020-08-18 19:10:38 +02:00
<!DOCTYPE html>
< html >
< head >
< title > AIL-Framework< / title >
< link rel = "icon" href = "{{ url_for('static', filename='image/ail-icon.png')}}" >
<!-- Core CSS -->
< link href = "{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel = "stylesheet" >
< link href = "{{ url_for('static', filename='css/font-awesome.min.css') }}" rel = "stylesheet" >
<!-- JS -->
< script src = "{{ url_for('static', filename='js/jquery.js')}}" > < / script >
< script src = "{{ url_for('static', filename='js/bootstrap4.min.js')}}" > < / script >
< / head >
< body >
{% include 'nav_bar.html' %}
< div class = "container-fluid" >
< div class = "row" >
{% include 'crawler/menu_sidebar.html' %}
< div class = "col-12 col-lg-10" id = "core_content" >
< div class = "row" >
< div class = "col-xl-6" >
< / div >
< div class = "col-xl-6" >
< / div >
< / div >
< div class = "card mb-3 mt-1" >
< div class = "card-header bg-dark text-white" >
< span class = "badge badge-pill badge-light flex-row-reverse float-right" >
2020-08-24 22:31:41 +02:00
{% if is_manager_connected['status'] %}
2020-08-18 19:10:38 +02:00
< div style = "color:Green;" >
< i class = "fas fa-check-circle fa-2x" > < / i >
Connected
< / div >
{% else %}
< div style = "color:Red;" >
< i class = "fas fa-times-circle fa-2x" > < / i >
Error
< / div >
{% endif %}
< / span >
2023-02-21 12:22:49 +01:00
< h4 > AIL Lacus Crawler< / h4 >
2020-08-18 19:10:38 +02:00
< / div >
< div class = "card-body" >
2020-08-24 22:31:41 +02:00
{%if not is_manager_connected['status']%}
{% include 'crawler/crawler_disabled.html' %}
{%endif%}
2020-08-18 19:10:38 +02:00
< div class = "row mb-3 justify-content-center" >
< div class = "col-xl-6" >
< div class = "card text-center border-secondary" >
< div class = "card-body px-1 py-0" >
< table class = "table table-sm" >
< tbody >
< tr >
2022-10-25 16:25:19 +02:00
< td > Lacus URL< / td >
< td > {{lacus_url}}< / td >
2020-08-18 19:10:38 +02:00
< / tr >
< tr >
2022-10-25 16:25:19 +02:00
{# < td > API Key< / td > #}
{# < td > #}
{# {{api_key}}#}
{# <!-- <a class="ml - 3" href="/settings/new_token"><i class="fa fa - random"></i></a> --> #}
{# < / td > #}
2020-08-18 19:10:38 +02:00
< td >
2022-10-25 16:25:19 +02:00
< a href = "{{ url_for('crawler_splash.crawler_lacus_settings_crawler_manager') }}" >
2020-08-24 22:31:41 +02:00
< button type = "button" class = "btn btn-info" >
Edit < i class = "fas fa-pencil-alt" > < / i >
< / button >
< / a >
< / td >
2020-08-18 19:10:38 +02:00
< / tr >
< / tbody >
< / table >
< / div >
< / div >
< / div >
2021-03-29 20:27:20 +02:00
< / div >
2024-08-12 15:18:54 +02:00
< a href = "{{ url_for('crawler_splash.crawlers_dashboard_captures_delete') }}" >
< button type = "button" class = "btn btn-danger" > Delete Lacus Queues< / button >
< / a >
2023-02-21 12:22:49 +01:00
{# < div class = "card border-secondary" > #}
{# < div class = "card-body text-dark" > #}
{# < h5 class = "card-title" > All Proxies:< / h5 > #}
{# < table class = "table table-striped" > #}
{# < thead class = "bg-info text-white" > #}
{# < tr > #}
{# < th > Proxy name< / th > #}
{# < th > URL< / th > #}
{# < th > Description< / th > #}
{# < th > < / th > #}
{# < / tr > #}
{# < / thead > #}
{# < tbody > #}
{# {% for proxy_name in all_proxies %}#}
{# < tr > #}
{# < td > #}
{# {{proxy_name}}#}
{# < / td > #}
{# < td > #}
{# {{all_proxies[proxy_name]['url']}}#}
{# < / td > #}
{# < td > #}
{# {{all_proxies[proxy_name]['description']}}#}
{# < / td > #}
{# < td > #}
{# < div class = "d-flex justify-content-end" > #}
{# <!-- <button class="btn btn - outline - dark px - 1 py - 0">#}
{# < i class = "fas fa-pencil-alt" > < / i > #}
{# < / button > -->#}
{# < / div > #}
{# < / td > #}
{# < / tr > #}
{# {% endfor %}#}
{# < / tbody > #}
{# < / table > #}
{# < / div > #}
{# < / div > #}
2021-03-29 20:27:20 +02:00
2020-08-24 22:31:41 +02:00
< / div >
2021-03-29 20:27:20 +02:00
< / div >
2020-08-24 22:31:41 +02:00
2021-03-29 20:27:20 +02:00
< div class = "card mb-3 mt-1" >
< div class = "card-header bg-dark text-white" >
< span class = "badge badge-pill badge-light flex-row-reverse float-right" >
{% if is_crawler_working %}
< div style = "color:Green;" >
< i class = "fas fa-check-circle fa-2x" > < / i >
{{crawler_error_mess}}
< / div >
{% else %}
< div style = "color:Red;" >
< i class = "fas fa-times-circle fa-2x" > < / i >
Error
< / div >
{% endif %}
< / span >
< h4 > Crawlers< / h4 >
< / div >
< div class = "card-body" >
< pre class = "bg-dark text-white" >
----------------------------
- TOR CRAWLER TEST OUTPUT: -
----------------------------
2020-08-24 22:31:41 +02:00
2021-03-29 20:27:20 +02:00
{{crawler_error_mess}}
< / pre >
2022-10-25 16:25:19 +02:00
< a href = "{{ url_for('crawler_splash.crawler_settings_crawler_test') }}" >
2021-03-29 20:27:20 +02:00
< button type = "button" class = "btn btn-primary" >
ReRun Test < i class = "fas fa-rocket" > < / i >
< / button >
< / a >
< div class = "card border-secondary my-4" >
2020-08-24 22:31:41 +02:00
< div class = "card-body text-dark" >
2023-02-21 12:22:49 +01:00
< h5 class = "card-title" >
Number of Concurrent Crawlers to Launch: < b class = "text-primary" > {{ nb_captures }}< / b >
< / h5 >
< a href = "{{ url_for('crawler_splash.crawler_settings_crawlers_to_launch') }}" >
2020-08-24 22:31:41 +02:00
< button type = "button" class = "btn btn-info" >
2023-02-21 12:22:49 +01:00
Edit < i class = "fas fa-pencil-alt" > < / i >
2021-03-29 20:27:20 +02:00
< / button >
< / a >
2020-08-18 19:10:38 +02:00
< / div >
< / div >
2021-03-30 09:14:18 +02:00
<!-- <div class="card border - secondary">
2020-08-18 19:10:38 +02:00
< div class = "card-body text-dark" >
2021-03-29 20:27:20 +02:00
< h5 class = "card-title" > Crawlers Settings:< / h5 >
< table class = "table table-striped table-hover" >
2020-08-18 19:10:38 +02:00
< thead class = "bg-info text-white" >
< th >
2021-03-29 20:27:20 +02:00
Key
2020-08-18 19:10:38 +02:00
< / th >
< th >
2021-03-29 20:27:20 +02:00
Description
2020-08-18 19:10:38 +02:00
< / th >
< th >
2021-03-29 20:27:20 +02:00
Value
2020-08-18 19:10:38 +02:00
< / th >
2020-08-20 13:20:31 +02:00
< th > < / th >
2020-08-18 19:10:38 +02:00
< / thead >
< tbody >
2021-03-29 20:27:20 +02:00
{% for config_field in crawler_full_config %}
2020-08-18 19:10:38 +02:00
< tr >
< td >
2021-03-29 20:27:20 +02:00
{{config_field}}
2020-08-18 19:10:38 +02:00
< / td >
< td >
2021-03-29 20:27:20 +02:00
{{crawler_full_config[config_field]['info']}}
2020-08-18 19:10:38 +02:00
< / td >
< td >
2021-03-29 20:27:20 +02:00
{{crawler_full_config[config_field]['value']}}
2020-08-18 19:10:38 +02:00
< / td >
2020-08-20 13:20:31 +02:00
< td >
< div class = "d-flex justify-content-end" >
2021-03-30 09:14:18 +02:00
< button class = "btn btn-outline-dark px-1 py-0" >
2020-08-20 13:20:31 +02:00
< i class = "fas fa-pencil-alt" > < / i >
2021-03-30 09:14:18 +02:00
< / button >
2020-08-20 13:20:31 +02:00
< / div >
< / td >
2020-08-18 19:10:38 +02:00
< / tr >
{% endfor %}
< / tbody >
< / table >
2021-03-29 20:27:20 +02:00
2020-08-18 19:10:38 +02:00
< / div >
2021-03-30 09:14:18 +02:00
< / div > -->
2020-08-20 13:20:31 +02:00
2025-02-06 11:39:29 +01:00
< div class = "card border-secondary my-4" >
< div class = "card-body text-dark" >
< h5 class = "card-title" >
Filter Unsafe Onion: < b class = "text-primary" > < span class = "text-{% if is_onion_filter_enabled %}success{% else %}danger{% endif %}" > {{ is_onion_filter_enabled }}< / span > < / b >
< / h5 >
< p >
This option enables filtering of onion domains that are considered unsafe due to containing violent content, child sexual abuse material (CSAM), or other harmful materials. When enabled, the system will attempt to identify and exclude such domains from crawling.< br > < br >
< span class = "text-danger" > < i class = "fa-solid fa-triangle-exclamation fa-2x" > < / i > < / span > < b > Disabling this option may result in crawling and downloading content that includes CSAM, extreme violence, or other harmful materials.< / b > < br > Users are strongly advised to keep this feature enabled to avoid unintentional exposure to such content.< br > < br >
🔍 How It Works: The filtering mechanism leverages known blocklists, heuristics, and automated detection techniques to reduce the risk of crawling unsafe content. While no filtering system is perfect, we continuously strive to improve detection and minimize exposure to harmful materials.< br > < br >
By using this feature, you benefit from an added layer of protection, but please note that some unsafe onion domains may still bypass detection due to evolving content and obfuscation techniques.< br >
We encourage users to remain cautious and use this feature as an additional safeguard.
< / p >
{% if is_onion_filter_enabled %}
< a href = "{{ url_for('crawler_splash.crawler_filter_unsafe_onion') }}?state=disable" >
2025-02-06 15:10:06 +01:00
< button class = "btn btn-danger my-2" >
2025-02-06 11:39:29 +01:00
< i class = "fa-solid fa-xmark" > < / i > Disable Onion Filter
< / button >
< / a >
{% else %}
< a href = "{{ url_for('crawler_splash.crawler_filter_unsafe_onion') }}?state=enable" >
< button class = "btn btn-success my-2" >
< i class = "fa-solid fa-check" > < / i > Enable Onion Filter
< / button >
< / a >
{% endif %}
2025-02-06 15:10:06 +01:00
< hr class = "border-1 my-4" >
< h5 class = "card-title" >
Crawl Unknown Onion: < b class = "text-primary" > < span class = "text-{% if is_onion_filter_unknown %}success{% else %}secondary{% endif %}" > {% if is_onion_filter_unknown %}Enabled{% else %}Disabled{% endif %}< / span > < / b >
< / h5 >
< p > This option controls whether the crawler should proceed with crawling onion domains that have < strong > not yet been classified< / strong > as safe or unsafe.< / p >
< ul >
< li > < strong > If disabled:< / strong > The crawler will process domains that have never been checked, potentially discovering new useful content but also increasing the risk of encountering unsafe materials.< / li >
< li > < strong > If enabled:< / strong > The crawler will only process domains that have been explicitly identified as safe, reducing risk but potentially missing new, unclassified domains.< / li >
< / ul >
< p > This option is useful for users who want to explore uncharted domains while still benefiting from the < code > filter_unsafe_onion< / code > protection. However, enabling this option increases the likelihood of encountering harmful content, so caution is advised.< / p >
{% if is_onion_filter_unknown %}
< a href = "{{ url_for('crawler_splash.crawler_filter_unknown_onion') }}?state=disable" >
< button class = "btn btn-secondary my-2" >
< i class = "fa-solid fa-xmark" > < / i > Disable Unknown Onion Filter
< / button >
< / a >
{% else %}
< a href = "{{ url_for('crawler_splash.crawler_filter_unknown_onion') }}?state=enable" >
< button class = "btn btn-info my-2" >
< i class = "fa-solid fa-check" > < / i > Enable Unknown Onion Filter
< / button >
< / a >
{% endif %}
2025-02-06 11:39:29 +01:00
< / div >
< / div >
2023-03-31 09:25:06 +02:00
< a href = "{{ url_for('crawler_splash.crawler_blacklist') }}" >
< button type = "button" class = "btn btn-outline-danger" > Blacklisted domains< / button >
< / a >
2021-03-29 20:27:20 +02:00
< / div >
2020-08-18 19:10:38 +02:00
< / div >
< / div >
< / div >
< / div >
< / body >
< script >
var to_refresh = false
$(document).ready(function(){
$("#page-Crawler").addClass("active");
$("#nav_settings").addClass("active");
});
< / script >
