2019-02-26 13:50:48 +00:00
<!DOCTYPE html>
< html >
< head >
< title > AIL-Framework< / title >
< link rel = "icon" href = "{{ url_for('static', filename='image/ail-icon.png')}}" >
<!-- Core CSS -->
< link href = "{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel = "stylesheet" >
< link href = "{{ url_for('static', filename='css/font-awesome.min.css') }}" rel = "stylesheet" >
< link href = "{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel = "stylesheet" >
2023-06-20 06:11:44 +00:00
< link href = "{{ url_for('static', filename='css/tags.css') }}" rel = "stylesheet" >
2019-02-26 13:50:48 +00:00
<!-- JS -->
< script src = "{{ url_for('static', filename='js/jquery.js')}}" > < / script >
< script src = "{{ url_for('static', filename='js/popper.min.js')}}" > < / script >
< script src = "{{ url_for('static', filename='js/bootstrap4.min.js')}}" > < / script >
2023-06-20 06:11:44 +00:00
< script src = "{{ url_for('static', filename='js/moment.min.js') }}" > < / script >
< script src = "{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}" > < / script >
< script src = "{{ url_for('static', filename='js/d3.min.js') }}" > < / script >
< script src = "{{ url_for('static', filename='js/tags.js') }}" > < / script >
2019-02-26 13:50:48 +00:00
< / head >
< body >
{% include 'nav_bar.html' %}
< div class = "container-fluid" >
< div class = "row" >
2019-02-28 13:56:14 +00:00
{% include 'crawler/menu_sidebar.html' %}
2019-02-26 13:50:48 +00:00
2019-02-28 13:56:14 +00:00
< div class = "col-12 col-lg-10" id = "core_content" >
2019-02-26 13:50:48 +00:00
2019-06-04 12:22:46 +00:00
{% include 'crawler/crawler_disabled.html' %}
2019-02-26 13:50:48 +00:00
< div class = "card text-white bg-dark mb-3 mt-1" >
< div class = "card-header" >
< h5 class = "card-title" > Crawl a Domain< / h5 >
< / div >
< div class = "card-body" >
2022-10-25 14:25:19 +00:00
< p class = "card-text" > Enter an url or a domain and choose what kind of option you want.< / p >
2020-03-30 16:43:50 +00:00
< form action = "{{ url_for('crawler_splash.send_to_spider') }}" method = 'post' >
2019-02-26 13:50:48 +00:00
< div class = "row" >
< div class = "col-12 col-lg-6" >
2024-10-09 13:05:27 +00:00
{% if urls %}
{% for url in urls %}
< div class = "input-group mb-1" >
< input type = "text" class = "form-control col-10" name = "urls" value = "{{ url }}" >
< span class = "btn btn-danger col-1" id = "" onclick = "$(this).parent().remove();" > < i class = "fas fa-trash-alt" > < / i > < / span >
< / div >
{% endfor %}
{% else %}
< div class = "input-group" id = "single_urls" >
< input type = "text" class = "form-control" id = "url_to_crawl" name = "url_to_crawl" placeholder = "Address or Domain" >
< div class = "input-group-append" >
< button class = "btn btn-secondary" type = "button" onclick = "btn_multiple_urls()" > < i class = "fa fa-plus" > < / i > Multiple Urls< / button >
< / div >
< / div >
< div class = "input-group" id = "multiple_urls" >
< textarea type = "text" class = "form-control" id = "urls_to_crawl" name = "urls_to_crawl" rows = "3" placeholder = "List Of Urls or Free Text" > < / textarea >
< div class = "input-group-append" >
< button class = "btn btn-secondary" type = "button" onclick = "btn_single_url()" > < i class = "fa fa-minus" > < / i > One Url< / button >
< / div >
< / div >
{% endif %}
2020-08-24 20:31:41 +00:00
< div class = "d-flex mt-2" >
< i class = "fas fa-spider mt-1" > < / i > Crawler Type
< div class = "custom-control custom-switch" >
< input class = "custom-control-input" type = "checkbox" name = "queue_type_selector" value = "True" id = "queue_type_selector" >
< label class = "custom-control-label" for = "queue_type_selector" >
2022-10-25 14:25:19 +00:00
< i class = "fas fa-splotch" > < / i > Proxy
2020-08-24 20:31:41 +00:00
< / label >
< / div >
< / div >
< div id = "div_crawler_queue_type" >
< select class = "custom-select form-control" name = "crawler_queue_type" id = "crawler_queue_type" >
2022-10-25 14:25:19 +00:00
{%for crawler_type in crawlers_types%}
2020-08-24 20:31:41 +00:00
< option value = "{{crawler_type}}" { % if crawler_type = ='tor'%}selected{%endif%} > {{crawler_type}}< / option >
{%endfor%}
< / select >
< / div >
2023-02-21 11:22:49 +00:00
< div class = "input-group" id = "div_proxy_name" >
< input type = "text" class = "form-control" id = "proxy_name" name = "proxy_name" placeholder = "Expected Format: [scheme]://[username]:[password]@[hostname]:[port]" >
2020-08-24 20:31:41 +00:00
< / div >
< div class = "d-flex mt-3" >
2019-02-26 13:50:48 +00:00
< i class = "fas fa-user-ninja mt-1" > < / i > Manual
< div class = "custom-control custom-switch" >
2023-03-14 16:36:42 +00:00
< input class = "custom-control-input" type = "checkbox" name = "crawler_scheduler" value = "True" id = "crawler_scheduler" >
< label class = "custom-control-label" for = "crawler_scheduler" >
< i class = "fas fa-clock" > < / i > Scheduler
2019-02-26 13:50:48 +00:00
< / label >
< / div >
< / div >
2023-03-14 16:36:42 +00:00
< div id = "frequency_inputs" >
< div class = "mb-4" >
< select class = "custom-select" id = "frequency" name = "frequency" onchange = "frequency_selector_update(this);" >
< option value = "hourly" > Hourly< / option >
< option value = "daily" > Daily< / option >
< option value = "weekly" > Weekly< / option >
< option value = "monthly" > Monthly< / option >
< option value = "custom" > Custom< / option >
< / select >
< / div >
< div id = "custom_frequency" >
< h5 > < i class = "fas fa-clock" > < / i > Adjust crawling interval as needed< / h5 >
< div class = "input-group" >
< div class = "input-group-prepend" >
< span class = "input-group-text bg-light" style = "width: 90px" > < b > Months< / b > < / span >
< / div >
< input class = "form-control" type = "number" id = "frequency_months" value = "0" min = "0" name = "frequency_months" required >
< / div >
< div class = "input-group" >
< div class = "input-group-prepend" >
< span class = "input-group-text bg-light" style = "width: 90px" > < b > Weeks< / b > < / span >
< / div >
< input class = "form-control" type = "number" id = "frequency_weeks" value = "0" min = "0" name = "frequency_weeks" required >
< / div >
< div class = "input-group" >
< div class = "input-group-prepend" >
< span class = "input-group-text bg-light" style = "width: 90px" > < b > Days< / b > < / span >
< / div >
< input class = "form-control" type = "number" id = "frequency_days" value = "0" min = "0" name = "frequency_days" required >
< / div >
< div class = "input-group" >
< div class = "input-group-prepend" >
< span class = "input-group-text bg-light" style = "width: 90px" > < b > Hours< / b > < / span >
< / div >
< input class = "form-control" type = "number" id = "frequency_hours" value = "0" min = "0" name = "frequency_hours" required >
< / div >
< div class = "input-group" >
< div class = "input-group-prepend" >
< span class = "input-group-text bg-light" style = "width: 90px" > < b > Minutes< / b > < / span >
< / div >
< input class = "form-control" type = "number" id = "frequency_minutes" value = "0" min = "0" name = "frequency_minutes" required >
< / div >
< / div >
< / div >
2023-06-20 06:11:44 +00:00
< div class = "card bg-light my-4" >
< div class = "card-header bg-dark text-white" >
< b > Tags< / b >
< / div >
< div class = "card-body" >
< div class = "input-group mb-2 mr-sm-2" >
< div class = "input-group-prepend" >
< div class = "input-group-text bg-danger text-white" > < i class = "fas fa-tag" > < / i > < / div >
< / div >
< input id = "tags" name = "tags" class = "form-control" placeholder = "Custom Tags (optional, space separated)" type = "text" { % if dict_tracker % } { % if dict_tracker [ ' tags ' ] % } value = "{%for tag in dict_tracker['tags']%}{{tag}} {%endfor%}" { % endif % } { % endif % } >
< / div >
{% include 'tags/block_tags_selector.html' %}
< / div >
< / div >
2019-02-26 13:50:48 +00:00
< / div >
< div class = "col-12 col-lg-6 mt-2 mt-lg-0" >
< div class = "row" >
< div class = "col-12 col-xl-6" >
< div class = "custom-control custom-switch" >
< input class = "custom-control-input" type = "checkbox" name = "html_content" value = "True" id = "html_content_id" checked disabled >
< label class = "custom-control-label" for = "html_content_id" >
< i class = "fab fa-html5" > < / i > HTML
< / label >
< / div >
< div class = "custom-control custom-switch mt-1" >
< input class = "custom-control-input" type = "checkbox" name = "screenshot" value = "True" id = "screenshot_id" >
< label class = "custom-control-label" for = "screenshot_id" >
< i class = "fas fa-image" > < / i > Screenshot
< / label >
< / div >
< div class = "custom-control custom-switch mt-1" >
< input class = "custom-control-input" type = "checkbox" name = "har" value = "True" id = "har_id" >
< label class = "custom-control-label" for = "har_id" >
< i class = "fas fa-file" > < / i > HAR
< / label >
< / div >
< / div >
< div class = "col-12 col-xl-6" >
< div class = "input-group form-group mb-0" >
< div class = "input-group-prepend" >
< span class = "input-group-text bg-light" > < i class = "fas fa-water" > < / i > < / span >
< / div >
2019-02-26 14:39:52 +00:00
< input class = "form-control" type = "number" id = "depth_limit" name = "depth_limit" min = "0" value = "0" required >
2019-02-26 13:50:48 +00:00
< div class = "input-group-append" >
< span class = "input-group-text" > Depth Limit< / span >
< / div >
< / div >
2022-10-25 14:25:19 +00:00
{# TEMPORARY DISABLED #}
{# < div class = "input-group mt-2" > #}
{# < div class = "input-group-prepend" > #}
{# < span class = "input-group-text bg-light" > < i class = "fas fa-copy" > < / i > < / span > #}
{# < / div > #}
{# < input class = "form-control" type = "number" id = "max_pages" name = "max_pages" min = "1" value = "1" required > #}
{# < div class = "input-group-append" > #}
{# < span class = "input-group-text" > Max Pages< / span > #}
{# < / div > #}
{# < / div > #}
2019-02-26 13:50:48 +00:00
< / div >
< / div >
2020-03-30 16:43:50 +00:00
< div class = "mt-1" >
< i class = "mt-2 text-white fas fa-cookie-bite" > < / i > Cookiejar:
< select class = "custom-select form-control mt-1" name = "cookiejar" id = "cookiejar" >
< option value = "None" selected > Don't use any cookiejar< / option >
{%for cookiejar in l_cookiejar%}
< option value = "{{cookiejar}}" > {{cookiejar}}< / option >
{%endfor%}
< / select >
< / div >
2019-02-26 13:50:48 +00:00
< / div >
< / div >
< button class = "btn btn-primary mt-2" >
< i class = "fas fa-spider" > < / i > Send to Spider
< / button >
< form >
< / div >
< / div >
< / div >
< / div >
< / div >
< / body >
< script >
var chart = {};
$(document).ready(function(){
$("#page-Crawler").addClass("active");
2019-02-28 13:56:14 +00:00
$("#nav_manual_crawler").addClass("active");
2020-08-24 20:31:41 +00:00
queue_type_selector_input_controler()
2019-02-26 13:50:48 +00:00
manual_crawler_input_controler();
2023-03-14 16:36:42 +00:00
$("#custom_frequency").hide();
2024-10-09 13:05:27 +00:00
$("#multiple_urls").hide();
2019-02-26 13:50:48 +00:00
2023-03-14 16:36:42 +00:00
$('#crawler_scheduler').on("change", function () {
2019-02-26 13:50:48 +00:00
manual_crawler_input_controler();
});
2020-08-24 20:31:41 +00:00
$('#queue_type_selector').on("change", function () {
queue_type_selector_input_controler();
});
2019-02-26 13:50:48 +00:00
});
2019-02-28 13:56:14 +00:00
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
2024-10-09 13:05:27 +00:00
function btn_single_url() {
$("#multiple_urls").hide();
$("#single_urls").show()
}
function btn_multiple_urls() {
$("#single_urls").hide()
$("#multiple_urls").show();
}
2019-02-26 13:50:48 +00:00
function manual_crawler_input_controler() {
2023-03-14 16:36:42 +00:00
if($('#crawler_scheduler').is(':checked')){
$("#frequency_inputs").show();
2019-02-26 13:50:48 +00:00
}else{
2023-03-14 16:36:42 +00:00
$("#frequency_inputs").hide();
}
}
function frequency_selector_update(obj) {
if(obj.value === "custom") {
$("#custom_frequency").show();
}else{
$("#custom_frequency").hide();
2019-02-26 13:50:48 +00:00
}
}
2020-08-24 20:31:41 +00:00
function queue_type_selector_input_controler() {
if($('#queue_type_selector').is(':checked')){
$("#div_crawler_queue_type").hide();
2022-10-25 14:25:19 +00:00
$("#div_proxy_name").show();
2020-08-24 20:31:41 +00:00
}else{
$("#div_crawler_queue_type").show();
2022-10-25 14:25:19 +00:00
$("#div_proxy_name").hide();
2020-08-24 20:31:41 +00:00
}
}
2019-02-26 13:50:48 +00:00
< / script >
