mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 00:28:22 +00:00
chg: [passiveDns D4 Client] add passiveDns D4 Client
This commit is contained in:
parent
b37d05842b
commit
9974823464
17 changed files with 301 additions and 19 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -46,6 +46,7 @@ configs/core.cfg.backup
|
|||
configs/update.cfg
|
||||
update/current_version
|
||||
files
|
||||
configs/d4client_passiveDNS_conf/uuid
|
||||
|
||||
# Trackers
|
||||
bin/trackers/yara/custom-rules/*
|
||||
|
|
|
@ -9,13 +9,18 @@ The DomClassifier modules extract and classify Internet domains/hostnames/IP add
|
|||
the out output of the Global module.
|
||||
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from packages import Paste
|
||||
from pubsublogger import publisher
|
||||
|
||||
import DomainClassifier.domainclassifier
|
||||
from Helper import Process
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||
import d4
|
||||
import item_basic
|
||||
|
||||
|
||||
def main():
|
||||
publisher.port = 6380
|
||||
|
@ -35,35 +40,42 @@ def main():
|
|||
|
||||
while True:
|
||||
try:
|
||||
message = p.get_from_set()
|
||||
item_id = p.get_from_set()
|
||||
|
||||
if message is not None:
|
||||
PST = Paste.Paste(message)
|
||||
else:
|
||||
if item_id is None:
|
||||
publisher.debug("Script DomClassifier is idling 1s")
|
||||
time.sleep(1)
|
||||
continue
|
||||
paste = PST.get_p_content()
|
||||
mimetype = PST._get_p_encoding()
|
||||
|
||||
if mimetype == "text/plain":
|
||||
c.text(rawtext=paste)
|
||||
item_content = item_basic.get_item_content(item_id)
|
||||
mimetype = item_basic.get_item_mimetype(item_id)
|
||||
item_basename = item_basic.get_basename(item_id)
|
||||
item_source = item_basic.get_source(item_id)
|
||||
item_date = item_basic.get_item_date(item_id)
|
||||
|
||||
if mimetype.split('/')[0] == "text":
|
||||
c.text(rawtext=item_content)
|
||||
c.potentialdomain()
|
||||
c.validdomain(rtype=['A'], extended=True)
|
||||
c.validdomain(rtype=['A'], passive_dns=True, extended=False)
|
||||
print(c.vdomain)
|
||||
|
||||
if c.vdomain and d4.is_passive_dns_enabled():
|
||||
for dns_record in c.vdomain:
|
||||
p.populate_set_out(dns_record)
|
||||
|
||||
localizeddomains = c.include(expression=cc_tld)
|
||||
if localizeddomains:
|
||||
print(localizeddomains)
|
||||
publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
|
||||
PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_rel_path))
|
||||
publisher.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {cc_tld};{item_id}")
|
||||
localizeddomains = c.localizedomain(cc=cc)
|
||||
|
||||
if localizeddomains:
|
||||
print(localizeddomains)
|
||||
publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
|
||||
PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_rel_path))
|
||||
publisher.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {cc};{item_id}")
|
||||
|
||||
except IOError:
|
||||
print("CRC Checksum Failed on :", PST.p_rel_path)
|
||||
publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(
|
||||
PST.p_source, PST.p_date, PST.p_name))
|
||||
print("CRC Checksum Failed on :", item_id)
|
||||
publisher.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
@ -152,6 +152,8 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Crawler_manager" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./Crawler_manager.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "D4_client" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./D4_client.py; read x"
|
||||
sleep 0.1
|
||||
|
||||
|
||||
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x"
|
||||
|
|
52
bin/core/D4_client.py
Executable file
52
bin/core/D4_client.py
Executable file
|
@ -0,0 +1,52 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
The D4_Client Module
|
||||
============================
|
||||
|
||||
The D4_Client modules send all DNS records to a D4 Server.
|
||||
Data produced by D4 sensors are ingested into
|
||||
a Passive DNS server which can be queried later to search for the Passive DNS records.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pubsublogger import publisher
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from Helper import Process
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||
import ConfigLoader
|
||||
import d4
|
||||
|
||||
# # TODO: lauch me in core screen
|
||||
# # TODO: check if already launched in core screen
|
||||
|
||||
if __name__ == '__main__':
|
||||
publisher.port = 6380
|
||||
publisher.channel = "Script"
|
||||
|
||||
config_section = 'D4_client'
|
||||
p = Process(config_section)
|
||||
publisher.info("""D4_client is Running""")
|
||||
|
||||
last_refresh = time.time()
|
||||
d4_client = d4.create_d4_client()
|
||||
|
||||
while True:
|
||||
if last_refresh < d4.get_config_last_update_time():
|
||||
d4_client = d4.create_d4_client()
|
||||
last_refresh = time.time()
|
||||
print('D4 Client: config updated')
|
||||
|
||||
dns_record = p.get_from_set()
|
||||
if dns_record is None:
|
||||
publisher.debug("Script D4_client is idling 1s")
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
if d4_client:
|
||||
# Send DNS Record to D4Server
|
||||
d4_client.send_manual_data(dns_record)
|
74
bin/lib/d4.py
Executable file
74
bin/lib/d4.py
Executable file
|
@ -0,0 +1,74 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import d4_pyclient
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||
import ConfigLoader
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
||||
config_loader = None
|
||||
|
||||
def get_ail_uuid():
|
||||
return r_serv_db.get('ail:uuid')
|
||||
|
||||
def get_d4_client_config_dir():
|
||||
return os.path.join(os.environ['AIL_HOME'], 'configs', 'd4client_passiveDNS_conf')
|
||||
|
||||
def create_d4_config_file(filename, content):
|
||||
if not os.path.isfile(filename):
|
||||
with open(filename, 'a') as f:
|
||||
f.write(content)
|
||||
|
||||
def get_d4_client_config():
|
||||
d4_client_config = get_d4_client_config_dir()
|
||||
filename = os.path.join(d4_client_config, 'uuid')
|
||||
if not os.path.isfile(filename):
|
||||
create_d4_config_file(filename, get_ail_uuid())
|
||||
return d4_client_config
|
||||
|
||||
def is_passive_dns_enabled(cache=True):
|
||||
if cache:
|
||||
res = r_cache.get('d4:passivedns:enabled')
|
||||
if res is None:
|
||||
res = r_serv_db.hget('d4:passivedns', 'enabled') == 'True'
|
||||
r_cache.set('d4:passivedns:enabled', res)
|
||||
return res
|
||||
else:
|
||||
return res == 'True'
|
||||
else:
|
||||
return r_serv_db.hget('d4:passivedns', 'enabled') == 'True'
|
||||
|
||||
def change_passive_dns_state(new_state):
|
||||
old_state = is_passive_dns_enabled(cache=False)
|
||||
if old_state != new_state:
|
||||
r_serv_db.hset('d4:passivedns', 'enabled', bool(new_state))
|
||||
r_cache.set('d4:passivedns:enabled', bool(new_state))
|
||||
update_time = time.time()
|
||||
r_serv_db.hset('d4:passivedns', 'update_time', update_time)
|
||||
r_cache.set('d4:passivedns:last_update_time', update_time)
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_config_last_update_time():
|
||||
last_update_time = r_cache.get('d4:passivedns:last_update_time')
|
||||
if not last_update_time:
|
||||
last_update_time = r_serv_db.hget('d4:passivedns', 'update_time')
|
||||
if not last_update_time:
|
||||
last_update_time = 0
|
||||
last_update_time = float(last_update_time)
|
||||
r_cache.set('d4:passivedns:last_update_time', last_update_time)
|
||||
return float(last_update_time)
|
||||
|
||||
def create_d4_client():
|
||||
if is_passive_dns_enabled():
|
||||
d4_client = d4_pyclient.D4Client(get_d4_client_config(), False)
|
||||
return d4_client
|
||||
else:
|
||||
return None
|
|
@ -5,6 +5,8 @@ import os
|
|||
import sys
|
||||
import gzip
|
||||
|
||||
import magic
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
|
@ -35,6 +37,9 @@ def get_item_date(item_id, add_separator=False):
|
|||
else:
|
||||
return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
|
||||
|
||||
def get_basename(item_id):
|
||||
return os.path.basename(item_id)
|
||||
|
||||
def get_source(item_id):
|
||||
return item_id.split('/')[-5]
|
||||
|
||||
|
@ -63,6 +68,9 @@ def get_item_content(item_id):
|
|||
item_content = ''
|
||||
return str(item_content)
|
||||
|
||||
def get_item_mimetype(item_id):
|
||||
return magic.from_buffer(get_item_content(item_id), mime=True)
|
||||
|
||||
#### TREE CHILD/FATHER ####
|
||||
def is_father(item_id):
|
||||
return r_serv_metadata.exists('paste_children:{}'.format(item_id))
|
||||
|
|
|
@ -21,6 +21,10 @@ subscribe = Redis_Global
|
|||
|
||||
[DomClassifier]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_D4_client
|
||||
|
||||
[D4_client]
|
||||
subscribe = Redis_D4_client
|
||||
|
||||
[TermTrackerMod]
|
||||
subscribe = Redis_Global
|
||||
|
|
1
configs/d4client_passiveDNS_conf/destination
Normal file
1
configs/d4client_passiveDNS_conf/destination
Normal file
|
@ -0,0 +1 @@
|
|||
d4pdns.circl.lu:4443
|
1
configs/d4client_passiveDNS_conf/key
Normal file
1
configs/d4client_passiveDNS_conf/key
Normal file
|
@ -0,0 +1 @@
|
|||
ail passivedns sensor key
|
1
configs/d4client_passiveDNS_conf/snaplen
Normal file
1
configs/d4client_passiveDNS_conf/snaplen
Normal file
|
@ -0,0 +1 @@
|
|||
4096
|
1
configs/d4client_passiveDNS_conf/source
Normal file
1
configs/d4client_passiveDNS_conf/source
Normal file
|
@ -0,0 +1 @@
|
|||
stdin
|
1
configs/d4client_passiveDNS_conf/type
Normal file
1
configs/d4client_passiveDNS_conf/type
Normal file
|
@ -0,0 +1 @@
|
|||
8
|
1
configs/d4client_passiveDNS_conf/version
Normal file
1
configs/d4client_passiveDNS_conf/version
Normal file
|
@ -0,0 +1 @@
|
|||
1
|
|
@ -14,6 +14,7 @@ import json
|
|||
import datetime
|
||||
|
||||
import git_status
|
||||
import d4
|
||||
|
||||
# ============ VARIABLES ============
|
||||
import Flask_config
|
||||
|
@ -21,8 +22,6 @@ import Flask_config
|
|||
app = Flask_config.app
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_db = Flask_config.r_serv_db
|
||||
max_preview_char = Flask_config.max_preview_char
|
||||
max_preview_modal = Flask_config.max_preview_modal
|
||||
REPO_ORIGIN = Flask_config.REPO_ORIGIN
|
||||
dict_update_description = Flask_config.dict_update_description
|
||||
email_regex = Flask_config.email_regex
|
||||
|
@ -274,5 +273,20 @@ def get_background_update_stats_json():
|
|||
else:
|
||||
return jsonify({})
|
||||
|
||||
@settings.route("/settings/passivedns", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def passive_dns():
|
||||
passivedns_enabled = d4.is_passive_dns_enabled()
|
||||
return render_template("passive_dns.html", passivedns_enabled=passivedns_enabled)
|
||||
|
||||
@settings.route("/settings/passivedns/change_state", methods=['GET'])
|
||||
@login_required
|
||||
@login_admin
|
||||
def passive_dns_change_state():
|
||||
new_state = request.args.get('state') == 'enable'
|
||||
passivedns_enabled = d4.change_passive_dns_state(new_state)
|
||||
return redirect(url_for('settings.passive_dns'))
|
||||
|
||||
# ========= REGISTRATION =========
|
||||
app.register_blueprint(settings, url_prefix=baseUrl)
|
||||
|
|
98
var/www/modules/settings/templates/passive_dns.html
Normal file
98
var/www/modules/settings/templates/passive_dns.html
Normal file
|
@ -0,0 +1,98 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Passive DNS - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'settings/menu_sidebar.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="d-flex justify-content-center my-4">
|
||||
<a href="https://d4-project.org/">
|
||||
<img src="{{ url_for('static', filename='image/d4-logo.png')}}" alt="D4 project">
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<p class="lead px-4">
|
||||
Passive DNS or pDNS is a service which records domain name system server (DNS) answers to DNS client requests.<br>
|
||||
In order to see the evolution of records over time, a history is recorded.<br>
|
||||
Various sources can be used to build a large sensor network.<br>
|
||||
<br>
|
||||
Enabling the D4 passive DNS sensor in AIL will contribute resolved domains and host to the global Passive DNS community operated by
|
||||
<a href="https://www.circl.lu/">
|
||||
CIRCL.lu
|
||||
</a>
|
||||
<br>
|
||||
<br>
|
||||
(if you want to have access to the global Passive DNS community
|
||||
<a href="https://www.circl.lu/services/passive-dns/">
|
||||
https://www.circl.lu/services/passive-dns
|
||||
</a>
|
||||
)
|
||||
</p>
|
||||
|
||||
|
||||
{% if passivedns_enabled %}
|
||||
<a href="{{ url_for('settings.passive_dns_change_state') }}?state=disable">
|
||||
<button class="btn btn-danger mx-4 my-2">
|
||||
Disable D4 Client
|
||||
</button>
|
||||
</a>
|
||||
{% else %}
|
||||
<a href="{{ url_for('settings.passive_dns_change_state') }}?state=enable">
|
||||
<button class="btn btn-primary mx-4 my-2">
|
||||
Enable D4 Client
|
||||
</button>
|
||||
</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</body>
|
||||
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
$("#nav_settings").addClass("active");
|
||||
$("#passive_dns").removeClass("text-muted");
|
||||
} );
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
|
||||
</script>
|
||||
|
||||
</html>
|
BIN
var/www/static/image/d4-logo.png
Normal file
BIN
var/www/static/image/d4-logo.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 17 KiB |
|
@ -17,6 +17,17 @@
|
|||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
<h5 class="d-flex text-muted w-100" id="nav_settings">
|
||||
<span>Settings</span>
|
||||
</h5>
|
||||
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('settings.passive_dns')}}" id="passive_dns">
|
||||
<img src="{{ url_for('static', filename='image/d4-logo.png')}}" alt="D4 project" style="width:25px;">
|
||||
<span>Passive DNS</span>
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
<h5 class="d-flex text-muted w-100 py-2" id="nav_my_profile">
|
||||
<span>My Profile</span>
|
||||
</h5>
|
||||
|
|
Loading…
Reference in a new issue