chg: [passiveDns D4 Client] add passiveDns D4 Client

This commit is contained in:
Terrtia 2021-03-31 11:25:09 +02:00
parent b37d05842b
commit 9974823464
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
17 changed files with 301 additions and 19 deletions

1
.gitignore vendored
View file

@ -46,6 +46,7 @@ configs/core.cfg.backup
configs/update.cfg configs/update.cfg
update/current_version update/current_version
files files
configs/d4client_passiveDNS_conf/uuid
# Trackers # Trackers
bin/trackers/yara/custom-rules/* bin/trackers/yara/custom-rules/*

View file

@ -9,13 +9,18 @@ The DomClassifier modules extract and classify Internet domains/hostnames/IP add
the out output of the Global module. the out output of the Global module.
""" """
import os
import sys
import time import time
from packages import Paste
from pubsublogger import publisher from pubsublogger import publisher
import DomainClassifier.domainclassifier import DomainClassifier.domainclassifier
from Helper import Process from Helper import Process
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import d4
import item_basic
def main(): def main():
publisher.port = 6380 publisher.port = 6380
@ -35,35 +40,42 @@ def main():
while True: while True:
try: try:
message = p.get_from_set() item_id = p.get_from_set()
if message is not None: if item_id is None:
PST = Paste.Paste(message)
else:
publisher.debug("Script DomClassifier is idling 1s") publisher.debug("Script DomClassifier is idling 1s")
time.sleep(1) time.sleep(1)
continue continue
paste = PST.get_p_content()
mimetype = PST._get_p_encoding()
if mimetype == "text/plain": item_content = item_basic.get_item_content(item_id)
c.text(rawtext=paste) mimetype = item_basic.get_item_mimetype(item_id)
item_basename = item_basic.get_basename(item_id)
item_source = item_basic.get_source(item_id)
item_date = item_basic.get_item_date(item_id)
if mimetype.split('/')[0] == "text":
c.text(rawtext=item_content)
c.potentialdomain() c.potentialdomain()
c.validdomain(rtype=['A'], extended=True) c.validdomain(rtype=['A'], passive_dns=True, extended=False)
print(c.vdomain)
if c.vdomain and d4.is_passive_dns_enabled():
for dns_record in c.vdomain:
p.populate_set_out(dns_record)
localizeddomains = c.include(expression=cc_tld) localizeddomains = c.include(expression=cc_tld)
if localizeddomains: if localizeddomains:
print(localizeddomains) print(localizeddomains)
publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format( publisher.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {cc_tld};{item_id}")
PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_rel_path))
localizeddomains = c.localizedomain(cc=cc) localizeddomains = c.localizedomain(cc=cc)
if localizeddomains: if localizeddomains:
print(localizeddomains) print(localizeddomains)
publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format( publisher.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {cc};{item_id}")
PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_rel_path))
except IOError: except IOError:
print("CRC Checksum Failed on :", PST.p_rel_path) print("CRC Checksum Failed on :", item_id)
publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format( publisher.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed")
PST.p_source, PST.p_date, PST.p_name))
if __name__ == "__main__": if __name__ == "__main__":
main() main()

View file

@ -152,6 +152,8 @@ function launching_scripts {
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Crawler_manager" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./Crawler_manager.py; read x" screen -S "Script_AIL" -X screen -t "Crawler_manager" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./Crawler_manager.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "D4_client" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./D4_client.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x" screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x"

52
bin/core/D4_client.py Executable file
View file

@ -0,0 +1,52 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The D4_Client Module
============================
The D4_Client modules send all DNS records to a D4 Server.
Data produced by D4 sensors are ingested into
a Passive DNS server which can be queried later to search for the Passive DNS records.
"""
import os
import sys
import time
from pubsublogger import publisher
sys.path.append(os.environ['AIL_BIN'])
from Helper import Process
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import ConfigLoader
import d4
# # TODO: lauch me in core screen
# # TODO: check if already launched in core screen
if __name__ == '__main__':
publisher.port = 6380
publisher.channel = "Script"
config_section = 'D4_client'
p = Process(config_section)
publisher.info("""D4_client is Running""")
last_refresh = time.time()
d4_client = d4.create_d4_client()
while True:
if last_refresh < d4.get_config_last_update_time():
d4_client = d4.create_d4_client()
last_refresh = time.time()
print('D4 Client: config updated')
dns_record = p.get_from_set()
if dns_record is None:
publisher.debug("Script D4_client is idling 1s")
time.sleep(1)
continue
if d4_client:
# Send DNS Record to D4Server
d4_client.send_manual_data(dns_record)

74
bin/lib/d4.py Executable file
View file

@ -0,0 +1,74 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import time
import redis
import d4_pyclient
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import ConfigLoader
config_loader = ConfigLoader.ConfigLoader()
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_cache = config_loader.get_redis_conn("Redis_Cache")
config_loader = None
def get_ail_uuid():
return r_serv_db.get('ail:uuid')
def get_d4_client_config_dir():
return os.path.join(os.environ['AIL_HOME'], 'configs', 'd4client_passiveDNS_conf')
def create_d4_config_file(filename, content):
if not os.path.isfile(filename):
with open(filename, 'a') as f:
f.write(content)
def get_d4_client_config():
d4_client_config = get_d4_client_config_dir()
filename = os.path.join(d4_client_config, 'uuid')
if not os.path.isfile(filename):
create_d4_config_file(filename, get_ail_uuid())
return d4_client_config
def is_passive_dns_enabled(cache=True):
if cache:
res = r_cache.get('d4:passivedns:enabled')
if res is None:
res = r_serv_db.hget('d4:passivedns', 'enabled') == 'True'
r_cache.set('d4:passivedns:enabled', res)
return res
else:
return res == 'True'
else:
return r_serv_db.hget('d4:passivedns', 'enabled') == 'True'
def change_passive_dns_state(new_state):
old_state = is_passive_dns_enabled(cache=False)
if old_state != new_state:
r_serv_db.hset('d4:passivedns', 'enabled', bool(new_state))
r_cache.set('d4:passivedns:enabled', bool(new_state))
update_time = time.time()
r_serv_db.hset('d4:passivedns', 'update_time', update_time)
r_cache.set('d4:passivedns:last_update_time', update_time)
return True
return False
def get_config_last_update_time():
last_update_time = r_cache.get('d4:passivedns:last_update_time')
if not last_update_time:
last_update_time = r_serv_db.hget('d4:passivedns', 'update_time')
if not last_update_time:
last_update_time = 0
last_update_time = float(last_update_time)
r_cache.set('d4:passivedns:last_update_time', last_update_time)
return float(last_update_time)
def create_d4_client():
if is_passive_dns_enabled():
d4_client = d4_pyclient.D4Client(get_d4_client_config(), False)
return d4_client
else:
return None

View file

@ -5,6 +5,8 @@ import os
import sys import sys
import gzip import gzip
import magic
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader import ConfigLoader
@ -35,6 +37,9 @@ def get_item_date(item_id, add_separator=False):
else: else:
return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2]) return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
def get_basename(item_id):
return os.path.basename(item_id)
def get_source(item_id): def get_source(item_id):
return item_id.split('/')[-5] return item_id.split('/')[-5]
@ -63,6 +68,9 @@ def get_item_content(item_id):
item_content = '' item_content = ''
return str(item_content) return str(item_content)
def get_item_mimetype(item_id):
return magic.from_buffer(get_item_content(item_id), mime=True)
#### TREE CHILD/FATHER #### #### TREE CHILD/FATHER ####
def is_father(item_id): def is_father(item_id):
return r_serv_metadata.exists('paste_children:{}'.format(item_id)) return r_serv_metadata.exists('paste_children:{}'.format(item_id))

View file

@ -21,6 +21,10 @@ subscribe = Redis_Global
[DomClassifier] [DomClassifier]
subscribe = Redis_Global subscribe = Redis_Global
publish = Redis_D4_client
[D4_client]
subscribe = Redis_D4_client
[TermTrackerMod] [TermTrackerMod]
subscribe = Redis_Global subscribe = Redis_Global

View file

@ -0,0 +1 @@
d4pdns.circl.lu:4443

View file

@ -0,0 +1 @@
ail passivedns sensor key

View file

@ -0,0 +1 @@
4096

View file

@ -0,0 +1 @@
stdin

View file

@ -0,0 +1 @@
8

View file

@ -0,0 +1 @@
1

View file

@ -14,6 +14,7 @@ import json
import datetime import datetime
import git_status import git_status
import d4
# ============ VARIABLES ============ # ============ VARIABLES ============
import Flask_config import Flask_config
@ -21,8 +22,6 @@ import Flask_config
app = Flask_config.app app = Flask_config.app
baseUrl = Flask_config.baseUrl baseUrl = Flask_config.baseUrl
r_serv_db = Flask_config.r_serv_db r_serv_db = Flask_config.r_serv_db
max_preview_char = Flask_config.max_preview_char
max_preview_modal = Flask_config.max_preview_modal
REPO_ORIGIN = Flask_config.REPO_ORIGIN REPO_ORIGIN = Flask_config.REPO_ORIGIN
dict_update_description = Flask_config.dict_update_description dict_update_description = Flask_config.dict_update_description
email_regex = Flask_config.email_regex email_regex = Flask_config.email_regex
@ -274,5 +273,20 @@ def get_background_update_stats_json():
else: else:
return jsonify({}) return jsonify({})
@settings.route("/settings/passivedns", methods=['GET'])
@login_required
@login_read_only
def passive_dns():
passivedns_enabled = d4.is_passive_dns_enabled()
return render_template("passive_dns.html", passivedns_enabled=passivedns_enabled)
@settings.route("/settings/passivedns/change_state", methods=['GET'])
@login_required
@login_admin
def passive_dns_change_state():
new_state = request.args.get('state') == 'enable'
passivedns_enabled = d4.change_passive_dns_state(new_state)
return redirect(url_for('settings.passive_dns'))
# ========= REGISTRATION ========= # ========= REGISTRATION =========
app.register_blueprint(settings, url_prefix=baseUrl) app.register_blueprint(settings, url_prefix=baseUrl)

View file

@ -0,0 +1,98 @@
<!DOCTYPE html>
<html>
<head>
<title>Passive DNS - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'settings/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="d-flex justify-content-center my-4">
<a href="https://d4-project.org/">
<img src="{{ url_for('static', filename='image/d4-logo.png')}}" alt="D4 project">
</a>
</div>
<p class="lead px-4">
Passive DNS or pDNS is a service which records domain name system server (DNS) answers to DNS client requests.<br>
In order to see the evolution of records over time, a history is recorded.<br>
Various sources can be used to build a large sensor network.<br>
<br>
Enabling the D4 passive DNS sensor in AIL will contribute resolved domains and host to the global Passive DNS community operated by
<a href="https://www.circl.lu/">
CIRCL.lu
</a>
<br>
<br>
(if you want to have access to the global Passive DNS community
<a href="https://www.circl.lu/services/passive-dns/">
https://www.circl.lu/services/passive-dns
</a>
)
</p>
{% if passivedns_enabled %}
<a href="{{ url_for('settings.passive_dns_change_state') }}?state=disable">
<button class="btn btn-danger mx-4 my-2">
Disable D4 Client
</button>
</a>
{% else %}
<a href="{{ url_for('settings.passive_dns_change_state') }}?state=enable">
<button class="btn btn-primary mx-4 my-2">
Enable D4 Client
</button>
</a>
{% endif %}
</div>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$("#nav_settings").addClass("active");
$("#passive_dns").removeClass("text-muted");
} );
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</html>

Binary file not shown.

After

Width:  |  Height:  |  Size: 17 KiB

View file

@ -17,6 +17,17 @@
</a> </a>
</li> </li>
</ul> </ul>
<h5 class="d-flex text-muted w-100" id="nav_settings">
<span>Settings</span>
</h5>
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
<li class="nav-item">
<a class="nav-link" href="{{url_for('settings.passive_dns')}}" id="passive_dns">
<img src="{{ url_for('static', filename='image/d4-logo.png')}}" alt="D4 project" style="width:25px;">
<span>Passive DNS</span>
</a>
</li>
</ul>
<h5 class="d-flex text-muted w-100 py-2" id="nav_my_profile"> <h5 class="d-flex text-muted w-100 py-2" id="nav_my_profile">
<span>My Profile</span> <span>My Profile</span>
</h5> </h5>