From e455d244d82a322748b9045120c95c71157c6bb0 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 31 Jul 2018 10:36:29 +0200 Subject: [PATCH] chg: [statistics] add domain statistics --- bin/DomClassifier.py | 42 ++++++++++++++++------ doc/api/create_lu_graph.py | 71 ++++++++++++++++++++++++++++++-------- 2 files changed, 88 insertions(+), 25 deletions(-) diff --git a/bin/DomClassifier.py b/bin/DomClassifier.py index aed87a55..268607e8 100755 --- a/bin/DomClassifier.py +++ b/bin/DomClassifier.py @@ -10,6 +10,8 @@ the out output of the Global module. """ import time +import datetime +import redis from packages import Paste from pubsublogger import publisher @@ -26,6 +28,13 @@ def main(): p = Process(config_section) addr_dns = p.config.get("DomClassifier", "dns") + # ARDB # + server_statistics = redis.StrictRedis( + host=p.config.get("ARDB_Statistics", "host"), + port=p.config.getint("ARDB_Statistics", "port"), + db=p.config.getint("ARDB_Statistics", "db"), + decode_responses=True) + publisher.info("""ZMQ DomainClassifier is Running""") c = DomainClassifier.domainclassifier.Extract(rawtext="", nameservers=[addr_dns]) @@ -46,20 +55,31 @@ def main(): paste = PST.get_p_content() mimetype = PST._get_p_encoding() + nb_domain = 0 + nb_tld_domain = 0 + if mimetype == "text/plain": c.text(rawtext=paste) c.potentialdomain() - c.validdomain(rtype=['A'], extended=True) - localizeddomains = c.include(expression=cc_tld) - if localizeddomains: - print(localizeddomains) - publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format( - PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_path)) - localizeddomains = c.localizedomain(cc=cc) - if localizeddomains: - print(localizeddomains) - publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format( - PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_path)) + valid = c.validdomain(rtype=['A'], extended=True) + nb_domain = len(set(valid)) + if nb_domain > 0: + localizeddomains = c.include(expression=cc_tld) + if localizeddomains: + nb_tld_domain = len(set(localizeddomains)) + publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format( + PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_path)) + + localizeddomains = c.localizedomain(cc=cc) + if localizeddomains: + nb_tld_domain = nb_tld_domain + len(set(localizeddomains)) + publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format( + PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_path)) + + date = datetime.datetime.now().strftime("%Y%m") + server_statistics.hincrby('domain_by_tld:'+date, 'ALL', nb_domain) + if nb_tld_domain > 0: + server_statistics.hincrby('domain_by_tld:'+date, cc, nb_tld_domain) except IOError: print("CRC Checksum Failed on :", PST.p_path) publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format( diff --git a/doc/api/create_lu_graph.py b/doc/api/create_lu_graph.py index 4bcff2a6..1c9440ef 100755 --- a/doc/api/create_lu_graph.py +++ b/doc/api/create_lu_graph.py @@ -8,6 +8,7 @@ lu import os import sys import redis +import argparse import datetime import heapq import operator @@ -18,7 +19,7 @@ sys.path.append(os.environ['AIL_BIN']) from Helper import Process -def create_pie_chart(db_key, date, pie_title, path, save_name): +def create_pie_chart(country ,db_key, date, pie_title, path, save_name): monthly_credential_by_tld = server_statistics.hkeys(db_key + date) @@ -37,19 +38,23 @@ def create_pie_chart(db_key, date, pie_title, path, save_name): labels = [] sizes = [] explode = [] # only "explode" the 2nd slice (i.e. 'Hogs') + explode_value = 0 for tld in mail_tld_top5: - labels.append(tld[0]) + labels.append(tld[0] +' ('+str(tld[1])+')') sizes.append(tld[1]) - explode.append(0) + explode.append(explode_value) + explode_value = explode_value +0.1 - nb_tld = server_statistics.hget(db_key + date, 'lu') + nb_tld = server_statistics.hget(db_key + date, country) if nb_tld is not None: nb_tld = int(nb_tld) else: nb_tld = 0 - labels.append('lu') - sizes.append(nb_tld) - explode.append(0.3) # only "explode" lu slice + country_label = country + ' ('+str(nb_tld)+')' + if country_label not in labels: + labels.append(country_label) + sizes.append(nb_tld) + explode.append(explode_value) explode = tuple(explode) fig1, ax1 = plt.subplots() @@ -85,13 +90,14 @@ def create_donut_chart(db_key, date, pie_title, path, save_name): recipe.append(tld[0]) data.append(tld[1]) - nb_tld = server_statistics.hget(db_key + date, 'lu') + nb_tld = server_statistics.hget(db_key + date, country) if nb_tld is not None: nb_tld = int(nb_tld) else: nb_tld = 0 - recipe.append('lu') - data.append(nb_tld) + if country not in recipe: + recipe.append(country) + data.append(nb_tld) fig1, ax1 = plt.subplots(figsize=(6, 3), subplot_kw=dict(aspect="equal")) @@ -118,6 +124,29 @@ def create_donut_chart(db_key, date, pie_title, path, save_name): if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='''This script is a part of the Analysis Information Leak + framework. It create pie charts on a country statistics".''', + epilog='Example: ./create_lu_graph.py 0 lu now, create_lu_graph.py 0 lu 201807') + + parser.add_argument('type', type=int, default=0, + help='''The graph type (default 0), + 0: all, + 1: credential_pie, + 2: mail_pie + 3: sqlinjection_pie, + 4: domain_pie,''', + choices=[0, 1, 2, 3, 4], action='store') + + parser.add_argument('country', type=str, default="de", + help='''The country code, de:default''', + action='store') + + parser.add_argument('date', type=str, default="now", + help='''month %Y%m, example: 201810''', action='store') + + args = parser.parse_args() + path = os.path.join(os.environ['AIL_HOME'], 'doc') # path to module config file config_section = 'ARDB_Statistics' @@ -131,7 +160,21 @@ if __name__ == '__main__': db=p.config.getint("ARDB_Statistics", "db"), decode_responses=True) - date = datetime.datetime.now().strftime("%Y%m") - create_pie_chart('credential_by_tld:', date, "AIL: Credential leak by tld", path, 'AIL_credential_by_tld.png') - create_pie_chart('mail_by_tld:', date, "AIL: mail leak by tld", path, 'AIL_mail_by_tld.png') - create_pie_chart('SQLInjection_by_tld:', date, "AIL: sqlInjection by tld", path, 'AIL_sqlInjectionl_by_tld.png') + if args.date == 'now' or len(args.date) != 6: + date = datetime.datetime.now().strftime("%Y%m") + else: + date = args.date + + if args.type == 0: + create_pie_chart(args.country, 'credential_by_tld:', date, "AIL: Credential leak by tld", path, 'AIL_credential_by_tld.png') + create_pie_chart(args.country, 'mail_by_tld:', date, "AIL: mail leak by tld", path, 'AIL_mail_by_tld.png') + create_pie_chart(args.country, 'SQLInjection_by_tld:', date, "AIL: SQLInjection by tld", path, 'AIL_SQLInjection_by_tld.png') + create_pie_chart(args.country.upper(), 'domain_by_tld:', date, "AIL: Domain by tld", path, 'AIL_domain_by_tld.png') + elif args.type == 1: + create_pie_chart(args.country, 'credential_by_tld:', date, "AIL: Credential leak by tld", path, 'AIL_credential_by_tld.png') + elif args.type == 2: + create_pie_chart(args.country, 'mail_by_tld:', date, "AIL: mail leak by tld", path, 'AIL_mail_by_tld.png') + elif args.type == 3: + create_pie_chart(args.country, 'SQLInjection_by_tld:', date, "AIL: sqlInjection by tld", path, 'AIL_sqlInjectionl_by_tld.png') + elif args.type == 4: + create_pie_chart(args.country.upper(), 'domain_by_tld:', date, "AIL: Domain by tld", path, 'AIL_domain_by_tld.png')