chg: [statistics] add domain statistics

bin/DomClassifier.py

@@ -10,6 +10,8 @@ the out output of the Global module.
 
 """
 import time
+import datetime
+import redis
 from packages import Paste
 from pubsublogger import publisher
 
@@ -26,6 +28,13 @@ def main():
     p = Process(config_section)
     addr_dns = p.config.get("DomClassifier", "dns")
 
+    # ARDB #
+    server_statistics = redis.StrictRedis(
+        host=p.config.get("ARDB_Statistics", "host"),
+        port=p.config.getint("ARDB_Statistics", "port"),
+        db=p.config.getint("ARDB_Statistics", "db"),
+        decode_responses=True)
+
     publisher.info("""ZMQ DomainClassifier is Running""")
 
     c = DomainClassifier.domainclassifier.Extract(rawtext="", nameservers=[addr_dns])
@@ -46,20 +55,31 @@ def main():
             paste = PST.get_p_content()
             mimetype = PST._get_p_encoding()
 
+            nb_domain = 0
+            nb_tld_domain = 0
+
             if mimetype == "text/plain":
                 c.text(rawtext=paste)
                 c.potentialdomain()
-                c.validdomain(rtype=['A'], extended=True)
+                valid = c.validdomain(rtype=['A'], extended=True)
-                localizeddomains = c.include(expression=cc_tld)
+                nb_domain = len(set(valid))
-                if localizeddomains:
+                if nb_domain > 0:
-                    print(localizeddomains)
+                    localizeddomains = c.include(expression=cc_tld)
-                    publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
+                    if localizeddomains:
-                        PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_path))
+                        nb_tld_domain = len(set(localizeddomains))
-                localizeddomains = c.localizedomain(cc=cc)
+                        publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
-                if localizeddomains:
+                            PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_path))
-                    print(localizeddomains)
+
-                    publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
+                    localizeddomains = c.localizedomain(cc=cc)
-                        PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_path))
+                    if localizeddomains:
+                        nb_tld_domain = nb_tld_domain + len(set(localizeddomains))
+                        publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
+                            PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_path))
+
+                    date = datetime.datetime.now().strftime("%Y%m")
+                    server_statistics.hincrby('domain_by_tld:'+date, 'ALL', nb_domain)
+                    if nb_tld_domain > 0:
+                        server_statistics.hincrby('domain_by_tld:'+date, cc, nb_tld_domain)
         except IOError:
             print("CRC Checksum Failed on :", PST.p_path)
             publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(

doc/api/create_lu_graph.py

@@ -8,6 +8,7 @@ lu
 import os
 import sys
 import redis
+import argparse
 import datetime
 import heapq
 import operator
@@ -18,7 +19,7 @@ sys.path.append(os.environ['AIL_BIN'])
 
 from Helper import Process
 
-def create_pie_chart(db_key, date, pie_title, path, save_name):
+def create_pie_chart(country ,db_key, date, pie_title, path, save_name):
 
     monthly_credential_by_tld = server_statistics.hkeys(db_key + date)
 
@@ -37,19 +38,23 @@ def create_pie_chart(db_key, date, pie_title, path, save_name):
     labels = []
     sizes = []
     explode = []  # only "explode" the 2nd slice (i.e. 'Hogs')
+    explode_value = 0
     for tld in mail_tld_top5:
-        labels.append(tld[0])
+        labels.append(tld[0] +' ('+str(tld[1])+')')
         sizes.append(tld[1])
-        explode.append(0)
+        explode.append(explode_value)
+        explode_value = explode_value +0.1
 
-    nb_tld = server_statistics.hget(db_key + date, 'lu')
+    nb_tld = server_statistics.hget(db_key + date, country)
     if nb_tld is not None:
         nb_tld = int(nb_tld)
     else:
         nb_tld = 0
-    labels.append('lu')
+    country_label = country + ' ('+str(nb_tld)+')'
-    sizes.append(nb_tld)
+    if country_label not in labels:
-    explode.append(0.3)  # only "explode" lu slice
+        labels.append(country_label)
+        sizes.append(nb_tld)
+        explode.append(explode_value)
     explode = tuple(explode)
 
     fig1, ax1 = plt.subplots()
@@ -85,13 +90,14 @@ def create_donut_chart(db_key, date, pie_title, path, save_name):
         recipe.append(tld[0])
         data.append(tld[1])
 
-    nb_tld = server_statistics.hget(db_key + date, 'lu')
+    nb_tld = server_statistics.hget(db_key + date, country)
     if nb_tld is not None:
         nb_tld = int(nb_tld)
     else:
         nb_tld = 0
-    recipe.append('lu')
+    if country not in recipe:
-    data.append(nb_tld)
+        recipe.append(country)
+        data.append(nb_tld)
 
     fig1, ax1 = plt.subplots(figsize=(6, 3), subplot_kw=dict(aspect="equal"))
 
@@ -118,6 +124,29 @@ def create_donut_chart(db_key, date, pie_title, path, save_name):
 
 if __name__ == '__main__':
 
+    parser = argparse.ArgumentParser(
+        description='''This script is a part of the Analysis Information Leak
+        framework. It create pie charts on a country statistics".''',
+        epilog='Example: ./create_lu_graph.py 0 lu now, create_lu_graph.py 0 lu 201807')
+
+    parser.add_argument('type', type=int, default=0,
+                        help='''The graph type (default 0),
+                        0: all,
+                        1: credential_pie,
+                        2: mail_pie
+                        3: sqlinjection_pie,
+                        4: domain_pie,''',
+                        choices=[0, 1, 2, 3, 4], action='store')
+
+    parser.add_argument('country', type=str, default="de",
+                        help='''The country code, de:default''',
+                        action='store')
+
+    parser.add_argument('date', type=str, default="now",
+                        help='''month %Y%m, example: 201810''', action='store')
+
+    args = parser.parse_args()
+
     path = os.path.join(os.environ['AIL_HOME'], 'doc') # path to module config file
 
     config_section = 'ARDB_Statistics'
@@ -131,7 +160,21 @@ if __name__ == '__main__':
         db=p.config.getint("ARDB_Statistics", "db"),
         decode_responses=True)
 
-    date = datetime.datetime.now().strftime("%Y%m")
+    if args.date == 'now' or len(args.date) != 6:
-    create_pie_chart('credential_by_tld:', date, "AIL: Credential leak by tld", path, 'AIL_credential_by_tld.png')
+        date = datetime.datetime.now().strftime("%Y%m")
-    create_pie_chart('mail_by_tld:', date, "AIL: mail leak by tld", path, 'AIL_mail_by_tld.png')
+    else:
-    create_pie_chart('SQLInjection_by_tld:', date, "AIL: sqlInjection by tld", path, 'AIL_sqlInjectionl_by_tld.png')
+        date = args.date
+
+    if args.type == 0:
+        create_pie_chart(args.country, 'credential_by_tld:', date, "AIL: Credential leak by tld", path, 'AIL_credential_by_tld.png')
+        create_pie_chart(args.country, 'mail_by_tld:', date, "AIL: mail leak by tld", path, 'AIL_mail_by_tld.png')
+        create_pie_chart(args.country, 'SQLInjection_by_tld:', date, "AIL: SQLInjection by tld", path, 'AIL_SQLInjection_by_tld.png')
+        create_pie_chart(args.country.upper(), 'domain_by_tld:', date, "AIL: Domain by tld", path, 'AIL_domain_by_tld.png')
+    elif args.type == 1:
+        create_pie_chart(args.country, 'credential_by_tld:', date, "AIL: Credential leak by tld", path, 'AIL_credential_by_tld.png')
+    elif args.type == 2:
+        create_pie_chart(args.country, 'mail_by_tld:', date, "AIL: mail leak by tld", path, 'AIL_mail_by_tld.png')
+    elif args.type == 3:
+        create_pie_chart(args.country, 'SQLInjection_by_tld:', date, "AIL: sqlInjection by tld", path, 'AIL_sqlInjectionl_by_tld.png')
+    elif args.type == 4:
+        create_pie_chart(args.country.upper(), 'domain_by_tld:', date, "AIL: Domain by tld", path, 'AIL_domain_by_tld.png')