Added a seemingly working version of ailleak to misp

bin/ailleakObject.py

@@ -4,36 +4,56 @@
 from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator
 from packages import Paste
 import datetime
+import json
 
 class AilleakObject(AbstractMISPObjectGenerator):
-    def __init__(self, moduleName, p_source, p_date):
+    def __init__(self, moduleName, p_source, p_date, p_content, p_duplicate):
-    #def __init__(self, moduleName, p_source, p_date, p_content):
         super(AbstractMISPObjectGenerator, self).__init__('ail-leak')
         self.moduleName = moduleName
         self.p_source = p_source
         self.p_date = p_date
-        #self.p_content = p_content
+        self.p_content = p_content
+        self.p_duplicate = p_duplicate
         self.generate_attributes()
 
     def generate_attributes(self):
         self.add_attribute('type', value=self.moduleName)
         self.add_attribute('origin', value=self.p_source)
         self.add_attribute('last-seen', value=self.p_date)
-        #self.add_attribute('raw-data', value=self.p_content)
+        self.add_attribute('duplicate-list', value=self.p_duplicate)
+        self.add_attribute('raw-data', value=self.p_content)
 
-class objectWrapper:
+class ObjectWrapper:
-    def __init__(self, moduleName, path, pymisp):
+    def __init__(self, pymisp):
+        self.pymisp = pymisp
+        self.currentID_date = None
+        self.eventID_to_push = self.get_daily_event_id()
+
+    def add_new_object(self, moduleName, path):
         self.moduleName = moduleName
         self.path = path
-        self.pymisp = pymisp
         self.paste = Paste.Paste(path)
         self.p_date = self.date_to_str(self.paste.p_date)
         self.p_source = self.paste.supposed_url
-        self.p_content = self.paste.get_p_content()
+        self.p_content = self.paste.get_p_content().decode('utf8')
-    
+        
-        self.eventID_to_push = self.get_daily_event_id()
+        temp = self.paste._get_p_duplicate()
-        self.mispObject = AilleakObject(self.moduleName, self.p_source, self.p_date)
+        try:
-        #self.mispObject = AilleakObject(self.moduleName, self.p_source, self.p_date, self.p_content)
+            temp = temp.decode('utf8')
+        except AttributeError:
+            print('decode error')
+        #beautifier
+        temp = json.loads(temp)
+        to_ret = []
+        for dup in temp:
+            algo = dup[0]
+            path = dup[1].split('/')[-5:]
+            perc = dup[2]
+            to_ret.append([path, algo, perc])
+        self.p_duplicate = str(to_ret)
+        
+
+        self.mispObject = AilleakObject(self.moduleName, self.p_source, self.p_date, self.p_content, self.p_duplicate)
 
         '''
         # duplicated
@@ -42,12 +62,13 @@ class objectWrapper:
         self.add_attribute('duplicate', value=is_duplicate)
         '''
 
+
     def date_to_str(self, date):
         return "{0}-{1}-{2}".format(date.year, date.month, date.day)
 
     def get_all_related_events(self):
         to_search = "Daily AIL-leaks"
-        result = pymisp.search_all(to_search)
+        result = self.pymisp.search_all(to_search)
         events = []
         for e in result['response']:
             events.append({'id': e['Event']['id'], 'org_id': e['Event']['org_id'], 'info': e['Event']['info']})
@@ -61,10 +82,12 @@ class objectWrapper:
             e_id = dic['id']
             if info == to_match:
                 print('Found: ', info, '->', e_id)
+                self.currentID_date = datetime.date.today()
                 return e_id
         created_event = self.create_daily_event()['Event']
         new_id = created_event['id']
         print('New event created:', new_id)
+        self.currentID_date = datetime.date.today()
         return new_id
 
 
@@ -89,11 +112,14 @@ class objectWrapper:
 
     # Publish object to MISP
     def pushToMISP(self):
+        if self.currentID_date != datetime.date.today(): #refresh id
+            self.eventID_to_push = self.get_daily_event_id()
+
         mispTYPE = 'ail-leak'
         try:
-            templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0]
+            templateID = [x['ObjectTemplate']['id'] for x in self.pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0]
         except IndexError:
-            valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()])
+            valid_types = ", ".join([x['ObjectTemplate']['name'] for x in self.pymisp.get_object_templates_list()])
             print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types))
         r = self.pymisp.add_object(self.eventID_to_push, templateID, self.mispObject)
         if 'errors' in r:
@@ -101,7 +127,7 @@ class objectWrapper:
         else:
             print('Pushed:', self.moduleName, '->', self.p_source)
 
-
+'''
 if __name__ == "__main__":
 
     import sys
@@ -114,5 +140,6 @@ if __name__ == "__main__":
     moduleName = "Credentials"
     path = "/home/sami/git/AIL-framework/PASTES/archive/pastebin.com_pro/2017/08/23/bPFaJymf.gz"
 
-    wrapper = objectWrapper(moduleName, path, pymisp)
+    wrapper = ObjectWrapper(moduleName, path, pymisp)
     wrapper.pushToMISP()
+'''

bin/alertHandler.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 """
@@ -34,8 +34,8 @@ if __name__ == "__main__":
 
     p = Process(config_section)
     pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
-    eventID = "9356"
+    print('Connected to MISP:', misp_url)
-    mispTYPE = 'ail-leak'
+    wrapper = ailleakObject.ObjectWrapper(pymisp)
 
     # port generated automatically depending on the date
     curYear = datetime.now().year
@@ -50,6 +50,7 @@ if __name__ == "__main__":
     while True:
             message = p.get_from_set()
             if message is not None:
+                message = message.decode('utf8') #decode because of pyhton3
                 module_name, p_path = message.split(';')
                 #PST = Paste.Paste(p_path)
             else:
@@ -64,16 +65,10 @@ if __name__ == "__main__":
 
             publisher.info('Saved warning paste {}'.format(p_path))
 
-            # Create MISP AIL-leak object
+            # Create MISP AIL-leak object and push it
-            misp_object = AilleakObject(moduleName, path)
+            allowed_modules = ['credential']
-            print('validate mispobj', misp_object._validate())
+            if module_name in allowed_modules:
-            print(misp_object)
+                wrapper.add_new_object(module_name, p_path)
-
+                wrapper.pushToMISP()
-            # Publish object to MISP
+            else:
-            try:
+                print('not pushing to MISP:', module_name, p_path)
-                templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0]
-            except IndexError:
-                valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()])
-                print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types))
-                continue
-            #r = pymisp.add_object(eventID, templateID, misp_object)