Merge branch 'master' into otp

README.md

@@ -116,7 +116,7 @@ Requirement:
 
 ## Installation Notes
 
-For Lacus Crawler installation instructions, refer to the [HOWTO](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md#crawler)
+For Lacus Crawler and LibreTranslate installation instructions (if you want to use those features), refer to the [HOWTO](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md#crawler)
 
 ## Starting AIL
 
@@ -126,7 +126,6 @@ To start AIL, use the following commands:
 cd bin/
 ./LAUNCH.sh -l
 ```
-
 You can access the AIL framework web interface at the following URL: 
 
 ```
@@ -171,7 +170,6 @@ If you use or reference AIL in an academic paper, you can cite it using the foll
 
 ## Screenshots
 
-
 ### Websites, Forums and Tor Hidden-Services
 
 ![Domain CIRCL](./doc/screenshots/domain_circl.png?raw=true "Tor hidden service crawler")
@@ -222,11 +220,11 @@ If you use or reference AIL in an academic paper, you can cite it using the foll
 ```
     Copyright (C) 2014 Jules Debra
     Copyright (c) 2021 Olivier Sagit
-    Copyright (C) 2014-2023 CIRCL - Computer Incident Response Center Luxembourg (c/o smile, security made in Lëtzebuerg, Groupement d'Intérêt Economique)
+    Copyright (C) 2014-2024 CIRCL - Computer Incident Response Center Luxembourg (c/o smile, security made in Lëtzebuerg, Groupement d'Intérêt Economique)
-    Copyright (c) 2014-2023 Raphaël Vinot
+    Copyright (c) 2014-2024 Raphaël Vinot
-    Copyright (c) 2014-2023 Alexandre Dulaunoy
+    Copyright (c) 2014-2024 Alexandre Dulaunoy
-    Copyright (c) 2016-2023 Sami Mokaddem
+    Copyright (c) 2016-2024 Sami Mokaddem
-    Copyright (c) 2018-2023 Thirion Aurélien
+    Copyright (c) 2018-2024 Thirion Aurélien
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU Affero General Public License as published by

bin/exporter/WebHookExporter.py

@@ -7,6 +7,8 @@ Importer Class
 Import Content
 
 """
+import json
+import logging
 import os
 import requests
 import sys
@@ -18,10 +20,9 @@ sys.path.append(os.environ['AIL_BIN'])
 # Import Project packages
 #################################
 from exporter.abstract_exporter import AbstractExporter
+from lib.ail_core import get_ail_uuid
 
-# from ConfigLoader import ConfigLoader
+logger = logging.getLogger()
-# from lib.objects.abstract_object import AbstractObject
-# from lib.Tracker import Tracker
 
 class WebHookExporter(AbstractExporter, ABC):
     def __init__(self, url=''):
@@ -35,11 +36,9 @@ class WebHookExporter(AbstractExporter, ABC):
         try:
             response = requests.post(self.url, json=data)
             if response.status_code >= 400:
-                print(f"Webhook request failed for {self.url}\nReason: {response.reason}")
+                logger.error(f"Webhook request failed for {self.url}\nReason: {response.reason}")
-                # self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}")
         except Exception as e:
-            print(f"Webhook request failed for {self.url}\nReason: Something went wrong {e}")
+            logger.error(f"Webhook request failed for {self.url}\nReason: Something went wrong {e}")
-            # self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong")
 
 
 class WebHookExporterTracker(WebHookExporter):
@@ -48,17 +47,26 @@ class WebHookExporterTracker(WebHookExporter):
         super().__init__(url=url)
 
     # TODO Change exported keys
-    def export(self, tracker, obj):
+    def export(self, tracker, obj, matches=[]):
         self.set_url(tracker.get_webhook())
-        data = {'trackerId': tracker.get_uuid(),
+        data = {'version': 0,
-                'trackerType': tracker.get_type(),
+                'type': 'tracker:match',
-                'tags': tracker.get_tags(),
+                'ail_uuid': get_ail_uuid(),
-                'tracker': tracker.get_tracked(),
+                'tracker': {
-                # object
+                    'uuid': tracker.get_uuid(),
-                'itemId': obj.get_id(),
+                    'type': tracker.get_type(),
-                'itemURL': obj.get_link()}
+                    'tags': list(tracker.get_tags()),
-        # Item
+                    'tracker': tracker.get_tracked(),
-        # data['itemDate'] = obj.get_date()
+                },
-        # data["itemSource"] = obj.get_source()
+                'obj': {'type': obj.get_type(),
+                        'subtype': obj.get_subtype(r_str=True),
+                        'id': obj.get_id(),
+                        'tags': list(obj.get_tags()),
+                        'url': obj.get_link()
+                        },
+                }
+        if matches:
+            data['matches'] = matches
 
+        # data = json.dumps(data)
         self._export(data)

bin/trackers/Tracker_Regex.py

@@ -133,7 +133,9 @@ class Tracker_Regex(AbstractModule):
                 self.exporters['mail'].export(tracker, obj, matches)
 
             if tracker.webhook_export():
-                self.exporters['webhook'].export(tracker, obj)
+                if not matches:
+                    matches = self.extract_matches(re_matches)
+                self.exporters['webhook'].export(tracker, obj, matches)
 
 
 if __name__ == "__main__":

bin/trackers/Tracker_Yara.py

@@ -151,7 +151,9 @@ class Tracker_Yara(AbstractModule):
 
             # Webhook
             if tracker.webhook_export():
-                self.exporters['webhook'].export(tracker, self.obj)
+                if not matches:
+                    matches = self.extract_matches(data)
+                self.exporters['webhook'].export(tracker, self.obj, matches)
 
         return yara.CALLBACK_CONTINUE