Merge branch 'master' into otp

This commit is contained in:
terrtia 2024-07-10 15:10:57 +02:00
commit c0ff9af8ae
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
4 changed files with 38 additions and 28 deletions

View file

@ -116,7 +116,7 @@ Requirement:
## Installation Notes
For Lacus Crawler installation instructions, refer to the [HOWTO](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md#crawler)
For Lacus Crawler and LibreTranslate installation instructions (if you want to use those features), refer to the [HOWTO](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md#crawler)
## Starting AIL
@ -126,7 +126,6 @@ To start AIL, use the following commands:
cd bin/
./LAUNCH.sh -l
```
You can access the AIL framework web interface at the following URL:
```
@ -171,7 +170,6 @@ If you use or reference AIL in an academic paper, you can cite it using the foll
## Screenshots
### Websites, Forums and Tor Hidden-Services
![Domain CIRCL](./doc/screenshots/domain_circl.png?raw=true "Tor hidden service crawler")
@ -222,11 +220,11 @@ If you use or reference AIL in an academic paper, you can cite it using the foll
```
Copyright (C) 2014 Jules Debra
Copyright (c) 2021 Olivier Sagit
Copyright (C) 2014-2023 CIRCL - Computer Incident Response Center Luxembourg (c/o smile, security made in Lëtzebuerg, Groupement d'Intérêt Economique)
Copyright (c) 2014-2023 Raphaël Vinot
Copyright (c) 2014-2023 Alexandre Dulaunoy
Copyright (c) 2016-2023 Sami Mokaddem
Copyright (c) 2018-2023 Thirion Aurélien
Copyright (C) 2014-2024 CIRCL - Computer Incident Response Center Luxembourg (c/o smile, security made in Lëtzebuerg, Groupement d'Intérêt Economique)
Copyright (c) 2014-2024 Raphaël Vinot
Copyright (c) 2014-2024 Alexandre Dulaunoy
Copyright (c) 2016-2024 Sami Mokaddem
Copyright (c) 2018-2024 Thirion Aurélien
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by

View file

@ -7,6 +7,8 @@ Importer Class
Import Content
"""
import json
import logging
import os
import requests
import sys
@ -18,10 +20,9 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
#################################
from exporter.abstract_exporter import AbstractExporter
from lib.ail_core import get_ail_uuid
# from ConfigLoader import ConfigLoader
# from lib.objects.abstract_object import AbstractObject
# from lib.Tracker import Tracker
logger = logging.getLogger()
class WebHookExporter(AbstractExporter, ABC):
def __init__(self, url=''):
@ -35,11 +36,9 @@ class WebHookExporter(AbstractExporter, ABC):
try:
response = requests.post(self.url, json=data)
if response.status_code >= 400:
print(f"Webhook request failed for {self.url}\nReason: {response.reason}")
# self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}")
logger.error(f"Webhook request failed for {self.url}\nReason: {response.reason}")
except Exception as e:
print(f"Webhook request failed for {self.url}\nReason: Something went wrong {e}")
# self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong")
logger.error(f"Webhook request failed for {self.url}\nReason: Something went wrong {e}")
class WebHookExporterTracker(WebHookExporter):
@ -48,17 +47,26 @@ class WebHookExporterTracker(WebHookExporter):
super().__init__(url=url)
# TODO Change exported keys
def export(self, tracker, obj):
def export(self, tracker, obj, matches=[]):
self.set_url(tracker.get_webhook())
data = {'trackerId': tracker.get_uuid(),
'trackerType': tracker.get_type(),
'tags': tracker.get_tags(),
data = {'version': 0,
'type': 'tracker:match',
'ail_uuid': get_ail_uuid(),
'tracker': {
'uuid': tracker.get_uuid(),
'type': tracker.get_type(),
'tags': list(tracker.get_tags()),
'tracker': tracker.get_tracked(),
# object
'itemId': obj.get_id(),
'itemURL': obj.get_link()}
# Item
# data['itemDate'] = obj.get_date()
# data["itemSource"] = obj.get_source()
},
'obj': {'type': obj.get_type(),
'subtype': obj.get_subtype(r_str=True),
'id': obj.get_id(),
'tags': list(obj.get_tags()),
'url': obj.get_link()
},
}
if matches:
data['matches'] = matches
# data = json.dumps(data)
self._export(data)

View file

@ -133,7 +133,9 @@ class Tracker_Regex(AbstractModule):
self.exporters['mail'].export(tracker, obj, matches)
if tracker.webhook_export():
self.exporters['webhook'].export(tracker, obj)
if not matches:
matches = self.extract_matches(re_matches)
self.exporters['webhook'].export(tracker, obj, matches)
if __name__ == "__main__":

View file

@ -151,7 +151,9 @@ class Tracker_Yara(AbstractModule):
# Webhook
if tracker.webhook_export():
self.exporters['webhook'].export(tracker, self.obj)
if not matches:
matches = self.extract_matches(data)
self.exporters['webhook'].export(tracker, self.obj, matches)
return yara.CALLBACK_CONTINUE