From c008366f0220ba149d4c9156b85b731343b3e4ff Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 25 May 2023 14:33:12 +0200 Subject: [PATCH 001/238] chg: [new title object] add new title object + correlation on page title --- bin/crawlers/Crawler.py | 9 +- bin/lib/ail_core.py | 2 +- bin/lib/correlations_engine.py | 7 +- bin/lib/crawlers.py | 49 +- bin/lib/module_extractor.py | 19 +- bin/lib/objects/Titles.py | 114 ++++ bin/lib/objects/abstract_daterange_object.py | 95 ++- bin/lib/objects/abstract_object.py | 2 +- bin/lib/objects/ail_objects.py | 7 +- bin/modules/Phone.py | 2 +- var/www/Flask_server.py | 2 + var/www/blueprints/objects_title.py | 86 +++ .../correlation/metadata_card_cve.html | 2 +- .../correlation/metadata_card_title.html | 173 +++++ .../correlation/show_correlation.html | 2 + .../crawler/crawler_splash/showDomain.html | 43 ++ .../objects/title/TitleDaterange.html | 611 ++++++++++++++++++ .../templates/sidebars/sidebar_objects.html | 7 +- 18 files changed, 1205 insertions(+), 27 deletions(-) create mode 100755 bin/lib/objects/Titles.py create mode 100644 var/www/blueprints/objects_title.py create mode 100644 var/www/templates/correlation/metadata_card_title.html create mode 100644 var/www/templates/objects/title/TitleDaterange.html diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index fd8a758f..4b499eb9 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -19,6 +19,7 @@ from lib.ConfigLoader import ConfigLoader from lib.objects.Domains import Domain from lib.objects.Items import Item from lib.objects import Screenshots +from lib.objects import Titles logging.config.dictConfig(ail_logger.get_config(name='crawlers')) @@ -252,6 +253,13 @@ class Crawler(AbstractModule): self.root_item = item_id parent_id = item_id + item = Item(item_id) + + title_content = crawlers.extract_title_from_html(entries['html']) + if title_content: + title = Titles.create_title(title_content) + title.add(item.get_date(), item_id) + # SCREENSHOT if self.screenshot: if 'png' in entries and entries['png']: @@ -260,7 +268,6 @@ class Crawler(AbstractModule): if not screenshot.is_tags_safe(): unsafe_tag = 'dark-web:topic="pornography-child-exploitation"' self.domain.add_tag(unsafe_tag) - item = Item(item_id) item.add_tag(unsafe_tag) # Remove Placeholder pages # TODO Replace with warning list ??? if screenshot.id not in self.placeholder_screenshots: diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 6f4ed42e..a9128489 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -15,7 +15,7 @@ config_loader = ConfigLoader() r_serv_db = config_loader.get_db_conn("Kvrocks_DB") config_loader = None -AIL_OBJECTS = sorted({'cve', 'cryptocurrency', 'decoded', 'domain', 'item', 'pgp', 'screenshot', 'username'}) +AIL_OBJECTS = sorted({'cve', 'cryptocurrency', 'decoded', 'domain', 'item', 'pgp', 'screenshot', 'title', 'username'}) def get_ail_uuid(): ail_uuid = r_serv_db.get('ail:uuid') diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index c0e8d370..f09d5c5f 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -44,11 +44,12 @@ CORRELATION_TYPES_BY_OBJ = { "cryptocurrency": ["domain", "item"], "cve": ["domain", "item"], "decoded": ["domain", "item"], - "domain": ["cve", "cryptocurrency", "decoded", "item", "pgp", "username", "screenshot"], - "item": ["cve", "cryptocurrency", "decoded", "domain", "pgp", "username", "screenshot"], + "domain": ["cve", "cryptocurrency", "decoded", "item", "pgp", "title", "screenshot", "username"], + "item": ["cve", "cryptocurrency", "decoded", "domain", "pgp", "screenshot", "title", "username"], "pgp": ["domain", "item"], - "username": ["domain", "item"], "screenshot": ["domain", "item"], + "title": ["domain", "item"], + "username": ["domain", "item"], } def get_obj_correl_types(obj_type): diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 2a9c2a01..d8f4c890 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -183,6 +183,47 @@ def extract_favicon_from_html(html, url): # # # - - # # # +# # # # # # # # +# # +# TITLE # +# # +# # # # # # # # + +def extract_title_from_html(html): + soup = BeautifulSoup(html, 'html.parser') + title = soup.title + if title: + return str(title.string) + return '' + +def extract_description_from_html(html): + soup = BeautifulSoup(html, 'html.parser') + description = soup.find('meta', attrs={'name': 'description'}) + if description: + return description['content'] + return '' + +def extract_description_from_html(html): + soup = BeautifulSoup(html, 'html.parser') + description = soup.find('meta', attrs={'name': 'description'}) + if description: + return description['content'] + return '' + +def extract_keywords_from_html(html): + soup = BeautifulSoup(html, 'html.parser') + keywords = soup.find('meta', attrs={'name': 'keywords'}) + if keywords: + return keywords['content'] + return '' + +def extract_author_from_html(html): + soup = BeautifulSoup(html, 'html.parser') + keywords = soup.find('meta', attrs={'name': 'author'}) + if keywords: + return keywords['content'] + return '' +# # # - - # # # ################################################################################ @@ -1711,7 +1752,7 @@ def test_ail_crawlers(): load_blacklist() # if __name__ == '__main__': - # task = CrawlerTask('2dffcae9-8f66-4cfa-8e2c-de1df738a6cd') - # print(task.get_meta()) - # _clear_captures() - +# item = Item('crawled/2023/03/06/foo.bec50a87b5-0c21-4ed4-9cb2-2d717a7a6507') +# content = item.get_content() +# r = extract_author_from_html(content) +# print(r) diff --git a/bin/lib/module_extractor.py b/bin/lib/module_extractor.py index 99d51a6b..02f499fd 100755 --- a/bin/lib/module_extractor.py +++ b/bin/lib/module_extractor.py @@ -3,7 +3,6 @@ import json import os import sys -import time import yara @@ -15,6 +14,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from lib.objects import ail_objects from lib.objects.Items import Item +from lib.objects.Titles import Title from lib import correlations_engine from lib import regex_helper from lib.ConfigLoader import ConfigLoader @@ -58,18 +58,25 @@ def get_correl_match(extract_type, obj_id, content): correl = correlations_engine.get_correlation_by_correl_type('item', '', obj_id, extract_type) to_extract = [] map_subtype = {} + map_value_id = {} for c in correl: subtype, value = c.split(':', 1) - map_subtype[value] = subtype - to_extract.append(value) + if extract_type == 'title': + title = Title(value).get_content() + to_extract.append(title) + map_value_id[title] = value + else: + map_subtype[value] = subtype + to_extract.append(value) + map_value_id[value] = value if to_extract: objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj_id, content) for obj in objs: - if map_subtype[obj[2]]: + if map_subtype.get(obj[2]): subtype = map_subtype[obj[2]] else: subtype = '' - extracted.append([obj[0], obj[1], obj[2], f'{extract_type}:{subtype}:{obj[2]}']) + extracted.append([obj[0], obj[1], obj[2], f'{extract_type}:{subtype}:{map_value_id[obj[2]]}']) return extracted def _get_yara_match(data): @@ -173,7 +180,7 @@ def extract(obj_id, content=None): if matches: extracted = extracted + matches - for obj_t in ['cve', 'cryptocurrency', 'username']: # Decoded, PGP->extract bloc + for obj_t in ['cve', 'cryptocurrency', 'title', 'username']: # Decoded, PGP->extract bloc matches = get_correl_match(obj_t, obj_id, content) if matches: extracted = extracted + matches diff --git a/bin/lib/objects/Titles.py b/bin/lib/objects/Titles.py new file mode 100755 index 00000000..3c682224 --- /dev/null +++ b/bin/lib/objects/Titles.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys + +from hashlib import sha256 +from flask import url_for + +from pymisp import MISPObject + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader +from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects + +config_loader = ConfigLoader() +r_objects = config_loader.get_db_conn("Kvrocks_Objects") +baseurl = config_loader.get_config_str("Notifications", "ail_domain") +config_loader = None + + +class Title(AbstractDaterangeObject): + """ + AIL Title Object. + """ + + def __init__(self, id): + super(Title, self).__init__('title', id) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True), + # 'compress': 'gzip'} + # return payload + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + # # TODO: + pass + + def get_content(self, r_type='str'): + if r_type == 'str': + return self._get_field('content') + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', type=self.type, id=self.id) + else: + url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}' + return url + + # TODO # CHANGE COLOR + def get_svg_icon(self): + return {'style': 'fas', 'icon': '\uf1dc', 'color': '#1E88E5', 'radius': 5} + + def get_misp_object(self): + obj_attrs = [] + obj = MISPObject('tsk-web-history') + obj.first_seen = self.get_first_seen() + obj.last_seen = self.get_last_seen() + + obj_attrs.append(obj.add_attribute('title', value=self.get_content())) + for obj_attr in obj_attrs: + for tag in self.get_tags(): + obj_attr.add_tag(tag) + return obj + + def get_meta(self, options=set()): + meta = self._get_meta(options=options) + meta['id'] = self.id + meta['tags'] = self.get_tags(r_list=True) + meta['content'] = self.get_content() + return meta + + def add(self, date, item_id): + self._add(date, item_id) + + def create(self, content, _first_seen=None, _last_seen=None): + self._set_field('content', content) + self._create() + + +def create_title(content): + title_id = sha256(content.encode()).hexdigest() + title = Title(title_id) + if not title.exists(): + title.create(content) + return title + +class Titles(AbstractDaterangeObjects): + """ + Titles Objects + """ + def __init__(self): + super().__init__('title') + + def get_metas(self, obj_ids, options=set()): + return self._get_metas(Title, obj_ids, options=options) + + def sanitize_name_to_search(self, name_to_search): + return name_to_search + + +# if __name__ == '__main__': +# from lib import crawlers +# from lib.objects import Items +# for item in Items.get_all_items_objects(filters={'sources': ['crawled']}): +# title_content = crawlers.extract_title_from_html(item.get_content()) +# if title_content: +# print(item.id, title_content) +# title = create_title(title_content) +# title.add(item.get_date(), item.id) diff --git a/bin/lib/objects/abstract_daterange_object.py b/bin/lib/objects/abstract_daterange_object.py index 882c11c7..59a579f4 100755 --- a/bin/lib/objects/abstract_daterange_object.py +++ b/bin/lib/objects/abstract_daterange_object.py @@ -7,6 +7,7 @@ Base Class for AIL Objects # Import External packages ################################## import os +import re import sys from abc import abstractmethod, ABC @@ -44,8 +45,14 @@ class AbstractDaterangeObject(AbstractObject, ABC): def exists(self): return r_object.exists(f'meta:{self.type}:{self.id}') + def _get_field(self, field): + return r_object.hget(f'meta:{self.type}:{self.id}', field) + + def _set_field(self, field, value): + return r_object.hset(f'meta:{self.type}:{self.id}', field, value) + def get_first_seen(self, r_int=False): - first_seen = r_object.hget(f'meta:{self.type}:{self.id}', 'first_seen') + first_seen = self._get_field('first_seen') if r_int: if first_seen: return int(first_seen) @@ -55,7 +62,7 @@ class AbstractDaterangeObject(AbstractObject, ABC): return first_seen def get_last_seen(self, r_int=False): - last_seen = r_object.hget(f'meta:{self.type}:{self.id}', 'last_seen') + last_seen = self._get_field('last_seen') if r_int: if last_seen: return int(last_seen) @@ -83,10 +90,10 @@ class AbstractDaterangeObject(AbstractObject, ABC): return meta_dict def set_first_seen(self, first_seen): - r_object.hset(f'meta:{self.type}:{self.id}', 'first_seen', first_seen) + self._set_field('first_seen', first_seen) def set_last_seen(self, last_seen): - r_object.hset(f'meta:{self.type}:{self.id}', 'last_seen', last_seen) + self._set_field('last_seen', last_seen) def update_daterange(self, date): date = int(date) @@ -139,11 +146,85 @@ class AbstractDaterangeObject(AbstractObject, ABC): self.add_correlation('domain', '', domain) # TODO:ADD objects + Stats - def _create(self, first_seen, last_seen): - self.set_first_seen(first_seen) - self.set_last_seen(last_seen) + def _create(self, first_seen=None, last_seen=None): + if first_seen: + self.set_first_seen(first_seen) + if last_seen: + self.set_last_seen(last_seen) r_object.sadd(f'{self.type}:all', self.id) # TODO def _delete(self): pass + + +class AbstractDaterangeObjects(ABC): + """ + Abstract Daterange Objects + """ + + def __init__(self, obj_type): + """ Abstract for Daterange Objects + + :param obj_type: object type (item, ...) + """ + self.type = obj_type + + def get_all(self): + return r_object.smembers(f'{self.type}:all') + + def get_by_date(self, date): + return r_object.zrange(f'{self.type}:date:{date}', 0, -1) + + def get_nb_by_date(self, date): + return r_object.zcard(f'{self.type}:date:{date}') + + def get_by_daterange(self, date_from, date_to): + obj_ids = set() + for date in Date.substract_date(date_from, date_to): + obj_ids = obj_ids | set(self.get_by_date(date)) + return obj_ids + + @abstractmethod + def get_metas(self, obj_ids, options=set()): + pass + + def _get_metas(self, obj_class_ref, obj_ids, options=set()): + dict_obj = {} + for obj_id in obj_ids: + obj = obj_class_ref(obj_id) + dict_obj[obj_id] = obj.get_meta(options=options) + return dict_obj + + @abstractmethod + def sanitize_name_to_search(self, name_to_search): + return name_to_search + + def search_by_name(self, name_to_search, r_pos=False): + objs = {} + # for subtype in subtypes: + r_name = self.sanitize_name_to_search(name_to_search) + if not name_to_search or isinstance(r_name, dict): + return objs + r_name = re.compile(r_name) + for title_name in self.get_all(): + res = re.search(r_name, title_name) + if res: + objs[title_name] = {} + if r_pos: + objs[title_name]['hl-start'] = res.start() + objs[title_name]['hl-end'] = res.end() + return objs + + def api_get_chart_nb_by_daterange(self, date_from, date_to): + date_type = [] + for date in Date.substract_date(date_from, date_to): + d = {'date': f'{date[0:4]}-{date[4:6]}-{date[6:8]}', + self.type: self.get_nb_by_date(date)} + date_type.append(d) + return date_type + + def api_get_meta_by_daterange(self, date_from, date_to): + date = Date.sanitise_date_range(date_from, date_to) + return self.get_metas(self.get_by_daterange(date['date_from'], date['date_to']), options={'sparkline'}) + diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index b65c9ace..631597c4 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -187,7 +187,7 @@ class AbstractObject(ABC): pass @staticmethod - def get_misp_object_first_last_seen(misp_obj): + def get_misp_object_first_last_seen(misp_obj): # TODO REMOVE ME ???? """ :type misp_obj: MISPObject """ diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 01445c9e..d52a0bbd 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -21,6 +21,7 @@ from lib.objects.Domains import Domain from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects from lib.objects import Pgps from lib.objects.Screenshots import Screenshot +from lib.objects import Titles from lib.objects import Usernames config_loader = ConfigLoader() @@ -59,6 +60,8 @@ def get_object(obj_type, subtype, id): return CryptoCurrencies.CryptoCurrency(id, subtype) elif obj_type == 'pgp': return Pgps.Pgp(id, subtype) + elif obj_type == 'title': + return Titles.Title(id) elif obj_type == 'username': return Usernames.Username(id, subtype) @@ -160,10 +163,12 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False): obj = get_object(obj_type, subtype, id) meta = obj.get_meta() meta['icon'] = obj.get_svg_icon() - if subtype or obj_type == 'cve': + if subtype or obj_type == 'cve' or obj_type == 'title': meta['sparkline'] = obj.get_sparkline() if obj_type == 'cve': meta['cve_search'] = obj.get_cve_search() + # if obj_type == 'title': + # meta['cve_search'] = obj.get_cve_search() if subtype == 'bitcoin' and related_btc: meta["related_btc"] = btc_ail.get_bitcoin_info(obj.id) if obj.get_type() == 'decoded': diff --git a/bin/modules/Phone.py b/bin/modules/Phone.py index a7537eda..16af8303 100755 --- a/bin/modules/Phone.py +++ b/bin/modules/Phone.py @@ -43,7 +43,7 @@ class Phone(AbstractModule): def extract(self, obj_id, content, tag): extracted = [] - phones = self.regex_phone_iter('US', obj_id, content) + phones = self.regex_phone_iter('ZZ', obj_id, content) for phone in phones: extracted.append([phone[0], phone[1], phone[2], f'tag:{tag}']) return extracted diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index c6669ef6..63addc22 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -49,6 +49,7 @@ from blueprints.settings_b import settings_b from blueprints.objects_cve import objects_cve from blueprints.objects_decoded import objects_decoded from blueprints.objects_subtypes import objects_subtypes +from blueprints.objects_title import objects_title Flask_dir = os.environ['AIL_FLASK'] @@ -102,6 +103,7 @@ app.register_blueprint(settings_b, url_prefix=baseUrl) app.register_blueprint(objects_cve, url_prefix=baseUrl) app.register_blueprint(objects_decoded, url_prefix=baseUrl) app.register_blueprint(objects_subtypes, url_prefix=baseUrl) +app.register_blueprint(objects_title, url_prefix=baseUrl) # ========= =========# # ========= Cookie name ======== diff --git a/var/www/blueprints/objects_title.py b/var/www/blueprints/objects_title.py new file mode 100644 index 00000000..eef7f69c --- /dev/null +++ b/var/www/blueprints/objects_title.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... +''' + +import os +import sys + +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file +from flask_login import login_required, current_user + +# Import Role_Manager +from Role_Manager import login_admin, login_analyst, login_read_only + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.objects import Titles +from packages import Date + +# ============ BLUEPRINT ============ +objects_title = Blueprint('objects_title', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/title')) + +# ============ VARIABLES ============ +bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] + + +# ============ FUNCTIONS ============ +@objects_title.route("/objects/title", methods=['GET']) +@login_required +@login_read_only +def objects_titles(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + show_objects = request.args.get('show_objects') + date = Date.sanitise_date_range(date_from, date_to) + date_from = date['date_from'] + date_to = date['date_to'] + + if show_objects: + dict_objects = Titles.Titles().api_get_meta_by_daterange(date_from, date_to) + else: + dict_objects = {} + + return render_template("TitleDaterange.html", date_from=date_from, date_to=date_to, + dict_objects=dict_objects, show_objects=show_objects) + +@objects_title.route("/objects/title/post", methods=['POST']) +@login_required +@login_read_only +def objects_titles_post(): + date_from = request.form.get('date_from') + date_to = request.form.get('date_to') + show_objects = request.form.get('show_objects') + return redirect(url_for('objects_title.objects_titles', date_from=date_from, date_to=date_to, show_objects=show_objects)) + +@objects_title.route("/objects/title/range/json", methods=['GET']) +@login_required +@login_read_only +def objects_title_range_json(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + date = Date.sanitise_date_range(date_from, date_to) + date_from = date['date_from'] + date_to = date['date_to'] + return jsonify(Titles.Titles().api_get_chart_nb_by_daterange(date_from, date_to)) + +@objects_title.route("/objects/title/search", methods=['POST']) +@login_required +@login_read_only +def objects_title_search(): + to_search = request.form.get('object_id') + + # TODO SANITIZE ID + # TODO Search all + title = Titles.Title(to_search) + if not title.exists(): + abort(404) + else: + return redirect(title.get_link(flask_context=True)) + +# ============= ROUTES ============== + diff --git a/var/www/templates/correlation/metadata_card_cve.html b/var/www/templates/correlation/metadata_card_cve.html index d1cfab9f..1e166ddc 100644 --- a/var/www/templates/correlation/metadata_card_cve.html +++ b/var/www/templates/correlation/metadata_card_cve.html @@ -108,7 +108,7 @@ Tags: {% for tag in dict_object["metadata"]['tags'] %} {% endfor %} diff --git a/var/www/templates/correlation/metadata_card_title.html b/var/www/templates/correlation/metadata_card_title.html new file mode 100644 index 00000000..cd943349 --- /dev/null +++ b/var/www/templates/correlation/metadata_card_title.html @@ -0,0 +1,173 @@ + + + +{% with modal_add_tags=dict_object['metadata_card']['add_tags_modal']%} + {% include 'modals/add_tags.html' %} +{% endwith %} + +{% include 'modals/edit_tag.html' %} + +
+
+

{{ dict_object["metadata"]["content"] }}

+
{{ dict_object["correlation_id"] }}
+
    +
  • +
    +
    + + + + + + + + + + + + + + + + + +
    Object typeFirst seenLast seenNb seen
    + + + + {{ dict_object["metadata_card"]["icon"]["icon"] }} + + + {{ dict_object["object_type"] }} + {{ dict_object["metadata"]['first_seen'] }}{{ dict_object["metadata"]['last_seen'] }}{{ dict_object["metadata"]['nb_seen'] }}
    +
    +
    +
    +
    +
    +
    • + +
  • +
    +
    + Tags: + {% for tag in dict_object["metadata"]['tags'] %} + + {% endfor %} + +
    +
    • +
+ + {% with obj_type='title', obj_id=dict_object['correlation_id'], obj_subtype='' %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + +
+
+ + + + + + diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index b8a9dd79..8342e8b6 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -113,6 +113,8 @@ {% include 'correlation/metadata_card_domain.html' %} {% elif dict_object["object_type"] == "screenshot" %} {% include 'correlation/metadata_card_screenshot.html' %} + {% elif dict_object["object_type"] == "title" %} + {% include 'correlation/metadata_card_title.html' %} {% elif dict_object["object_type"] == "item" %} {% include 'correlation/metadata_card_item.html' %} {% endif %} diff --git a/var/www/templates/crawler/crawler_splash/showDomain.html b/var/www/templates/crawler/crawler_splash/showDomain.html index 43f00d4b..b2e2f0b3 100644 --- a/var/www/templates/crawler/crawler_splash/showDomain.html +++ b/var/www/templates/crawler/crawler_splash/showDomain.html @@ -347,6 +347,46 @@ {% endif %} + {% if 'title' in dict_domain%} +
+
+
+
+
+
+ Titles   +
{{dict_domain['title']|length}}
+
+
+
+ +
+
+
+
+
+ + + + + + + + {% for title in dict_domain['title']%} + + + + {% endfor %} + +
Tilte
{{ title[1] }}
+
+
+
+
+ {% endif %} + {% if dict_domain["history"] %}
@@ -489,6 +529,9 @@ {% endif %} {% if 'cryptocurrency' in dict_domain%} $('#tablecurrency').DataTable({}); + {% endif %} + {% if 'title' in dict_domain%} + $('#tabletitle').DataTable({}); {% endif %} table = $('#myTable_1').DataTable( { diff --git a/var/www/templates/objects/title/TitleDaterange.html b/var/www/templates/objects/title/TitleDaterange.html new file mode 100644 index 00000000..17efda7c --- /dev/null +++ b/var/www/templates/objects/title/TitleDaterange.html @@ -0,0 +1,611 @@ + + + + + Titles - AIL + + + + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ +
+
+
+ +
+
+
Search Title by name:
+
+
+ + +
+
+
+
+
+ + +
+ +
+
+
Select a date range :
+
+
+
+ +
+
+
+ +
+
+ + +
+ +
+
+
+ +
+
+
+
+
+
+ + {% if dict_objects %} + {% if date_from|string == date_to|string %} +

{{ date_from }} Title:

+ {% else %} +

{{ date_from }} to {{ date_to }} Title:

+ {% endif %} + + + + + + + + + + + + {% for obj_id in dict_objects %} + + + + + + + + {% endfor %} + +
First SeenLast SeenTotalLast days
{{ dict_objects[obj_id]['content'] }}{{ dict_objects[obj_id]['first_seen'] }}{{ dict_objects[obj_id]['last_seen'] }}{{ dict_objects[obj_id]['nb_seen'] }}
+ + + {% else %} + {% if show_objects %} + {% if date_from|string == date_to|string %} +

{{ date_from }}, No Title

+ {% else %} +

{{ date_from }} to {{ date_to }}, No Title

+ {% endif %} + {% endif %} + {% endif %} +
+ +
+
+ + + + + + + + + + + + + + + + + diff --git a/var/www/templates/sidebars/sidebar_objects.html b/var/www/templates/sidebars/sidebar_objects.html index 46fe58eb..4f09e138 100644 --- a/var/www/templates/sidebars/sidebar_objects.html +++ b/var/www/templates/sidebars/sidebar_objects.html @@ -34,6 +34,12 @@ CVE +
  • + + + Title + +
  • @@ -61,7 +67,6 @@ -
    MISP From ee951ca9486e9ac9e6c08385d94c8cea1e0389c7 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 25 May 2023 16:00:27 +0200 Subject: [PATCH 002/238] chg: [correlation] add direct correlation stats --- bin/lib/objects/ail_objects.py | 4 ++++ var/www/blueprints/correlation.py | 3 ++- .../templates/correlation/show_correlation.html | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index d52a0bbd..6886d6fc 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -356,6 +356,10 @@ def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types add_obj_tags(obj1_type, subtype1, id1, tags) return objs +def get_obj_nb_correlations(obj_type, subtype, obj_id, filter_types=[]): + obj = get_object(obj_type, subtype, obj_id) + return obj.get_nb_correlations(filter_types=filter_types) + ################################################################################ ################################################################################ TODO ################################################################################ diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index 35263974..e0eed996 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -125,7 +125,8 @@ def show_correlation(): "max_nodes": max_nodes, "mode": mode, "filter": filter_types, "filter_str": ",".join(filter_types), "metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, - options={'tags'}, flask_context=True) + options={'tags'}, flask_context=True), + "nb_correl": ail_objects.get_obj_nb_correlations(obj_type, subtype, obj_id) } if subtype: dict_object["metadata"]['type_id'] = subtype diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 8342e8b6..1329e2a5 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -262,6 +262,21 @@

    • +
      +
    • Direct Correlations
      • +
    • + {% for obj_type in dict_object['nb_correl'] %} +
      +
      + {{ obj_type }} +
      +
      + {{ dict_object['nb_correl'][obj_type] }} +
      +
      + {% endfor %} +
      • +
    From 9a4feb93a01492584eac8c2a2f14a5041d087f43 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 25 May 2023 16:11:55 +0200 Subject: [PATCH 003/238] fix: [correlation btc info] catch btc txs error --- bin/lib/btc_ail.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/bin/lib/btc_ail.py b/bin/lib/btc_ail.py index e17e257c..34ddd9d5 100755 --- a/bin/lib/btc_ail.py +++ b/bin/lib/btc_ail.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import json +import logging import os import sys import requests @@ -8,6 +10,8 @@ import requests sys.path.append(os.environ['AIL_BIN']) from lib.objects.CryptoCurrencies import CryptoCurrency +logger = logging.getLogger() + blockchain_all = 'https://blockchain.info/rawaddr' # pre-alpha script @@ -18,23 +22,26 @@ def get_bitcoin_info(bitcoin_address, nb_transaction=50): set_btc_in = set() set_btc_out = set() try: - req = requests.get('{}/{}?limit={}'.format(blockchain_all, bitcoin_address, nb_transaction)) + req = requests.get(f'{blockchain_all}/{bitcoin_address}?limit={nb_transaction}') jreq = req.json() except Exception as e: - print(e) + logger.warning(e) + return dict_btc + + if not jreq.get('n_tx'): + logger.critical(json.dumps(jreq)) return dict_btc - # print(json.dumps(jreq)) dict_btc['n_tx'] = jreq['n_tx'] dict_btc['total_received'] = float(jreq['total_received'] / 100000000) dict_btc['total_sent'] = float(jreq['total_sent'] / 100000000) dict_btc['final_balance'] = float(jreq['final_balance'] / 100000000) for transaction in jreq['txs']: - for input in transaction['inputs']: - if 'addr' in input['prev_out']: - if input['prev_out']['addr'] != bitcoin_address: - set_btc_in.add(input['prev_out']['addr']) + for t_input in transaction['inputs']: + if 'addr' in t_input['prev_out']: + if t_input['prev_out']['addr'] != bitcoin_address: + set_btc_in.add(t_input['prev_out']['addr']) for output in transaction['out']: if 'addr' in output: if output['addr'] != bitcoin_address: From 405d097024a0f4e7bc90dbb7b76a2fe5baa8dbb9 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 25 May 2023 16:26:48 +0200 Subject: [PATCH 004/238] fix: [crawler] fix undefined capture status --- bin/lib/crawlers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index d8f4c890..7dbb18b2 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -1091,7 +1091,10 @@ class CrawlerCapture: return self.get_task().get_start_time() def get_status(self): - return r_cache.hget(f'crawler:capture:{self.uuid}', 'status') + status = r_cache.hget(f'crawler:capture:{self.uuid}', 'status') + if not status: + status = -1 + return status def is_ongoing(self): return self.get_status() == CaptureStatus.ONGOING @@ -1109,7 +1112,7 @@ class CrawlerCapture: def update(self, status): # Error or Reload if not status: - r_cache.hset(f'crawler:capture:{self.uuid}', 'status', CaptureStatus.UNKNOWN) + r_cache.hset(f'crawler:capture:{self.uuid}', 'status', CaptureStatus.UNKNOWN.value) r_cache.zadd('crawler:captures', {self.uuid: 0}) else: last_check = int(time.time()) From b4f1a432082059805fc44ccc11afee561c33a1c7 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 26 May 2023 10:47:58 +0200 Subject: [PATCH 005/238] chg: [correlation] correlation graph: filter title objects --- bin/crawlers/Crawler.py | 3 ++- var/www/blueprints/correlation.py | 3 +++ var/www/templates/correlation/show_correlation.html | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index 4b499eb9..e8822561 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -61,7 +61,8 @@ class Crawler(AbstractModule): self.domain = None # TODO Replace with warning list ??? - self.placeholder_screenshots = {'27e14ace10b0f96acd2bd919aaa98a964597532c35b6409dff6cc8eec8214748'} + self.placeholder_screenshots = {'27e14ace10b0f96acd2bd919aaa98a964597532c35b6409dff6cc8eec8214748', + '3e66bf4cc250a68c10f8a30643d73e50e68bf1d4a38d4adc5bfc4659ca2974c0'} # 404 # Send module state to logs self.logger.info('Crawler initialized') diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index e0eed996..c3c9013a 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -95,6 +95,9 @@ def show_correlation(): correl_option = request.form.get('ItemCheck') if correl_option: filter_types.append('item') + correl_option = request.form.get('TitleCheck') + if correl_option: + filter_types.append('title') # list as params filter_types = ",".join(filter_types) diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 1329e2a5..94a8efad 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -206,6 +206,10 @@ +
    + + +
    From 5d4b718174d1249875ddb530ded23cbebb197879 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 26 May 2023 11:22:12 +0200 Subject: [PATCH 006/238] chg: [correlation graph] select correlation depth --- var/www/blueprints/correlation.py | 23 ++++++++++++++---- .../correlation/show_correlation.html | 24 +++++++++++++------ 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index c3c9013a..b6b9776d 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -52,6 +52,15 @@ def sanitise_nb_max_nodes(nb_max_nodes): nb_max_nodes = 300 return nb_max_nodes +def sanitise_level(level): + try: + level = int(level) + if level < 0: + level = 2 + except (TypeError, ValueError): + level = 2 + return level + # ============= ROUTES ============== @correlation.route('/correlation/show', methods=['GET', 'POST']) @login_required @@ -67,6 +76,7 @@ def show_correlation(): mode = 'inter' else: mode = 'union' + level = sanitise_level(request.form.get('level')) ## get all selected correlations filter_types = [] @@ -104,7 +114,7 @@ def show_correlation(): # redirect to keep history and bookmark return redirect(url_for('correlation.show_correlation', type=object_type, subtype=subtype, id=obj_id, mode=mode, - max_nodes=max_nodes, filter=filter_types)) + max_nodes=max_nodes, level=level, filter=filter_types)) # request.method == 'GET' else: @@ -113,6 +123,7 @@ def show_correlation(): obj_id = request.args.get('id') max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) mode = sanitise_graph_mode(request.args.get('mode')) + level = sanitise_level(request.args.get('level')) related_btc = bool(request.args.get('related_btc', False)) @@ -125,7 +136,7 @@ def show_correlation(): else: dict_object = {"object_type": obj_type, "correlation_id": obj_id, - "max_nodes": max_nodes, "mode": mode, + "max_nodes": max_nodes, "mode": mode, "level": level, "filter": filter_types, "filter_str": ",".join(filter_types), "metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags'}, flask_context=True), @@ -175,10 +186,11 @@ def graph_node_json(): subtype = request.args.get('subtype') obj_type = request.args.get('type') max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) + level = sanitise_level(request.args.get('level')) filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(',')) - json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=2, flask_context=True) + json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True) #json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes) return jsonify(json_graph) @@ -204,6 +216,7 @@ def correlation_tags_add(): subtype = request.form.get('tag_subtype', '') obj_type = request.form.get('tag_obj_type') nb_max = sanitise_nb_max_nodes(request.form.get('tag_nb_max')) + level = sanitise_level(request.form.get('tag_level')) filter_types = ail_objects.sanitize_objs_types(request.form.get('tag_filter', '').split(',')) if not ail_objects.exists_obj(obj_type, subtype, obj_id): @@ -232,8 +245,10 @@ def correlation_tags_add(): tags = [] if tags: - ail_objects.obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=filter_types, lvl=2, nb_max=nb_max) + ail_objects.obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=filter_types, + lvl=level + 1, nb_max=nb_max) return redirect(url_for('correlation.show_correlation', type=obj_type, subtype=subtype, id=obj_id, + level=level, filter=",".join(filter_types))) diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 94a8efad..d683eaf9 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -228,14 +228,23 @@
    +{#
  • #} +{#
    #} +{# Union  #} +{#
    #} +{# #} +{# #} +{#
    #} +{#
    #} +{#
    • #}
  • -
    - Union   -
    - - -
    + +
    + +
    + +
  • @@ -316,6 +325,7 @@ + {% include 'tags/block_tags_selector.html' %} @@ -336,7 +346,7 @@ var all_graph = {}; $(document).ready(function(){ $("#page-Decoded").addClass("active"); - all_graph.node_graph = create_graph("{{ url_for('correlation.graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&mode={{ dict_object["mode"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}"); + all_graph.node_graph = create_graph("{{ url_for('correlation.graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&mode={{ dict_object["mode"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}"); {% if dict_object["object_type"] in ["cryptocurrency", "pgp", "username"] %} all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('objects_subtypes.objects_cve_graphline_json') }}?type={{ dict_object["object_type"] }}&subtype={{dict_object["metadata"]["type_id"]}}&id={{dict_object["correlation_id"]}}"); {% elif dict_object["object_type"] == "decoded" %} From b3cafd2a1d1e20d8225436b26889f7e74cdb8cf3 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 26 May 2023 11:44:29 +0200 Subject: [PATCH 007/238] chg: [correlation graph] update node legend --- bin/lib/objects/Titles.py | 3 +-- .../correlation/legend_graph_correlation.html | 24 +++++++++++++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/bin/lib/objects/Titles.py b/bin/lib/objects/Titles.py index 3c682224..5a8186f0 100755 --- a/bin/lib/objects/Titles.py +++ b/bin/lib/objects/Titles.py @@ -51,9 +51,8 @@ class Title(AbstractDaterangeObject): url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}' return url - # TODO # CHANGE COLOR def get_svg_icon(self): - return {'style': 'fas', 'icon': '\uf1dc', 'color': '#1E88E5', 'radius': 5} + return {'style': 'fas', 'icon': '\uf1dc', 'color': '#3C7CFF', 'radius': 5} def get_misp_object(self): obj_attrs = [] diff --git a/var/www/templates/correlation/legend_graph_correlation.html b/var/www/templates/correlation/legend_graph_correlation.html index f749dc69..ce5dc7ed 100644 --- a/var/www/templates/correlation/legend_graph_correlation.html +++ b/var/www/templates/correlation/legend_graph_correlation.html @@ -8,7 +8,7 @@ Decoded: - Screenshot: + Objects: Pgp: @@ -20,7 +20,7 @@ Domain: - Paste: + Item: @@ -111,6 +111,16 @@ +
    + + + + + + + cve +
    @@ -121,6 +131,16 @@ screenshot
    +
    + + + + + + + title +
    From 1e7b527e41d273325a8b28615d4305291d46b4e9 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 26 May 2023 13:57:34 +0200 Subject: [PATCH 008/238] fix: [tracker] fix webhook --- bin/lib/Tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 15f5dc17..1050d56b 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -241,7 +241,7 @@ class Tracker: return self._get_field('user_id') def webhook_export(self): - return r_tracker.hexists(f'tracker:mail:{self.uuid}', 'webhook') + return r_tracker.hexists(f'tracker:{self.uuid}', 'webhook') def get_webhook(self): return r_tracker.hget(f'tracker:{self.uuid}', 'webhook') From 8252d6b69ee1079a4a744f623c22dfddc1a8aaa1 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 26 May 2023 14:09:12 +0200 Subject: [PATCH 009/238] fix: [tracker] fix tracker delete --- bin/lib/Tracker.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 1050d56b..2a5336ad 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -564,8 +564,20 @@ class Tracker: self._del_mails() self._del_tags() + + level = self.get_level() + + if level == 0: # user only + user = self.get_user() + r_tracker.srem(f'user:tracker:{user}', self.uuid) + r_tracker.srem(f'user:tracker:{user}:{tracker_type}', self.uuid) + elif level == 1: # global + r_tracker.srem('global:tracker', self.uuid) + r_tracker.srem(f'global:tracker:{tracker_type}', self.uuid) + # meta r_tracker.delete(f'tracker:{self.uuid}') + trigger_trackers_refresh(tracker_type) def create_tracker(tracker_type, to_track, user_id, level, description=None, filters={}, tags=[], mails=[], webhook=None, tracker_uuid=None): From 2ebe4845a70639e781b750742ab4ee62970531b1 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 30 May 2023 10:11:12 +0200 Subject: [PATCH 010/238] fix: [module extractor] fix tracker extractor --- bin/lib/module_extractor.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/bin/lib/module_extractor.py b/bin/lib/module_extractor.py index 02f499fd..d4ea6c78 100755 --- a/bin/lib/module_extractor.py +++ b/bin/lib/module_extractor.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* import json +import logging import os import sys import yara +from hashlib import sha256 from operator import itemgetter sys.path.append(os.environ['AIL_BIN']) @@ -28,6 +30,8 @@ from modules.Onion import Onion from modules.Phone import Phone from modules.Tools import Tools +logger = logging.getLogger() + config_loader = ConfigLoader() r_cache = config_loader.get_redis_conn("Redis_Cache") config_loader = None @@ -64,11 +68,12 @@ def get_correl_match(extract_type, obj_id, content): if extract_type == 'title': title = Title(value).get_content() to_extract.append(title) - map_value_id[title] = value + sha256_val = sha256(title.encode()).hexdigest() else: map_subtype[value] = subtype to_extract.append(value) - map_value_id[value] = value + sha256_val = sha256(value.encode()).hexdigest() + map_value_id[sha256_val] = value if to_extract: objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj_id, content) for obj in objs: @@ -76,7 +81,12 @@ def get_correl_match(extract_type, obj_id, content): subtype = map_subtype[obj[2]] else: subtype = '' - extracted.append([obj[0], obj[1], obj[2], f'{extract_type}:{subtype}:{map_value_id[obj[2]]}']) + sha256_val = sha256(obj[2].encode()).hexdigest() + value_id = map_value_id.get(sha256_val) + if not value_id: + logger.critical(f'Error module extractor: {sha256_val}\n{extract_type}\n{subtype}\n{value_id}\n{map_value_id}\n{objs}') + value_id = 'ERROR' + extracted.append([obj[0], obj[1], obj[2], f'{extract_type}:{subtype}:{value_id}']) return extracted def _get_yara_match(data): @@ -162,6 +172,7 @@ def extract(obj_id, content=None): # CHECK CACHE cached = r_cache.get(f'extractor:cache:{obj_id}') + # cached = None if cached: r_cache.expire(f'extractor:cache:{obj_id}', 300) return json.loads(cached) From 50abff66b496afa4c295a98b9d2eed862d5b5465 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 30 May 2023 14:48:06 +0200 Subject: [PATCH 011/238] chg: [HOWTO] improve HOWTO --- HOWTO.md | 80 +++++++++---------- .../{Module_Template.py => TemplateModule.py} | 9 +-- configs/modules.cfg | 5 ++ 3 files changed, 46 insertions(+), 48 deletions(-) rename bin/modules/{Module_Template.py => TemplateModule.py} (89%) diff --git a/HOWTO.md b/HOWTO.md index 1432d475..b7ebd1bf 100644 --- a/HOWTO.md +++ b/HOWTO.md @@ -1,17 +1,15 @@ -Feeding, adding new features and contributing -============================================= +# Feeding, adding new features and contributing -How to feed the AIL framework ------------------------------ +## How to feed the AIL framework -For the moment, there are three different ways to feed AIL with data: +Currently, there are three different ways to feed data into AIL: 1. Be a collaborator of CIRCL and ask to access our feed. It will be sent to the static IP you are using for AIL. 2. You can setup [pystemon](https://github.com/cvandeplas/pystemon) and use the custom feeder provided by AIL (see below). -3. You can feed your own data using the [./bin/file_dir_importer.py](./bin/import_dir.py) script. +3. You can feed your own data using the [./tool/file_dir_importer.py](./tool/file_dir_importer.py) script. ### Feeding AIL with pystemon @@ -21,10 +19,12 @@ However, if you want to collect some pastes and feed them to AIL, the procedure Feed data to AIL: 1. Clone the [pystemon's git repository](https://github.com/cvandeplas/pystemon): -``` git clone https://github.com/cvandeplas/pystemon.git ``` + ``` + git clone https://github.com/cvandeplas/pystemon.git + ``` 2. Edit configuration file for pystemon ```pystemon/pystemon.yaml```: - * Configuration of storage section (adapt to your needs): + - Configure the storage section according to your needs: ``` storage: archive: @@ -44,68 +44,61 @@ Feed data to AIL: database: 10 lookup: no ``` - * Change configuration for paste-sites according to your needs (don't forget to throttle download time and/or update time). + - Adjust the configuration for paste-sites based on your requirements (remember to throttle download and update times). + 3. Install python dependencies inside the virtual environment: - ``` + ```shell cd ail-framework/ . ./AILENV/bin/activate - cd pystemon/ #cd to pystemon folder - pip3 install -U -r requirements.txt + cd pystemon/ + pip install -U -r requirements.txt ``` -4. Edit configuration file ```ail-framework/configs/core.cfg```: - * Modify the "pystemonpath" path accordingly +4. Edit the configuration file ```ail-framework/configs/core.cfg```: + - Modify the "pystemonpath" path accordingly. -5. Launch ail-framework, pystemon and pystemon-feeder.py (still inside virtual environment): - * Option 1 (recommended): - ``` +5. Launch ail-framework, pystemon and PystemonImporter.py (all within the virtual environment): + - Option 1 (recommended): + ``` ./ail-framework/bin/LAUNCH.py -l #starts ail-framework - ./ail-framework/bin/LAUNCH.py -f #starts pystemon and the pystemon-feeder.py + ./ail-framework/bin/LAUNCH.py -f #starts pystemon and the PystemonImporter.py ``` - * Option 2 (you may need two terminal windows): - ``` - ./ail-framework/bin/LAUNCH.py -l #starts ail-framework - ./pystemon/pystemon.py - ./ail-framework/bin/feeder/pystemon-feeder.py - ``` + - Option 2 (may require two terminal windows): + ``` + ./ail-framework/bin/LAUNCH.py -l #starts ail-framework + ./pystemon/pystemon.py + ./ail-framework/bin/importer/PystemonImporter.py + ``` -How to create a new module --------------------------- +## How to create a new module -If you want to add a new processing or analysis module in AIL, follow these simple steps: +To add a new processing or analysis module to AIL, follow these steps: -1. Add your module name in [./bin/packages/modules.cfg](./bin/packages/modules.cfg) and subscribe to at least one module at minimum (Usually, Redis_Global). +1. Add your module name in [./configs/modules.cfg](./configs/modules.cfg) and subscribe to at least one module at minimum (Usually, `Item`). -2. Use [./bin/template.py](./bin/template.py) as a sample module and create a new file in bin/ with the module name used in the modules.cfg configuration. +2. Use [./bin/modules/modules/TemplateModule.py](./bin/modules/modules/TemplateModule.py) as a sample module and create a new file in bin/modules with the module name used in the `modules.cfg` configuration. -How to contribute a module --------------------------- +## How to contribute a module Feel free to fork the code, play with it, make some patches or add additional analysis modules. To contribute your module, feel free to pull your contribution. -Additional information -====================== +## Additional information -Crawler ---------------------- +### Crawler In AIL, you can crawl websites and Tor hidden services. Don't forget to review the proxy configuration of your Tor client and especially if you enabled the SOCKS5 proxy -[//]: # (and binding on the appropriate IP address reachable via the dockers where Splash runs.) - ### Installation - [Install Lacus](https://github.com/ail-project/lacus) ### Configuration 1. Lacus URL: -In the webinterface, go to ``Crawlers>Settings`` and click on the Edit button - +In the web interface, go to `Crawlers` > `Settings` and click on the Edit button ![Splash Manager Config](./doc/screenshots/lacus_config.png?raw=true "AIL Lacus Config") @@ -115,10 +108,11 @@ In the webinterface, go to ``Crawlers>Settings`` and click on the Edit button Choose the number of crawlers you want to launch ![Splash Manager Nb Crawlers Config](./doc/screenshots/crawler_nb_captures.png?raw=true "AIL Lacus Nb Crawlers Config") + ![Splash Manager Nb Crawlers Config](./doc/screenshots/crawler_nb_captures_edit.png?raw=true "AIL Lacus Nb Crawlers Config") -Kvrocks Migration +### Kvrocks Migration --------------------- **Important Note: We are currently working on a [migration script](https://github.com/ail-project/ail-framework/blob/master/bin/DB_KVROCKS_MIGRATION.py) to facilitate the migration to Kvrocks. @@ -130,12 +124,12 @@ Please note that the current version of this migration script only supports migr To migrate your database to Kvrocks: 1. Launch ARDB and Kvrocks 2. Pull from remote - ``` + ```shell git checkout master git pull ``` 3. Launch the migration script: - ``` + ```shell git checkout master git pull cd bin/ diff --git a/bin/modules/Module_Template.py b/bin/modules/TemplateModule.py similarity index 89% rename from bin/modules/Module_Template.py rename to bin/modules/TemplateModule.py index 5cb2105b..cdd6646f 100755 --- a/bin/modules/Module_Template.py +++ b/bin/modules/TemplateModule.py @@ -30,15 +30,15 @@ class Template(AbstractModule): def __init__(self): super(Template, self).__init__() - # Pending time between two computation (computeNone) in seconds - self.pending_seconds = 10 + # Pending time between two computation (computeNone) in seconds, 10 by default + # self.pending_seconds = 10 - # Send module state to logs + # logs self.logger.info(f'Module {self.module_name} initialized') # def computeNone(self): # """ - # Do something when there is no message in the queue + # Do something when there is no message in the queue. Optional # """ # self.logger.debug("No message in queue") @@ -53,6 +53,5 @@ class Template(AbstractModule): if __name__ == '__main__': - module = Template() module.run() diff --git a/configs/modules.cfg b/configs/modules.cfg index 26aa8580..6f6fa55d 100644 --- a/configs/modules.cfg +++ b/configs/modules.cfg @@ -168,4 +168,9 @@ subscribe = Url # [My_Module_Name] # subscribe = Global # Queue name # publish = Tags # Queue name +# +# [TemplateModule.] +# subscribe = Global # Queue name +# publish = Tags # Queue name + From e3e5e9aff248428941e6e12c3bdb8aa83184fef5 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 30 May 2023 14:49:17 +0200 Subject: [PATCH 012/238] fix: [module.cfg] fix templateModule example --- configs/modules.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/modules.cfg b/configs/modules.cfg index 6f6fa55d..b0b1f6df 100644 --- a/configs/modules.cfg +++ b/configs/modules.cfg @@ -169,7 +169,7 @@ subscribe = Url # subscribe = Global # Queue name # publish = Tags # Queue name # -# [TemplateModule.] +# [TemplateModule] # subscribe = Global # Queue name # publish = Tags # Queue name From 22a2c9afdbf31759caf398568cd57a901c5c7daa Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 1 Jun 2023 14:19:05 +0200 Subject: [PATCH 013/238] chg: [README.md] update --- README.md | 199 ++++++++---------- doc/SourceCode.info | 8 - doc/logo/logo-small.png | Bin 23954 -> 0 bytes doc/logo/logo.png | Bin 64183 -> 0 bytes doc/screenshots/DashboardAIL.png | Bin 281189 -> 0 bytes doc/screenshots/WordtrendingAIL.png | Bin 154365 -> 0 bytes doc/screenshots/ail-hashedfiles.png | Bin 141277 -> 0 bytes doc/screenshots/browse-important.png | Bin 148046 -> 0 bytes doc/screenshots/correlation_decoded_image.png | Bin 0 -> 249596 bytes doc/screenshots/dashboard.png | Bin 329515 -> 0 bytes doc/screenshots/dashboard0.png | Bin 0 -> 361044 bytes doc/screenshots/decodeds_dashboard.png | Bin 0 -> 129696 bytes doc/screenshots/domain_circl.png | Bin 0 -> 241493 bytes doc/screenshots/investigation_mixer.png | Bin 0 -> 93813 bytes doc/screenshots/misp_export.png | Bin 0 -> 78807 bytes doc/screenshots/module_information.png | Bin 172327 -> 0 bytes doc/screenshots/paste_submit.png | Bin 28280 -> 0 bytes doc/screenshots/paste_tags_edit.png | Bin 22277 -> 0 bytes doc/screenshots/retro_hunt.png | Bin 0 -> 162454 bytes doc/screenshots/tag_auto_export.png | Bin 83424 -> 0 bytes doc/screenshots/tags2.png | Bin 128424 -> 0 bytes doc/screenshots/tags_misp_auto.png | Bin 0 -> 121618 bytes doc/screenshots/tags_search_items.png | Bin 0 -> 198226 bytes doc/screenshots/terms-manager.png | Bin 64411 -> 0 bytes doc/screenshots/terms-plot.png | Bin 30946 -> 0 bytes doc/screenshots/terms-top.png | Bin 114022 -> 0 bytes doc/screenshots/tracker_create.png | Bin 0 -> 62103 bytes doc/screenshots/tracker_yara.png | Bin 0 -> 127749 bytes doc/screenshots/ui_submit.png | Bin 0 -> 68927 bytes doc/statistics/create_graph_by_tld.py | 184 ---------------- 30 files changed, 87 insertions(+), 304 deletions(-) delete mode 100644 doc/SourceCode.info delete mode 100644 doc/logo/logo-small.png delete mode 100644 doc/logo/logo.png delete mode 100644 doc/screenshots/DashboardAIL.png delete mode 100644 doc/screenshots/WordtrendingAIL.png delete mode 100644 doc/screenshots/ail-hashedfiles.png delete mode 100644 doc/screenshots/browse-important.png create mode 100644 doc/screenshots/correlation_decoded_image.png delete mode 100644 doc/screenshots/dashboard.png create mode 100644 doc/screenshots/dashboard0.png create mode 100644 doc/screenshots/decodeds_dashboard.png create mode 100644 doc/screenshots/domain_circl.png create mode 100644 doc/screenshots/investigation_mixer.png create mode 100644 doc/screenshots/misp_export.png delete mode 100644 doc/screenshots/module_information.png delete mode 100644 doc/screenshots/paste_submit.png delete mode 100644 doc/screenshots/paste_tags_edit.png create mode 100644 doc/screenshots/retro_hunt.png delete mode 100644 doc/screenshots/tag_auto_export.png delete mode 100644 doc/screenshots/tags2.png create mode 100644 doc/screenshots/tags_misp_auto.png create mode 100644 doc/screenshots/tags_search_items.png delete mode 100644 doc/screenshots/terms-manager.png delete mode 100644 doc/screenshots/terms-plot.png delete mode 100644 doc/screenshots/terms-top.png create mode 100644 doc/screenshots/tracker_create.png create mode 100644 doc/screenshots/tracker_yara.png create mode 100644 doc/screenshots/ui_submit.png delete mode 100755 doc/statistics/create_graph_by_tld.py diff --git a/README.md b/README.md index ddc7f472..b4c69ca2 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -AIL -=== +# AIL framework

    @@ -34,53 +33,49 @@ AIL framework - Framework for Analysis of Information Leaks AIL is a modular framework to analyse potential information leaks from unstructured data sources like pastes from Pastebin or similar services or unstructured data streams. AIL framework is flexible and can be extended to support other functionalities to mine or process sensitive information (e.g. data leak prevention). -![Dashboard](./doc/screenshots/dashboard.png?raw=true "AIL framework dashboard") +![Dashboard](./doc/screenshots/dashboard0.png?raw=true "AIL framework dashboard") -![Finding webshells with AIL](./doc/screenshots/webshells.gif?raw=true "Finding websheels with AIL") +![Finding webshells with AIL](./doc/screenshots/webshells.gif?raw=true "Finding webshells with AIL") -Features --------- +## Features -* Modular architecture to handle streams of unstructured or structured information -* Default support for external ZMQ feeds, such as provided by CIRCL or other providers -* Multiple feed support -* Each module can process and reprocess the information already processed by AIL -* Detecting and extracting URLs including their geographical location (e.g. IP address location) -* Extracting and validating potential leaks of credit card numbers, credentials, ... -* Extracting and validating leaked email addresses, including DNS MX validation -* Module for extracting Tor .onion addresses (to be further processed for analysis) -* Keep tracks of duplicates (and diffing between each duplicate found) -* Extracting and validating potential hostnames (e.g. to feed Passive DNS systems) -* A full-text indexer module to index unstructured information -* Statistics on modules and web -* Real-time modules manager in terminal -* Global sentiment analysis for each providers based on nltk vader module -* Terms, Set of terms and Regex tracking and occurrence -* Many more modules for extracting phone numbers, credentials and others -* Alerting to [MISP](https://github.com/MISP/MISP) to share found leaks within a threat intelligence platform using [MISP standard](https://www.misp-project.org/objects.html#_ail_leak) -* Detect and decode encoded file (Base64, hex encoded or your own decoding scheme) and store files -* Detect Amazon AWS and Google API keys -* Detect Bitcoin address and Bitcoin private keys -* Detect private keys, certificate, keys (including SSH, OpenVPN) -* Detect IBAN bank accounts -* Tagging system with [MISP Galaxy](https://github.com/MISP/misp-galaxy) and [MISP Taxonomies](https://github.com/MISP/misp-taxonomies) tags -* UI paste submission -* Create events on [MISP](https://github.com/MISP/MISP) and cases on [The Hive](https://github.com/TheHive-Project/TheHive) -* Automatic paste export at detection on [MISP](https://github.com/MISP/MISP) (events) and [The Hive](https://github.com/TheHive-Project/TheHive) (alerts) on selected tags -* Extracted and decoded files can be searched by date range, type of file (mime-type) and encoding discovered -* Graph relationships between decoded file (hashes), similar PGP UIDs and addresses of cryptocurrencies -* Tor hidden services crawler to crawl and parse output -* Tor onion availability is monitored to detect up and down of hidden services -* Browser hidden services are screenshot and integrated in the analysed output including a blurring screenshot interface (to avoid "burning the eyes" of the security analysis with specific content) -* Tor hidden services is part of the standard framework, all the AIL modules are available to the crawled hidden services -* Generic web crawler to trigger crawling on demand or at regular interval URL or Tor hidden services +- Modular architecture to handle streams of unstructured or structured information +- Default support for external ZMQ feeds, such as provided by CIRCL or other providers +- Multiple Importers and feeds support +- Each module can process and reprocess the information already analyzed by AIL +- Detecting and extracting URLs including their geographical location (e.g. IP address location) +- Extracting and validating potential leaks of credit card numbers, credentials, ... +- Extracting and validating leaked email addresses, including DNS MX validation +- Module for extracting Tor .onion addresses for further analysis +- Keep tracks of credentials duplicates (and diffing between each duplicate found) +- Extracting and validating potential hostnames (e.g. to feed Passive DNS systems) +- A full-text indexer module to index unstructured information +- Terms, Set of terms, Regex, typo squatting and YARA tracking and occurrence +- YARA Retro Hunt +- Many more modules for extracting phone numbers, credentials, and more +- Alerting to [MISP](https://github.com/MISP/MISP) to share found leaks within a threat intelligence platform using [MISP standard](https://www.misp-project.org/objects.html#_ail_leak) +- Detecting and decoding encoded file (Base64, hex encoded or your own decoding scheme) and storing files +- Detecting Amazon AWS and Google API keys +- Detecting Bitcoin address and Bitcoin private keys +- Detecting private keys, certificate, keys (including SSH, OpenVPN) +- Detecting IBAN bank accounts +- Tagging system with [MISP Galaxy](https://github.com/MISP/misp-galaxy) and [MISP Taxonomies](https://github.com/MISP/misp-taxonomies) tags +- UI submission +- Create events on [MISP](https://github.com/MISP/MISP) and cases on [The Hive](https://github.com/TheHive-Project/TheHive) +- Automatic export on detection with [MISP](https://github.com/MISP/MISP) (events) and [The Hive](https://github.com/TheHive-Project/TheHive) (alerts) on selected tags +- Extracted and decoded files can be searched by date range, type of file (mime-type) and encoding discovered +- Correlations engine and Graph to visualize relationships between decoded files (hashes), PGP UIDs, domains, username, and cryptocurrencies addresses +- Websites, Forums and Tor Hidden-Services hidden services crawler to crawl and parse output +- Domain availability monitoring to detect up and down of websites and hidden services +- Browsed hidden services are automatically captured and integrated into the analyzed output, including a blurring screenshot interface (to avoid "burning the eyes" of security analysts with sensitive content) +- Tor hidden services is part of the standard framework, all the AIL modules are available to the crawled hidden services +- Crawler scheduler to trigger crawling on demand or at regular intervals for URLs or Tor hidden services -Installation ------------- +## Installation -Type these command lines for a fully automated installation and start AIL framework: +To install the AIL framework, run the following commands: ```bash # Clone the repo first git clone https://github.com/ail-project/ail-framework.git @@ -89,10 +84,6 @@ cd ail-framework # For Debian and Ubuntu based distributions ./installing_deps.sh -# For Centos based distributions (Tested: Centos 8) -chmod u+x centos_installing_deps.sh -./centos_installing_deps.sh - # Launch ail cd ~/ail-framework/ cd bin/ @@ -101,59 +92,52 @@ cd bin/ The default [installing_deps.sh](./installing_deps.sh) is for Debian and Ubuntu based distributions. -There is also a [Travis file](.travis.yml) used for automating the installation that can be used to build and install AIL on other systems. - Requirement: -- Python 3.6+ +- Python 3.7+ -Installation Notes ------------- +## Installation Notes -In order to use AIL combined with **ZFS** or **unprivileged LXC** it's necessary to disable Direct I/O in `$AIL_HOME/configs/6382.conf` by changing the value of the directive `use_direct_io_for_flush_and_compaction` to `false`. +For Lacus Crawler installation instructions, refer to the [HOWTO](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md#crawler) -Tor installation instructions can be found in the [HOWTO](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md#installationconfiguration) +## Starting AIL -Starting AIL --------------------------- +To start AIL, use the following commands: ```bash cd bin/ ./LAUNCH.sh -l ``` -Eventually you can browse the status of the AIL framework website at the following URL: +You can access the AIL framework web interface at the following URL: ``` https://localhost:7000/ ``` -The default credentials for the web interface are located in ``DEFAULT_PASSWORD``. This file is removed when you change your password. +The default credentials for the web interface are located in the ``DEFAULT_PASSWORD``file, which is deleted when you change your password. -Training --------- +## Training -CIRCL organises training on how to use or extend the AIL framework. AIL training materials are available at [https://www.circl.lu/services/ail-training-materials/](https://www.circl.lu/services/ail-training-materials/). +CIRCL organises training on how to use or extend the AIL framework. AIL training materials are available at [https://github.com/ail-project/ail-training](https://github.com/ail-project/ail-training). -API ------ +## API The API documentation is available in [doc/README.md](doc/README.md) -HOWTO ------ +## HOWTO HOWTO are available in [HOWTO.md](HOWTO.md) -Privacy and GDPR ----------------- +## Privacy and GDPR -[AIL information leaks analysis and the GDPR in the context of collection, analysis and sharing information leaks](https://www.circl.lu/assets/files/information-leaks-analysis-and-gdpr.pdf) document provides an overview how to use AIL in a lawfulness context especially in the scope of General Data Protection Regulation. +For information on AIL's compliance with GDPR and privacy considerations, refer to the [AIL information leaks analysis and the GDPR in the context of collection, analysis and sharing information leaks](https://www.circl.lu/assets/files/information-leaks-analysis-and-gdpr.pdf) document. -Research using AIL ------------------- +this document provides an overview how to use AIL in a lawfulness context especially in the scope of General Data Protection Regulation. -If you write academic paper, relying or using AIL, it can be cited with the following BibTeX: +## Research using AIL + +If you use or reference AIL in an academic paper, you can cite it using the following BibTeX: ~~~~ @inproceedings{mokaddem2018ail, @@ -166,75 +150,66 @@ If you write academic paper, relying or using AIL, it can be cited with the foll } ~~~~ -Screenshots -=========== +## Screenshots -Tor hidden service crawler --------------------------- +### Websites, Forums and Tor Hidden-Services -![Tor hidden service](./doc/screenshots/ail-bitcoinmixer.png?raw=true "Tor hidden service crawler") +![Domain CIRCL](./doc/screenshots/domain_circl.png?raw=true "Tor hidden service crawler") -Trending charts ---------------- +#### Login protected, pre-recorded session cookies: +![Domain cookiejar](./doc/screenshots/crawler-cookiejar-domain-crawled.png?raw=true "Tor hidden service crawler") -![Trending-Modules](./doc/screenshots/trending-module.png?raw=true "AIL framework modulestrending") +### Extracted encoded files from items -Extracted encoded files from pastes ------------------------------------ +![Extracted files](./doc/screenshots/decodeds_dashboard.png?raw=true "AIL extracted decoded files statistics") -![Extracted files from pastes](./doc/screenshots/ail-hashedfiles.png?raw=true "AIL extracted decoded files statistics") -![Relationships between extracted files from encoded file in unstructured data](./doc/screenshots/hashedfile-graph.png?raw=true "Relationships between extracted files from encoded file in unstructured data") +### Correlation Engine -Browsing --------- +![Correlation decoded image](./doc/screenshots/correlation_decoded_image.png?raw=true "Correlation decoded image") -![Browse-Pastes](./doc/screenshots/browse-important.png?raw=true "AIL framework browseImportantPastes") +### Investigation -Tagging system --------- +![Investigation](./doc/screenshots/investigation_mixer.png?raw=true "AIL framework cookiejar") -![Tags](./doc/screenshots/tags.png?raw=true "AIL framework tags") +### Tagging system -MISP and The Hive, automatic events and alerts creation --------- +![Tags](./doc/screenshots/tags_search.png?raw=true "AIL framework tags") -![paste_submit](./doc/screenshots/tag_auto_export.png?raw=true "AIL framework MISP and Hive auto export") +![Tags search](./doc/screenshots/tags_search_items.png?raw=true "AIL framework tags items search") -Paste submission --------- +### MISP Export -![paste_submit](./doc/screenshots/paste_submit.png?raw=true "AIL framework paste submission") +![misp_export](./doc/screenshots/misp_export.png?raw=true "AIL framework MISP Export") -Sentiment analysis ------------------- +### MISP and The Hive, automatic events and alerts creation -![Sentiment](./doc/screenshots/sentiment.png?raw=true "AIL framework sentimentanalysis") +![tags_misp_auto](./doc/screenshots/tags_misp_auto.png?raw=true "AIL framework MISP and Hive auto export") -Terms tracker ---------------------------- +### UI submission -![Term-tracker](./doc/screenshots/term-tracker.png?raw=true "AIL framework termManager") +![ui_submit](./doc/screenshots/ui_submit.png?raw=true "AIL framework UI importer") +### Trackers + +![tracker-create](./doc/screenshots/tracker_create.png?raw=true "AIL framework create tracker") + +![tracker-yara](./doc/screenshots/tracker_yara.png?raw=true "AIL framework Yara tracker") + +![retro-hunt](./doc/screenshots/retro_hunt.png?raw=true "AIL framework Retro Hunt") [AIL framework screencast](https://www.youtube.com/watch?v=1_ZrZkRKmNo) -Command line module manager ---------------------------- - -![Module-Manager](./doc/screenshots/module_information.png?raw=true "AIL framework ModuleInformationV2.py") - -License -======= +## License ``` Copyright (C) 2014 Jules Debra - Copyright (C) 2014-2021 CIRCL - Computer Incident Response Center Luxembourg (c/o smile, security made in Lëtzebuerg, Groupement d'Intérêt Economique) - Copyright (c) 2014-2021 Raphaël Vinot - Copyright (c) 2014-2021 Alexandre Dulaunoy - Copyright (c) 2016-2021 Sami Mokaddem - Copyright (c) 2018-2021 Thirion Aurélien Copyright (c) 2021 Olivier Sagit + Copyright (C) 2014-2023 CIRCL - Computer Incident Response Center Luxembourg (c/o smile, security made in Lëtzebuerg, Groupement d'Intérêt Economique) + Copyright (c) 2014-2023 Raphaël Vinot + Copyright (c) 2014-2023 Alexandre Dulaunoy + Copyright (c) 2016-2023 Sami Mokaddem + Copyright (c) 2018-2023 Thirion Aurélien This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by diff --git a/doc/SourceCode.info b/doc/SourceCode.info deleted file mode 100644 index e3bafa10..00000000 --- a/doc/SourceCode.info +++ /dev/null @@ -1,8 +0,0 @@ -SourceCode listens to Global and select only keywords that are relevants to AIL's purpose (CVE, Exploits, Vulnerability,...), then send matching file to a new queue. - -SourceCode.py search for differents languages such as C, PHP, Python, BASH and some Unix shells with default configuration. - -Every records is send to the warning log because filters are high enough (hence the critical var set to 0 but can be changed). - -FOR NOW : Still have troubles detecting ASM - diff --git a/doc/logo/logo-small.png b/doc/logo/logo-small.png deleted file mode 100644 index 1f2e8f34cc4b01e0767b82f2c35390dba1c6b25b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23954 zcmb@t1w&h1(>9vm?(Qzdp%ixuUZl9Y7k3NpUR(-AN^y4xUW&CqaS77mu0_9aKkxYm z2a>=}cGk?EHEU+pbq`RyFqVQ;!6I1e*CB-i7k}u zW}V_XdzwYi1hdo323rH|Z#IHPM=<9AA0!dq+xLAtx~+bW-(~i1BaqX8Jq%>?h$4gG zxH0$OuHcd29^jt9g_2Y$@Kyj$2vG1TssZO*7kduwk0fh7p&oKIoDQH0tO{{~R7q>R zem!GeHV-c{nC@pJO7A9w#SDQCp^KJCitmT1F>(gL5j+f8xN&5p!~324_4R6%G-nUw zii>bm5RrfiaHDYaV3Qch1OXyCyb$1DAT`_ueAj#M|IWmlagZzKKvMlIKnKSRw8U{V zYvurRl;P>2bt6vzg5kaawQ1hG+As~H`HjZncd-tQF=H& zVR175T_>J$6;5O#o;(&jC*lZv0lrup(R<`TWKryHR3{|D=>H~zA*z%(vX#aY7!4l+ zP?KzoTtTCPDnL=qgzMq8DgG0NXPTx%a6gPe1t0)c04eds>PWVbh~RGF`BH^r-e3{F z%8Er-WJY^38a4)Rz-7S)!OjuXOl0`-NbbhvIQc(qEYvh(r44a}L$^cwXgMtvN_Icd zyra_@WKP{7Xd(NbB)&9hf_r`BE}Ggcwp2CxV@iRv;y^Y{3`58NQw1N76YU9M69g@i zq#e_><^EMIlQiienTYm386~RHFax3#Ku94kwAVOX79Mz_aHOZd3iLcL3Iyy))n+iZdXwNk=gzH)27 z`cB#K&(ab|XQBU7C?Ej=2$gj#47ttIciz4J_>=k^3cS3kZ>5j8u zVtXBY25yTEjU06#8jKpde*SLg-H^o{+8c}&j7T{x$>1W+(p{f7X&HZz98!`5TtbDg zS9ACFnFt>Az!XJqA3nk9Nphr>en8hde$r!&D~X^hEaAq!qjq&K@#(~DB&|HYBl^nL z!N0}q#0#c{&_Y}p==;zyr7Nhu0^R}Q!T%s_5Vf!vs(_=n0o54*(~m$$u;$D3O3^p~ zi6)E0aD)!}!mbd}V`;}3X|#(N)!FGk83_&8+7Wk!c*D+0GC zyC=6|7qH^MfFC4$rfL2w;~M9u|01z3{9aTHV~8l^G*kdSaN6v_qW~_Dli<1JC-V;? z0wUFa7^QT%&imN6 z&7d^n2WiZ2SJYoYcg+HY$V8zd_)H?f1C|75bgoRlYG#*!`=5gJznm(+i-Jq0apnl0 zntcvnhPssIskgCA^~-*vQIV@IejPD4qFvIdidGW!!c35~9epuv4}J#iAE__F8SEUF z$B1bV6!oF6!Hlt<{nrCIQcnup4bU>-c>6c79sZnisN{PPXt%Y=4tgXCN5rdDqagKl%Vz^0*V zQmt*FOCYDsXq+r}e!}ltQ!gJJUe@qOcCqEE%L4S!k3&Ro)CbBUb`5T=5xD^3GI({Z z5!s^#42~~qpAGn83FYihhxi@@Sz5WToR&Xc>9TsYWa*TZ6`A#4L(Io`M;ZQTL9Ib1 zZ4DfN*--pY`V?f>F6EeWwqkd9S8e>f09>g{JZ@sWWE3iRMR$@It2af-R_kGUorQ2N zG8QV^v?yzk$?sl!FCzRY*?rG30ls+sYfVu-T4IK%ciB!%7Joe~qZ`-0`}~Xe8-{ zPy^oen}n5>z+BcwcA^{W5qZqunHVolgHQo%NvreBD0M&|oWyQ0g?Iulj6`b7O}DOd z|HMJuj$8SW?BHP+@Pd68*E7DVO+=)+6gkA+7t4@o1lH5~C~qM5jRqDL8x-Rj(Qah; z5Zl+AtA%8glusml9$D9-DS1wUD;5Iq()3qOs+CpGkS-)u-njkQr8-fTK`lYYLC*?D zO5Ip@r5&R#mQ@ov(}hw)%3*Pj?0x-WNK=Yl`*=!3%|lEbjT2q~rwaqaP7O}pUkm1x zJJh{%NUCb!@F9c|B;}x9Z)yD!Btl0uEgeyeCJoEa@(in*dka&Ccvg%>h(;&%-P@zW z1<3^gkEOQa0nntpdF;uMHAj*KdX5WTJh4-RCaQMouc|~+d&JDy;X^Tmv_#b-=*hI- zW&Vw9J;JFsPDo(w(6?NNZORczB;%E=L}B;c>MqumT1~;XY3jWl#$-op(0>zG{-+#Fq+*ka8O(Jcgv`R@9feCX5i-*mOd+H*I_ z1)RKKpeTqWte$_f?WLGEnnRZ0VUalY1znzWixYdk=QS~)_(p#e=v!44Y zQxLD_Zh)I3EusB5QG$Qzo~I#Z>UZh1gfNZCLXK@LMo+N;?397qHz=@DpafL1HcFY& z-cqZGa(JCncF^*SzVid^DfaVJ09%frPFQyX8EONUYsB@->vd8Ds$t0UxiLZ}jl zChDa|-%nFW20((itWsANA|T=Yg!aks2Dm$v<)rWDj>-$@?NL8#FKZTE%<{rf3-xr8 zhh`m&rC?Lel8&RXXUo`4un(r-EC4&^o17xp_5qXg#q7R&m_D8sA1+)UUFKSo^ag+3 zk6o+~a^vU3j2tC%Km0|x3Pl8F5Xk2cFimh z(@E`$ArBmT8=)Ci_BG=8 zXzVegU`sjRmcO;kmV&J1TwhlZKL5SFt(`vMK0MSd602k2cp^|6B@6er7u(({!NcTnytzbPG30o-0yz+^mnLL6v`T+MXYnlC+7r&1$q&F?Q^i znq~CPEy0VG@mhH)QnLg06e0mDVNNHTDmzSI11EsnG5(7h*|i-wea*4)v|5R;M!tS} zAkmk9;8v87amAwcvv)Wy>z-ZzY>am0ow{UWa~;sA;l%lnSU?bCEGY=nUU^Yd3dKU> zp3>D!#fR?P3O^8NW8UWDRFPRH&|tj#t}=q?r$~zQIxJl%WyDsoT;X%4U_yB$>@pSMGQa>4D#Ns(eJ;d1DG z{^4sWbOU_gs!%nJ=WoHFe`LDkQ?gjrEjM(rM8~P>smtJJx@nrLph_g%ng66JMwHc~ zyYl}^emX>ZWM8guN0v9A1H`bd)# zBnELjev!wN;K`0fnHB(xgaUb1NBL6HIGkH#6Gessn2wpzR}>h}M9&P5de?$=G`Wh$ zRjtz)k3L4RZu1g?#T^In zdV>l5Fl0>e#<4f0 z5#`9}NNj1P?o_v!k!Dk|7#8TUmt~3hrjdiOA6o?bAS+STks_%VQvamZPq9@HPCiCT zk0wS~`QoB)Sc2j^C3Sv*K@>+HUh7V>(~31_Qih2^Rp{X_+12SfKo@?`CCtB_WB6&K4l@418iZ3N&l3g&!m73^WKB z?PEc(^p>4e?B#{U+)k8V_evC!`W~{@GwH_M1c?`lo)0pA@F7SpEbO_Oq)9&;MkghC+lVbjw;1&pj^&l4D5apo^2pF+2wd2$Kc{@epJAzW z^`_+CbkI@Ph+iYj4Ael%v zwuU!QpM8GXNO?HAU1wo<@auefs&p3JHiX%fE_Yj;%BpV}tq0s!;VYJ&%$4gNqVKSL z6jX~zNvHZm$W_F>U%jbTcgs8AV~Z^(+y77>ovkF}FTD|( zk*nyrY9?q!!{{LX1+)2WQ%$5v>CiMQLK{(M%H7mD#2Ds&MDQjdj+sR<```#sqe#7- z5qhzx3IG>!+)_OKhnSW#J%;CY9xR5+efXi}g*!dqgbWe)ZuDItJE?E|2r#h2@uqqR!Kv@{f+ zouU%CW{D}l!eMEt(8)F~gF3H#vm&AzH~Zo5@Yat3clqb(0k?gd)5(H8h3TOombYIM zY3pu{MPHuWOoQA+Xa3L|eG3&}_pJ!{zz9FC^gyeV>XQ=f?uZ{jv+$R?el5I)Qleqg z!}OgT^-%5&?^aSl{1D@m;_u7~~q&UWm1Le|-WQJGszXh&M zU<3ca>_6j1Lwr{-*l0r({4%&PEfTEh+&6srVqq|EwL@!68h&bYc;$aO{&E?@hgp4y zQWso;tY_9=<|wlB{I^VqFLR^Ub^R0Xm*5f?;0zNFi9RUMu?@$vJ5+CRU?1C=t}oBzhAr!?^ZZJlh`Pb( z$iQyeMouFwA0%f^T>9N5pqQGQ@%MI)hThvh-CE4~ou#(=`@>P5kJ$=W!teDUTKLR` z=a$ny;yhB1PRjK#AIs>5`R%6;xw@Y>4QG?8-OIb9+L_ZUo7I$skh@6S9f_lYrEMuK z5QD-OJ#SbMpQH4M@H^^Upl;HJi|8o{FFk2;M?43(0v8gYX(TGYrHTA~fK-XMS_EZc z_?0Y2k)9lO>e&zweBqw97N7G#IGP5(6f{nK`qS2~>Opi-?}T5%iV_L2|9M{%Nh3WK zm{{??15SU@`BsGa6^O{JZ%lro8hWxbaaS@d!FMeiyeQOb_qzPDn^bf)(H;E~wB0Es zW7>wX?QaH`N_Z38XF4{uFly|vVmI5!iFMXXd^Z*L&}wDV(_wb+ailUW*f+A@jTKAo zxc;vmFb+4qawY)Jt;(&!oggLv+frZuunk^YxAPq)BDQ$CM`?&?K|NA27C2+oRPlRD z7o053>yj5oU9BJ-G9i=3D316h&{Cj;NG}p)dc@*~va6H5; z{L!gH0yra%)c#Z%Vu|=3h7xgXDFtIy+6n>nL0)v8kDnvki$WxaE>|G;G^1#;^>qxL zCr^!;*f8h>PlT>UKWk5odTdA>(l@hqqJpv1a>D!p>%jJd> zEPQ%CN`+Ah3S;5zy?OoTl?HknH6j~%r1r5Send;hTD*+)fpqkb^q@oCjPwUq7;pob*bmLg#7c5admFUv-*9knu9LC7)SG_qb@=tg8nHz=xH$L&Pa$QO zECX3o%+Ou~t{Zjmtd6WX%cD^xQzL{Mr+B-+wY_p&>r6xk33C)}N6=3)`4engr6o&; z#uuu;nDeM3-ptX zoJ{ej@%?H4R%^8O68Ng4y&{XJf6 z{H7twzoU!ZKdhVg?3pT}>&-dL-9T_JSZfih`SJm{(~ZvmUayqto5xfviWiLakStdIOHc|Rhjr}$EL8y>p#W45t1#4PEmyBHT zs|K>sFb%Lv`}Ux(9>@TrG`=F(_+xIbCu0d69VYrbw*>U;@16Y(?eFVe`3NCNHlO2E za(k?|yIBceE#H1q%3*AxM<38bQpu@_EtFEt$CMhZ2ei?_^J%{H{c6|}7bjr*p8tVf zbj2II#O}5yC2YQm?7;ELP-NKQ#IhX4zwJ29+EWw=myMIAHUtxkMxDP`GhWM`rS3_` zzoC@s8C>a@&v6H8@;PtR%Qv!fL-?Ic8>c1G6Sk8^CVpTYeo+LE${f#>W%QlB`9M86 z`0bmEn)DAYh?QAK91LND0ZT%unWc(YAy$W_rq&&B(M+&~Henm>*M zBE(c&HYS|jD%RomH+;=mmevQG+eR-+#?i+d!q*s)DhFu7r<BEzjHBH^>R9hRr;S(q)V+!8|=jT}F8&a&JAxcHxJD zuHKhP^kmzNkbHl+oTv5-|TZmp6z5mS733iSxbj8y|owy7hOqcsx|SeX3em<`i>5=9F^~kxyDsUS8DLc{-NQ4|-tof58jO$x%G{lVx6Brg3xU zp~p`fSqyx{%0^{Zx>|izw~QYzmZdp#@^##E$I-N{)ug6gzkeZ>Wz^Hp&IN-S2s^W@ z7h)z{_R-Wz>l^~xN59#AJ!HSf(0xP6*$cp638+>yU&LmCt7V_!`sz47KQK_jO2f}l z2d63H?y#!2INx(sCB$UO4fDSpsZe zF!kS7s#2QgHgl&HLR8#6lD~^gXLPFQAec#pnP%SM4NE$gFc(OA18x`clG0u7uX-NJ zr-Q3@m#5aK8ZcffM8sJ5TqlZEKLp`n78vvF4tfk7zSu%f*Gn{sor(u(we zKi0mj&J_+*w#5e|RG9ggqRaEuk-;S=2xfW+)MCIz>cqC!s3r|tud4|ePj*}-Kj8x{ z`{xq%mp*c3Kj-V`W(G^u%=&S-(xe(1+>SKD;`R5A3(c941oQDJ&A;rAU~1jkvgYo_ z9tUZ_jF$1yN(J(0vUm#ZlfU_2Z-(1WHn)$<(ramd`XkKIT=~y{^}@d%aNvb$GW$Tw zOINZ+hw-p7#x9`%7T57oV4DlldI~|7U--%JRp8NY`K;4{&qsqI3)g!=+KLAEm=Hl} zoTKBJxJHELqNZ)+5@{l`3i5in= zdon>4eE8+t@waZ-OorZ{E+(owoVZxwX`bqNJlmlvt`XgVTmwYv-f7x~G>8p&Ht)8Z zWKs^Z7!v=>E_f=7nbLh2J1~&G>q=H#{Kp)?qdUrNnpx~&V&!f^#*cq0Ln1)s@)jbl z@$znb;Ajz?KhD+vtA^arJL7Dvk-yIXxd1EuD8X~W+liyPy0WaimMYp_v)fNZS*K=q zPsSa>JtqF(>CkH^J86sFkNY!ynR0B(7(tuS-?U`Nm7%y&NDxQrAMg9zJ<3-?KP?%w zHVRM%Cqh2M)KT&UpK@e((u-|Ra+wCrcZsn*Opw|BxCZ$pW{zSa(H_pBmJ-JhR&_V# ze#-39i)v?PmASLC%mopCegDaIlIW`^8MIjaU>1P z_+TGhdv1Y?u^2BWe@8Xp^QNN$pr>R1`)E4r^!540*Th)HGl?6S2R^kYFLC8Kvq~KN zH)ost0y;Gq{aQX_<`M3sNm`#SHaung2bxF1bvXk zr6FmFbe|UAc}6~g<}d?)>FKK3EYAwCK-+TGI&!Hu9Qq8Y6~(y}tOSfq;2ww{ZO%ns(iC zePmZrNV=~0U4HIzr(lpHK9lAw!$Y-}BIzP_w-mWu0+2Iep41RVtmMad^4PPE!XY#J z_q}G&9eO&Az1`qE1-7PcUzJ%&gyV-y^ZaWWlPosaez0cpBYVD2!OGhW#9E;6XfU0L zCiV~Jrn7cg>a=?Oj$ppq?kgqy&@Yo8@sZ-dEJ5TYkpIqNRPp% z&N9SrCCTErDG61AEakpNpkq6RUVa0W66c7J*R-xt+EG$Z7OU*Q&x+$R`_^VU+vM^n zy6KINmDgP~jBJj^*T;osZk}%mo3ZR$Vk=*5qQq~huC&`OV`SE>$hznt5f}IpApVp2 z>J#AmlpZNs#4zylR~cIlb29etKL}6?hOv^Cj#RDG>dj`BBq2aHhC!kHYner30W-67 zp$yxb2uHJDFPR~EPejwVbeU6k3;u4x-}7BXZPe6Cuod(IAI9Hg)~!SU6G%iu`I6W^ z?^UBxctgw%)~MF+tZ1kE@+bzlx7)wC-5e7BFjz$0k6)$z<&l_z3!*S<7K*=_%>li7 zQF!JMOO0%VdDUl2=bGKGT;inutNd-0%WZ+Tp3tXuHOL`jS;)tkWKq#y(R5B1$X8_^ zF38*%y~eG%x4n;r-T6*&Q^J)}B*C32yX#M~_iveTZ0pvoDEL|{C(*#OwEt=%TM?%v zhYU@!sjHHqY*@0LmtR}n!$bpfS<##wi;=Ujv>E)F2WY?auJbl@3bP+}J`c@WX@7iRrwMh7``Q^h-HvFc~|!2z~LKu z9jRHXTa5eNmGcnOdNDuc2ztm=?Bt?_b^{k({*~D5YvFhK#W{2wfNZ`t{r9 zehxL?HXm-(LOwruFdDo6_$F@HRmUQ-*m2Ly=%0*qM~c+Fw5F zD5self+;@{>(ZB3FXJGJ`~$g^{oAVamK4cI%XN@)O-rmWI^}L)ajV7kSOl<8PaC^_ z%T!rYWxM=PbooQURMXIOx_A)D{Zg*NeHO8TyE=pmH>&CH#gwh-EGu(7orL+_Ut4rP zmwBUaj(GGKiB}ji*wnMy1E`QPP_+r~ETSM^*s(280{&F?EQf+2cDR_+>NSilmHgz| z+!2FT$)17`e8H(p*Irp(8@*maXDk*{36w~sl>Rt2-tN1CbMwFXj>G-B?|%``%`fmd zEYj2pJur0cdRa-?Mn&0i~ir${Qsx!+?fBX|Sn^NSb z>_>9Y&FA3ft0B9V{W<3;dBra+^$wE>#ge zZ~5X8+3~asQNQU60gA^$yZp_WH~#b5vY=U59JJ*KB;R+R$=!n?pLJu855`@X?a_LS z6M823Yh7DpZ@FP@F&WF9{wDq*RCU3Rj9%9+@~6yU^<70YS3IA6d}<{=;yoPC=EWV5 z73Tb=!HA#Ag-AV|$n=a38Wsf&cb|qNJ@jVh=$BIAWNxyN@5fU-aK20BLldN%a6SYt z*zZ{zVRs|V$({vO!1v(y%)%_CWDy%xhVvQ0H&iDr_Gol}yZ7Z={prD1MVu5-Z;apQuaG*_D*jJw!-GS<$NFHMn~^95xYA9R2zshLV@ zIu>|_#>hv=wHdDO#NZmfGe)-jY@HKIkzVOA4|XDLwRVN|L)<^#2&FRSW56@rO-j8+ zu;^wz7&C;vp}-zh6H77JSi5n$+-EJweyVNeG1u46?TWcZJV=@eibdyd@v7cv4ENi3 znFycVu)ZI?ti1IlZF(_l!1?Rt2w>&4tEuz)c!qxK2fdS{C_ zYI5H{;ES&lS&09c(DT2E!-@L~k%Lti_ax-WWRJvf*8s87|A zrvO8`>;XHctQG@wn6k^vQ+60LA?JZb562=yiGj@LN35&7nxR!A>t`^=XZ1Q+M@UNz zE=s}LYJA}PkD;Y5U1@Kpf6JFup zH0MFD7n0GAPpE+sfHGJSY8f+um}z%0yLQ6X`FJro>3E4w1)!22vKX&trzmiD-HBW& zWB2zJH7O6`PVU&?=-=da%Vk>1*~d7@7>AgwkuKK}!~6Suf#o2Z;AhObz&pkwu6xAs z#`gOORH?|#l0S~w_ZRH*E^BJL`A)Th&Y}v5-z!|Nq_lMwel{srM4(9nD3U;tzQhds zJ0f^E#D}&ym4W^XYGzI&F$?T;DT9ft?# z9Ve@?`M!53QCEj@CTj0?sX`EJlSMFe-z*(Yn~Ga>QN&yfcqSxNn+wIyCe110Hen`A%GoSXl%A&2ovJ z9k91%Wju9GVvo;rU;oLeUg;bpir99R$~LPq!=WhX3Z*YWiF#Z5*=yQ&YF0gv5=pj1 ze&fsj-J0`Tf)=c-G0?R2Yr-kO^=e8g;tr&`GcEDqmS&=AGCv**%4EMo@ogy!Ck_3g zrB=Gd;2>Clr6yv7L7eYPU|&lRtfxQG!!>Dk{P#G(`B2Hted^p&sNyFxM%EJ1AaAf< zZ&YR7Ta^}|-E!d(DI=~tWTzgtl>>*zCA3M9W;uanN?^kDllM_R&tscUZ z`QBF~ugim1Q|q7GZ{d@Ici}C0ZO1J&Y{O$hk(NSjXRexBn%RB6LNAXShpQs02G8;V+%Sq_K!4Utj1e`7_XH# z0)C5ZI*k+11+EjnCSiS1D@9i;=vSY7(!W#HjgV`>vyf&|X#; zfd{Q%gO;R#ipiaO52l$q+Xb;!uQEZQE?0K5GguW-?|UuGjw9uJk4o_i4S_Z^H8*P0 zY5vdJ{|Au7TQfBlF`i&BTnZZ5(8n^6n(JMzLfD5mC|oY)`a!3EvS7q2&IH+O(@`^r zMJ6Himiq~}y>LF=*8(n!>txQ_PR;mOE9OIa4m~LeX)JLhKrI?&FrJA*&c^=@Y-2hq zHf!^~o3(!U>}X(6=-~5HJ^S~u325Bxh{sOMJS_^)ZAn04{||;N!SG<>DzYF)*l`bN zKRYMJvAuGEy%fump&iG^+rQe!Fb*GJ(~ z&dH(a0RbTEXG%^cZa7wI4yKIw8E&jYbFd~bk!^J4awxG}S}7h)4|>2EZIa6Eq?IjLMw9(MyIb*^}ir(`gEb!m<$DVpE`Og zho?=XXnTFFJbv5pfp3b1e{l?bGTk@W`1I?2g=z_y>7y}dMCc1(D1^s!b58ZrA`OOs z%gR`5+FEIs5yn38!W)iX{l?f-AC=`nz*@MH^)ElwtrF#`eaC(vd^XOBc}+fiZkNu> z13%HbW9GtFV7=|w;oH#}C&E3`X^2JijNWu3-xQNH6JO=NR5j_fTOqQlF;`HFMHzC6 z#G|WckPI?b4O{HtN3qR+EN?82N}_DsTdDCVsi5Xr&n08`2~v(8gSE?K^VGxI5f&kL zy6>=yOt&fj13hPMVjUrQu+Tv2w^*xk1+)v(hAt1fw12VvHhRf@rd$f|JMM#T-_UK> zWNpiO+d~)j^V8N&wI#aXO9cIxin`@YYP9=#grzJ=?GmJC)~4R!$wsGoBVpyR>|>j# z@Si9{aW235H{Ti@4r8TlCgc_yyflcSkm=5Y$EHY!IzG3-pr167twO}jU8O=qqZm(# zSfMUs&ROT|L$E#QFvSez0KTO>V+58=qhD`W0sn$j)V`@!gDvH8;x!6z&DJ1?X`*(LRDrx?(^+E1$Lpi zF}`3JgIHYC2@ zCj*8i;G7)7g>6F}jj`}q+soh6Y>KLG?(+@NZY}_+y}?cH9M2sVZj8blp)6Pp?cT(< zzwm{hr0Hg)$sXOe#_rFx4c+{}w4IbbW8a|@9XCsVGz;DcAR4}@CA$SyE3YsXd&vh9 z4Z?b-@k$xju^}(?Q)}vvRPIDLa|@1fF+tR6F96msK+S%7|-U7k=onA)XuLu=D#xRj8RNV(fnCZ>nSVn~7jp`?&j6bXC+2_6CXMUu@?X7~qLN6wmzq4_LW4}L7H zwE|5lB8#JU9=;8ZnV!x`w`BM4sQ*x+5xG0tU}^BRfvDk^8zWt53ym&2)OO3viFMTW zB$P%s=fixk-ZC9ezl+?EAv?QA9}Q1fYCYG5;oQYr*uKm;rIa92+{tYZu~%l*I~{DR z==x$M(Y8qr$xk8ALn=HLseG{ZqpMhTIm^h5%)O{cU5_48mj^y)Jpe)^QH*>vVu9rq1J zO^1eq#{u*hj^+ZT;ff6Gh~kmi?MXoyH1j7Q9kxnPql!T_(`zBk5T(P27PPqaGn}BP z7rB8#4emKHV}d1lK@>+|RR)B60?B4N@go2FbY6iXuUlnN>Z`2$^?9qxm#R}4cU)ic zk==ef#ho*i8-J0aD)s?&QXjI6A`$%7)vo$)`G4b=vMSC}JXU*I!>B(GuzF~cr1%VY z(-jKHRYI~Cl7LSJ`rygh8UAk34!R=USD=pFTzse(QNbT5O+<*bI)CX`XsZtXm zu0w~o^n%lGD@>=qXbS6tJ8up9&k2-BnjJLul1(eKjbVxv(YmZ@@AR)>vLyb#T|ODK z4aHC08Vaz?onwE@y#GDiezVp!s6UF8vtwhcX`m5lPj>XC4%*PR%U}#1@wEWLs1#sj zErGueX5ogJ->P=-8IR5fe!-I26;2g3DT|kNteJf)4DNRWJQB#k3D-lrqONSP-nMQ~ z-RUJtg_k8fero%ASwmsPE5$**oWOE5dfIP|CLyXBbz=E4@BJc7vck!gcFTnn#% z&JawF_;0u3bdDE_u*Qnj!{af9B#Z6Vf(gSbbgzc0Cgd2zDLZrCC~NS-#YkYG%)|9o zTI^`rCtQZM{>HC?w?Qy^oJpGCM84V@j8%^BHYuOxZvsCN)&A;?@Gp|JYp%QYaE?bW z_ofJZU=H&nOWfdenYTFQ)IEV{*Kpoi=CffK~+8`CW>hLnF>Q& zSWy{qQXqn~R_pRdsnz(Of+%?#n6yo~6Kf2s7dab!?%kK;SDvYa0n>psK|DU?gslx1 zgIpR%>FJ5=Q zr4=0(ybJXRRe#DZj&pq9xg=(N`YDa3BoRsw!io4^>Xl7y_BOCq%(WU;ClOWU7D%_( z>~r8C{5WODTeVnk-2E$94)aSfg;~j8{jNLPH?~u;3dYhYcswuCN;G1?jQbX={~8W- z65|FI>rvZ_aK_+ZUoP<;OOu^$`e6Bc?)~$?xNqx8LJLbZz(m%;d>yX(1J8eQ`Lzk; zNhONUh`a|NCL~COlPgP4eZ&pl=nw5#plzLe=8ikX}&=DQohxq-vHuVLK4Lm z4O{DcANl#UDx-R@K_cU1E=Xdp|BeAFZ^E^&K%8sK?L4sB3q1zlvm*#E3>6VaJ$ zIqg1lu!DM63$wyU+ODEts;XYjZ>C0ChSLX$h7BpXA)XN{lg+NfJSJM4?wP{mS+KXtY?f8vdN1`sA0R2;Z z1^fpxf&Zpjx7pV#a1q?js6f->emlfn|IL9V;7pOi&(De?z*D<-?k(n&&@)QZ_330% z;~EK15>b6UNZpGkaO2tYD24EMgV;)|hWoAdK_+{06wNAXH-abzJ3KpK*el2xpkRr% z9ZG4KjU-n1c&BjxE8Rs?`6uy*yjX2bB z*i73M7)^s-cnz73EqqTt1Ri1}1_Xr~Km$rQ{GIO2<)aCp0JFD0DZe5R(G;dHzEji9 z7@7r=#6DtW0v++iipyd1i|nLZM#({cBDsP#1oIZXxcp_lcDo_rV`4Tnge^`tf#mk%P<5cmGine%dFCn$DWaf4^{*qH60DPb`hyMigK)yY7nZ((gx28BFdBu_vRt;yuZ&32a zxfz@wK+#`At?D$l64;udrW$nea^@5bu+Cb6u~{5+4ObQX(^_7o=LhcV!*?kjOT&T} z7kEA7JcAeQmG^d>Ww&BcUA{790ciq( zfIujL1QD*FfJjlPp(D}*#2OH#*APIZgET|J8}I%8d^7LwcjnBTAA7H}*Is+}%vtMM z&vW>PW0kDR5UH}Ku7S+D-d%q9JZd;cyBU3wQ@Nt<(do23;nlvv~CEEuaQ zvUbn9yo4`{gl16nf6*Ufe;gKCYM<|rj;DHmsO&^UMjDqT-k0Qe-G}B-3f4V_P2(5T z$@3D7#o%*(tYD%3R{BUJp%P*@Aa6$OQ7~&ry=kwBIAg|-KeWhvf`WQy8uIgpVxpSQ z^rZHbUeJ}`VpJEwj(w-w0@pAxhOiMYRGBv5M5R)^=sXcG6T0w50pdBIHF>Xo-I3x9 z8Hm(ZE=~H*VPOP2QaWf83Kc&d47_?iqB2fCi{YR#+g~HJNY6k>8PSm1!ML>UN$j{O z;Z69?E_>c)V|2xRVqOWS>$1n^KhKW8@)UJ4wgMUVD~TGU@4*~kf9vQEO)seBOpMnhw>{Es>&@ zN;!;N-(%vRj;M}{eJhzqh{gxYeA{1a7y^1`&rTK%JtPamfh2VvdLASA!XgNBek-G2G;N>)d_;;=Zat8gUh(uQ$IBm;$O{^eMtrOs2Pce6GNwbB$4^V zD8ChcIK)QwYU^sOU_EbW*0|*AeHVeSb+4^)GjikP0oIY@6ZUl48KKUxBlr}r6XA~D zcqrNZGHK)ZsOwY!s(tCHaog9w)NWeVtDZqC@XLQusPBVwiw4%{ZF*I$Po9+}Y|h#h z_AFuFbW^9McBf@(h>RT7M9R>kci5UpAh{Fz_NM``pkcbm zsd%SUWSS)jByYueq(!_mYo27=dpN~$f>~JPcy0E1@Dd4O%d>6>{Ln?17bNb@Vp5UG zZS^8`_O%T|`%|YpFwI@X)^+S3)rZ}UIuXe_H<*b$EojXWPln9NvsYC(R29%rBx}O# zLE97fQr_@3I*{iMX0QD{uN$U<)HV0-q-YhuaK ztZ($9iuV1J?hb}_7X~?Bf(Z^*AAPFU<70*4S}w+{;Q}QpMY^|QGAu(T8rt3%0DqfK zPsgkix<^5{L- z7#Cp5I_;3fDK#YoGl-owu6wSwoQh|ia>+n!(;ruPy(ux@ffVCMTTS#(2yfwH~I-?a;(jDj6#GeR8Oot6kx!cIS$j)y?{7 zR8#jyLwGz@EbW1IBcGDGHeT42!F77C#)lAt*HXb@OqiPmFVIb+)>(jtcDEP388pqc z3tO#236kGrk5YK@sVU!*cyafezKs5ud;28)_IwGkIdVHdfj;$IEprplVRkm7`kS9q zn)h(=16+7!r^3gLcmfOvjYB_K{A!uX*RGL}d-NmAo+;P}sN0XGJB>?l!q`#@=?7mM zdwh2>`lDCZa5~2N{-_Y?b-zwO2#rx@Y~#n~K10luh&tWgy~h_3jBWk66G}2@AD!!N z$N$olX#c5D8GRxIc+-J!;^QoLm*{X;>87#Fq4K}QNQnUjaZaMLlOl&N^;9M9x+d8VWSM>;d*K@3^d5Ht&2@H%%^O!^RCY@!bGmjY zxr@4-)Al%qRuO=q6F`Poj%41@Lz-QxKU@;Py@>#Nyw@QVt?4B`%l?eKvV8~UmJc2u z{E6i2`UE^3GbwhWb53%eM-jUFi+sYaeR*KSM1jW0A4>L>;&i)LC0|&mNqTIGkXlEC z=XbQ|q67h-S|NGgzu?35gpRmFxo6Ehl)uDCc6WLr>d^Lk5}373MUq-XR_Dgn#I%Qf zvaD>e{wDX@Iz`w+q7aJ@-{XYL^etPGj=L2?PrLcsbwx(G+eEQ8@O!5*Ivyw;hW4N; zf&0!+d_A2wX}7uUHSIPFU(O9ugL2gg zs9(V6uO7yYc-ERDk%fb zqJq7t>J1J1t4wYk6lx)p!pEE&3!%z!fYubkLqT8Wues%X3lSo)p(DDj`D;nG4cId> zq9M%=_#|w|K|p=LRE2EvPkd>g;WZbPxzoI|lT-HSsBaM7qCw9}@R(3G<*gE4UFxe_ zIh!|yJ(V^Yl&3wv|C>EV!9H~uWEO7%e2dt2czcO2XaF30i%xTxbs3MBAGqclsE&KigEyb3P;F4zO~ z&lS0xz4_eoVSU)U;Nn`Zt@#@lJTwj+TH+;09kJcFi@z&(kC15_d`DPQdbBdafBl6) z(yBKAp`%a<%4_u_&m+vFwsr0@5tp&(`A`sp14k;Y<-pnPJ~B?&hftHpBAIwfHSUzX zN`4cT13PkPDKfH4#?Wddau;8owEqm~*v-zJJ7wFix}BsJZN&M`+uNp#44kebU)BmH zXE#;%vyE%EF*}^MfGM|L ktgFbtdd)*<04JdKOOcspy+g(r`a!RdE6!nSzQvTxI zXX-&DOKE|Q=d?LYf3wV;S&J#eK`r$so?Asg{jpAPLo||;qs(aNCV24Y&>ZjZiX-Sm z;oTgv99O(TrSU0r6PS8a0Tb^YrEjk&N49P4}BXo%^N(Vxi%qeLXy3C z;hT|P#zGFfk{&xcUOR#xJ}%Obf5cNB4*5{W>kG)pE%d>^jEa2mBtz)t*yTQ&1>dV$r-PlYdDJf z8HYR3j#$3mBTrcXO-uksUF@>4bGz31#*Mo-M}#xOndEOS`)Lvsq9R4l zg1hot)#JSgifw##J>+(?+e|NfD1fI|0H6vUYfD<+I5dDoDc5sq_#o^N&Xw{ol3M;3 zd><=rzeN@wv2SMIAC0`PD-7^M?da&QLb{9ZOM|zKqVsi3Byc)n(g1gjP*fTJpU7oH zLn$7k+C5z)nQ-=cf#GQK>zBP$72KCj)BA^eBDiH8VF$JYaFnnybFYzep5$S%jiEx6 zhjEwAp&#_zR=|OCrXxP_Q)Lnh_^?fXb3z-JtR(_yn?;k$2_N!N&sHtvq{tw`pMK2R zAqRG|_0Q$C3?9u9ph~=7+>R2b7#FXb`CimeX^YBWr|92Fbso}qbYndM@2A1i z*b+sd0zdI@a31gu=tFZCY+y|J5O(1C_0FTTE3*KE-Sr=zyg2ReooZzakC&_Ac3xK) zs7Uq6Oy;MM+MN0)=FYhl@sM2hrATvbkkw#T2r_O}QSAEWfQl@2553mbmPQCL{bfTH zz~;C<`tisZ-{~*%qH$M}mhOaFCRc-Emw5_O<#;;1mu=!tn7{7{5!c6<$G0hFqE~Xh zolx!{{7j}nMEg7!?E<-r0;gX)VRH(CyDAT+jlb2oq}n>17qS=?G!P!NM4l%CppC&= zO$Gdy7Z>ysSK|8o!7=kTa-=#!hX2mx;1ZYKY-IA9$I(|#)URAV&`VlkQI3i zbZLlZkC+^4+3)B$RP>=$9>(}aM zkd@=)JFPt0xzcYZ|=RT3s6U zz)w;mtK_H%^2n2l)OZ|ZXELC~>L6m6-qf_XKVenf>y|cylKhOh;A0CL!3)BA{)>jB z-jhlpUO!iJpZ%~XIsAW{*(kKEJwA4a5DzO zaIAjo`0mM>sE-j#a+9daYOY`hX1nm_wqyfX$vxO5CIeteBj(8{L(eSBKuLB8lCH`G z{;Aj5tQ(|Ox`+O!wU;S-$B$h@MVl^fm@ocE|J9GbGOw(7LhH$jq!*L)4&?-BLzcJ} zU&aa+AADC4a;d<&qQCgL8R!UbaFPPqT>i>7XrChMX9B9Nl$yAV=^auONzacrT?1&1 zcxdg$%u6_cGEHRPquv{QNk%}+VhewyrKWA^c0KKR|48TFRLw6=PWGX< zw(#MdW7J6X45rpj*GLc{*_otUT3U~45Q!e1y}8HuC&^QI!MC2lF?>HNB5z?5%PH%C z{17k8rRV=*+3BT7OI?qm2zAIekL~xOfpeHO0Aa%(_*L_Vq<9)w1K=3I$D%hfV6+y-bFm&Ra!5LKK;iq7!M=Ih6u4A|4R;GaVD7Il0D0 znKv$CoK(1B+j;es~6xwB0*+zJr$ z8z#V>RgkFT z_qgQHma5mSgR|s$YMLb@X03rWv{U!$iKj%8JA`0dY!+m`#FuGnRyDka;KZ!0#IK7d ztzQ4Vb0*{P*7g42cxnE_wG2W81egr?Ji)8xK{t8!eOEV!1E21vxm8sGhuomR-!48e zbHKivZ#X#5BwtM|7Ck=WWotdr0O-d}G(ycwTpcqLO~ycZBR*JTGf|-=-4w|&KJ-y; z1^iR7q+KN=nsxOtlRR{qKSI)OiUG$>Rt%L4m*mZA^o^1(1H@wa-+L2f?rv0x9$b1+ zqc@8_`pS=c_?AA}?1rIojtqL1dXc0Bv?^ zdo7Zo4Q|5ef{U9-wrs(~$Z~s>{T8qBhkUPT&+7VY(=vt~04Xw3o?q&nUJjyLyxoLg zuxag$R6kK4VI9d{LJxR?%yF9WGTcvzz%t;n8`SPd9%uOHA_Ogt_t?|->W$`(p^m9Z zo^N0QjW00b(Th>KbnS<8iBRj$zD={Rgkbip7z^;SW<1jOT)nj-_9k+{RPdgis{uya6oH&C| zZH(-mb`(j6vi4gf87mGm?%%?RM9X`0Y7NZQ3SvnoNax2?hY{+LqhmB1;Z40z+Sa+0 z8B$TO2lY9h+TP;9No*d28U(oMxfLaM8hsoI->H15vv~{I^Bc8hP|d+RZ0Rp&6(K29 z>y4DwX86gvPR{ZpIqM>k18#|9p27ZKceD6{$m+KN&i3D@r;i=6%!QAGGu^^Mr>B`Z zR$`+tYrz6vXT71^%z1IEQOuBrVo($$XPjEA(ia9$Z7vpq@EP72?Fw5WM|$eQfG6>W zve&YYKJOVpuP)LeI}gT3uIuwxZyE>*+@6Xr&XTo1>27^}te8zH5FegDIF}LO8q=RiH{`}0njOW`&QdWEzgKrW!Ht~~ znj9L5EcyVT*GQJ`IDD)T_Uev@6XzY|z)EXc(M`eE#zBt`L4k3J5({z(nQsx*% zzv4Z$>KPw1BSsdkX&$cV)_X*+v_nAia|(*R!i1Rwm%iL5Oau(cMNnEzF2-P0j~t@s*w z&-IGt1?p<~YySaQZ7RlW!%%9s!^NYxu0orif|)me<$v*PyPig$wiIIfc`DB%-P07E zRS7m(eG5{ogd0JJyS=LgJT_sg@&I_xyd`e^&sREvjh46mc{I_tt&gjouk(I+UQA)v zs8a%qTu+Huyky6)r0@8aBC!k7D3oTo`j>3**>ra3{YMDyPN-JR?RRyiMBk!5)sQOT zfN?KSr_E{ROiYoXv+Z9PihtN~z@%`fMd$#gY{?rq;8d8xFU!JqQ1QoRgkGnh8V-U!>EJ=)bIl^)sR z_hUK#)zr)3piNe4X;$QX|3&-!$R0Z)ndzop0+cT|@ooAOoi?HOCo%L<#nLU_e>*`+ zlhXU3gF?xm^Os$WlnFT~3$tsKC9|=Lbc6GJukl2%1IOZDxDQ2(2Wg^jnEr(I_0khr z8_?v4HO3|uCHfz%o0;BcmmtT*DtTo#1sbIpw7>}s+KwwR8TgBDmNvl){03{FydN>? zM{*l}!_Km0F*t5BfvwhC|3b^sfh^5}E&Kug?mp0#?N{pY6jkWtGN5d?W03n_Vz8*u z^{-9x5ul)H0Mn*{Xy6+(HQyT3q6<8eMDs2EtLm-m?lOHXLKWcWH{b&Ob-N(Q{Ywyz zD*c-O#`9?7YcE+Y`o?h$%djlf8P4Dco@dY6{R=Ql2TIUZR}S!hL2ph^1q{51WEf3I zP4G@A_xKA3->AM6|itr>%ed= z_9tYy2e@*Wo62TpjI{mx>01HctR~WF0M3mC_(D2g;TXLH;~`x!L-@bQuAuRbsj<7v zkWdO)@~RZ0Xy*nq+)y^Dj^4YZ*5j{dw+>8ZhXRw%_jvX^SAp^dG8HitF^wht=k~3{ zmwA1v34wC!EbrL4Osf+Q8TIH~{)=a+O0pxzG>;Rao57(+ti6GzZ z#6^<}Es=jk&d)zEwtwK|{=i$)Gt`^*qEl8>Ql(LNO`4ir?-n*8|VeZ^>_SwBZd*6>LO0q-*_Xt2B5Ru%A=V~Aj0R;$jjS&wQ z_(rzW91Hjl-~NS;BM8LBi2jG=mL={2e0kSNTHEQB-5V!YqjzSYXQp<>W)I|SjNY25 znHiZv9e$dDL7>y;a?hWsyZzpr61PrVYot3~NpcSfZO1;$mMgt3JY3{f`0Cpn!!7;i z8X&9uBx9_8#RO0puJ(iEmjwi{Kr?#9kY*=me;b);C(D=Du|jQRa1$qRq_DAUeHO9# zD#zX(o)AHj7ap4^e0#ok(b{{%O`_)o|IeATMiRLO$YsjR`Tn3XK0xcAYs|Ju1N-kE zAc?eF|32Sh#rgLchX8$K%;ob?1@G@?&_iFzf1e-VQ~b|Da{p_g7ubRSek4KmzgGPs zLI3YYJ=cI!h)4Reg1?-SX(a-1NTKYqHy?b<@h^K5w$%z1*KE`srp_@K?Oe z>le#8sa|T=$0W;@sPoY@gnX_c ze*o>Y{}%O6KNT5O@F-QC=sr1+Kk)d*9&^an9VZt~s(({f{x2aC$@mnKa$8!|rS0ziZ^)^Oe+2*U6bsLhYt;1xzk^o;o($=TrZ^AFY^ni#AvL zS9R(#eM@YJJqaD^4zmK|wg2v`F!)MV-=LQqc~Sk>&VAJIC|dji@YsCq{~h)|aM-GN zK(hQKw`iH@C4V`8y`<)<*ChVCjV7>-m(j?Ym=fxik$8kYWxC(=60fyl2V>UXpBQT3 z5TvVcT2LqBF}^&0A)VW5o4Bk`7##Z??L*PmY&s6XxTUmUi&xn<-h(5tZvVg)`M)x2 z_Qazoi*aK7_Os44!SxZ?MzLe`*T0(?z?1BNBBd_V;Ja%0KicI29@hGwhxKs?Eb5JQ z7FK1(lw$?ni0Wsbyz7+uN879waK8}XVh@$SsgB9T zjeIP~K3l{fKZIBZ_K$0+_!I+oq$BMk;l-BCk__5HFHweb>i^_anJ7VD!SVu+1IZQ6 z_~WmS^x0xkd0~T3)}o7d`>)nmv-wI|%0}LLBKdD|Lj)d$+I_zOl@KjxQYG_()jz_` zpZH3$+l{PkFNO|2e1T_nqE7Aa_s_W!F{`{V>lT@ENpO7#Rw2p_z`IQg*i$+XzW!YN z5fbzS1?%QxRiMzEAN}{d3Isba+Hz#A64h=&+pB)fa_beX|A?taJ`g_t zTn7(TURb6GabnbQ947RhOMb7GY*!pn*;h5h<}n@Tw2Qe0il+`rrO|q@iLysb|W^5;pzdj}Rman9yF~EyK;GtqmkitOGU-e1D#vy>h z05&V1j5gA?=TXGmm_!Rt))i5Y_;3T4OaBC?P z^^Job_I?MoTJ^t`Wh`li9SZ^YG=C2yJw7@12C2SsNyseUUrF50kfJ9{L|Zvz-c3uu zhX^{H@ae{{zqest*nym;fUf00<5jdI4F|bUV^uhRpKgAyvllN)sX*{$FSIj6*aIllfM>S9fPE^bW-b|f|ro9 zS_>IT`iP&z%a#b-g>d|}jOG%Zz3Lx;`f>vrGWmgNMiy%8QTWfWnwV9_R4|7Xdk8$< zi%=3jn@-kpQq|LgfbU0^U(q%I{wIq91QT`-W+7eoJ`bcj2tl~nG+ws%7IXE4 zmuQf%8uf&ntSdldEU6Z1a)ALaWD3Ydw4A8o+Lu<}Z2xi`d9j3! zUOr{O8M-h=OTvx?bCC=L*(*y|{W#t%d-aLM%rih9MQoILN5mfs^0l0n%5kK-zCv%l zwdTCNehGz1yJ4!>fdYIRP74vgGr_cZxXm&EH=40Slnl-+Xgi>qbHS%<5vV&M`U|{QbuK2OV!8KuSG(gQ8Tr@Hv94KUw3KorVQ^} zsEfUh-+0|YcMr1k{^sTqnt8B$O|SfP*OquY7bMs~PncuP!HTG69FTQI2Dwqw9p|gE z!OZ#c;b&e3r04fK`mjDxT3*ZxK@S5>*!_yM+CO;q4onuLRvm^4Z|Gc0QSx{KrC}5= zIciVJnqbGQuLiJUAmLNaNrw1_Q?LcCEi{82bl)LzWqJclLIh=Q)dE^?A}T*K%QIMu#E=3R%u@i^g%JFBX%z zX##o}1Xv*}WE1U@5^fOE2cYrvG({<4aXC^ zcC}4ei#ko04<{!!rJt$|F>1YQ2oO91IPr2NK_%ukIs_yiulaYI+@vob2u2k0U{2N2 z2hNCYIvInXxz;v8s~I*y^)8>rV)4O~zH zXRZ4ZZ?_hN`VG^zIIqb)@@Z+AGv%hts|;{^ky+K zM=xM#0Z^jhG-$bklo0@JZ+YTW#9;X_v&RZH%A+EwL+kKYR}Ehdx2bh%^*ek|2MlJf z!!AlGAMsvu^SQl5R{P)mPJ$HQ$of%i4DX7_6UQy*EAl5;av=r#7s@DY<83*510R6Yh_xpR8y7#BDD1Updgo50=FmHr+r$P!Xc8q~BSa%D;2VIeSqq=SHXX=TI;pn6(}2BG z(X^K#e}N>(DQFsHPSy#BweP1(ZYCBaSDhlO2R*(Tj7a_Qg~tp)ETeq(6?}TKJ_zR0 zz?oGRTRw3BVh#U~={Y&kZYkkdAp>#D!(Xey_u41EC9X67=UbZ?RH~QIYL7lJIO1yv z=o=#wC=Fjsk8A&m1*t)ss*W?J zw!>f!)N@1nq|Bd2`Dq%TaQo@y{F1o7wOo^4cJ_vp84UX>@1R?6i#{s|k7Dm_J71?i zKgsPzEmA|o^Z55^z{ANrMof6iDjM8-)cpms^#U(j0?a~2Lo5FT#%F1f5DUeQLGU4J zI0x#XPHL!-)N!5zJogoNt{#z@b5nf`8WX#8o0(JJqjY_(eH>Y%hS_hUVgrEd@W)XPmRBte!+@i4N+Iw8@+6u*3T_4=bNEyL-1%tkGI?Qt)TQ1t>Dn20|c zL>xGoCvcZ!&|@u8D<6x0NR{)f7}#2)-^MyR03Sg{)7VJK{7r`Vm%*vf%+&*Y^4PT% z+k7B00#`Hlno6f+NLsfckj$XWKIv;<-J9p@b%;H}POA*SSD>P4JOv!Nmi)8sn`&|h z#}#NnD!0LJoGfU}N+IAg&=&;A;uJ8)e-DV6AHqP9wvATqgwSF)<>+*Zox%FnI39ZE z?|L$EEyhrnDKNreP|-OJ6XKgkS5-P2(DvDA-YYVNH-GbsEvv-S1_bIWuyjR*s~g!4 z{Lx+Aa6y2K#(0UT_LY-q+?B%B+Xi>9|?A44#V#CH1fRFu#paSdnx)K zQy74SFy`90il&LGV@!iG4Oc1jgb-x8QqBc&rZh@arPXSWH%OvUrt+93{RUL^4d2I? zQW#&Fe>6#0X0@3OeIhM8{8pfNaOo{-#5v`L458Rgd;u*LY8VKpKZ(P3_Dr@Wx-lv% z4U`6a?Pci=IyrTz%RgFT^FfpX2*(kLySc+k6KfrE$I10OhZ@&jpv>eq4!zDi6M`Lj zsPIX=M1;MkERYLfd;%C4+c^>h&&t&n|b1WBG+&z;i9!!fbr~PdqP@A`HcM3%_1JsSw zV=_zz+|hUK1f&IR96*Ew1V6V7sE?{w_s&dC_kJE}A{`Ayi)vBR`O;d2c(WARv5aNsHb74Yz~Fy;3aY054tSIlQs~oMIRfpdnRyf9?0e*QXVMLLdk=!nWcrG z^YsYrH@h~vo8R2e!^3r8p164lKavtvdb~9JIZqt)DO07Uxpy~^q7Cy;U>`lp<9?A) zim^)~I9{E*-=KVSp$IyyZ#k0*NN>_NoT>im-yn!7TQw!qB5#a~g2#WFvgq?q&dOGy6^BOI6%_pd_qB-B~W6+jA>h>?YZ>aaAX+T=8_xf5XEfac-Q>z>Z=UKa((+7Dn=So#JDkZ9&8Lb2SBpp0=^I7!R$A~MoPvO@|@#$WsuYvjFaKS5cN)2 z+u3%}svYi@deJSXqSk=aZXKXBTg_f4e1$${^KU$gcXTUj2&0zXhEB%VX9nvLlNcda zcMDoHdXO|r36IO?Sz%v)NikH($MR{JY@9IzR zG*%`6Up%qA_&-MhloXL!n=~mm2j`7GXm=o;8@n}1Urw|D`pX|t-90#z#FXmO?M{jM z$fpvklXL%B)7Xts8eEZIN5h&9=HJ+_2i>P+4q#{ZvrDx+7xZ_@SHi1RHNxaiK8gUy zVg%H0{!UHg*ZRA6r5Z1w4)2VNN?%&gIwWd`Y(^DA>gD4IYITY=rU~kg4J)6B`P#$e_{>@LC$JQ53GOTf!y=G>M93& zeKIt5c;{pU6z7rLzoGTz0zTJ?jPE*|8AfF&7}92X97rW_OV`WwGIW}ANMvs2bO(Id zA%kp=SMl?}e7vM-;fdh4$rFEoSzz@=C17>Ylld)ztDktx29)&efzmvz09)wxs=u50 z=)GP(G0wed4M1<6NwQ9O=e$MPb-kWzYIxmhG2uS_OIDVS%^jaCPpka#o>j5$s)#KM zE68e~h3ex@#=5B4PS&0&a6ahpWzOdENEW&uTRUg!5hFXfJi!kcpqP!rm2b|*Kq~Re z$P8?f6*;7v&zpJ@Z#CcGJ%c!I!)IrdQ0i$yv{r(^J`(g04Ne7W7Lw7LE?p;w)5skw z?IeDEZ7aTT`0XijNY)9THE2#hg-+&Ib8Y)x3-omslxjwn^1XRZKX3nSzuyItJS@{D zM>Ygw)ikJ)hK5=p(Z;ert52p%&{y5f?sk?I2107&4a8)wHm7=6k(fIa$$xgla&Nh_%~c=(98P-17g%VBvyWZGH>>g zBs~UxbmH)7PunstM|+t+!tJ~_$}G}oqAMgk3wD}ZFu0WijpQ+)qZ?i{BC!$9Ei$-H z+xod1l;uHdN0s^!u^Sow&FlsNcVHTZ>6%h$kxeQ|^+F|`t>fJb!RPToP|k|QS5|zq z*-))utx9%lq8~Rj0oZGMk@uI@O!l7A_RN_xzxZKF5nhG53lB>zdCJ61m5|>JE>sX4 z&ND))od4(^zQwv2H{WJ7EW%B*NQ!vCO?lfAfYYU`%Ix;s>;@$>^_Lg)cYR~nVg+fJ z3&g)cn;VLQDw&7WN*2%P#&A>;ygnXzY77c1;N3&>!7vbq;3Nt_L5^jvqC&*#@q`Jjk67`|PVQm{SkpvQq(zCqP=@SW#yX}vp^nOb`n8++H6JeVU!*NuLXn5efx zH=>|~81gG{2;`)B?6k{+Y4q=VbwzlpB)RFJRa@y3GHGV)RK?u^TlGkQ_D`T&Dxezx zS5NbkhJ!1z(x14Uqc30>RMWN1KXSimt zE-Om=!p*@2T!C-s7$Pyv8o^@LBv8A8gdCm`PGNEqkmk!YdxUgwlls*y=>Ynf<&}3a zvUXqZ1Dyi2GX%=@O;Qz(@{9#m&L)%-kfiLipo~LPDi^M=r+2CJtDG2RrotC4IyKnE zU&e>P9q7z(3B>i2XbAtbZwtqEq7B`*Xj0nfE+J1=+1lmeNlu^*y>YS#CoOZuj!X|y z1q0#%kCP$U!&|xajbyMwaPQFoSo_G7F!j>b>+~b{sR$h3mS66}0}ibb*s1GV3_wysWv~Vh5{y$MoV?MymdS zfEMLyImNogz**c$quIlM{GdU-*85sH0W@rQj#(j|i2?Xci>d?ZxAOD5MqwOF!9~ut zJ=kb=%mzu!9p3%6MXW`=$Qx!B8CRpq>7OJ!$?O-nvnivJkZG*5$Iadnpk-z7D?9|q z05VYq39htduPd)a@ltzY>m1jUTE5(4=SLAd` zO65uPtag1Swxf@$uVb(T(8%*N`?{Mkv)s>fI(CkTm1PiIpMRs%v9_O4D?j-8wZ$tT*x&o-mtrFFxhk-6-Q{tNP0u^pDQ6F?`|iEd9LR0wQ@<^g<@G6ig|j^+ zug&F0Xrg@tfTjSk6qB|#4u`=GP0Gu-r+E`-aBg#@Kp7R!##Gp4K@EYN8TISzTmASk z#bEv78W8L88rvIpB?UCs&73^caY17#RT@93>@Zw7Ch%3Z=K0f{DV%50dSp*>?qf56lVmT*5JGyVTqkOhq=1(arz~qGxrr}n~ebg@v+Sn zOfQ+3eV>Qg3euII$?GYWPTJCzcF!$owM+wWRK*`S%AU%%1-BO9 z-uB9ah*eFYPLsS{X&?bu40WO1i6-@8YN?cU?Q%q-SdNaX0*g4ZwWr)f3gS0dJHk{xYbqtpKb^#qBMC8>_cG^-0Kn-7~@+g{;SL`fVn zQszC>CornRBb7pVxH`B@+L^y?VR^!hq4^Y#;z_RrzC5K-4;_$Y1s!SQLgSZN9D!`} zO9?D&TX(%WncD!oD+U;hc%iIQ1HvS_#4C0AnKk?Q``bU1&3wYjsM%lyrEN$$pd=JT zE`k0Sy~H(qiYG7nlT{=#?-`%;td%Jg+(tZr+c0P!8{8e|i-TT?H!ZjZanzQ?Neg`@ z0yq(Y`VjS7{Gq^7PAj1&dzP>$376C*Qt95$+g(!smdL z)%rum{b>m%cBa}iX>&Hez`u(1{B3TI#k3K2^BHhXpeBWAZnA>XiC&NZ*@QI%BT`8sf8jgaWJMjGQ>Zrhpd1SV$&jYy@z?lQd0 zJ$vQZR+d_Nb5D)F59r4w!yiB{m}6Ya?v5DUIy)Nms$TB<&K1Z3Lznql8iSnv%92DabRqYG$~M z#ZK?PniKp+Uaq@)@6zKvd?zXUASnCBlh$j3#^}cTATR+TVq5TL9>1WfVY(`IWk!_pAlcBAS=ACkqE{R(QL##1~5o81xqS zV=FarwF^`~9!dWq$ zAHQ9l_Ozwip+vPKzR$U< zA^T3jV95n+bx3$@*JOQy6N2fif3UgDG7yMgHHuhO$-4=orNpare-uy>I;*RrPv9EL zJ)y4p6*W?BCVGzPRN4#rN}8tuI+G$R3h|8jCAxMULVmWWpZq+7P}iz8VxtrPCj`i~ zM4_MdhIbUknAM|{Bwz++Ty(W1=euSAa6QV@gqxzd->{y$cqVA~$CJfN(DMU1Kj2!E z(OnhFcj}yj7t(+Xww&PaI<59{znv9*LyuoX2w7Z8m3L}k_TBmWp^LotG)<#+eB43gT@M0timtbezA7#-`?V!q-g<+DwbR1*_IC`r)4q>oG z6m9*%>_o+643&wBl^loBC6pT*tX*o|xR=fpc zvqB!cI6otueV;v9IG*DK|2R6|wi-nM9ubnOi;BoA$hM2_HDH1KmaYkwZFksexLr66 zWp^Dv$(+Afs!+sgx~)%u;w3BbHS;80;GemI28Q>mUZ0Ml&U)vcf{{fSvMxxx>EhwT zXP)}Yysu}E<`;h=7(NX*0W>^%e*vlE=_)> zvRqI9Q__E1Q=r|MXtVyj8jdsq#2PK@o_AlVBNJ?VcM%xulOxt-i?yle;Jm{kRX|}a zYQA1(LHeMFu?1oW40secz@2w*vJd22~Th@e4V*GYVV+<<$=Boyh|* z6CSg^_wKWD#yWie#4%wm7(tgQ+FHiq8_EszlN zuf9Q9Xr^B1tZgVwS9yHArn%$!^T>M;i>P|Z)Ar{PBum9x!>`Y#jtR}f=2re1DWl$H zGekQdv~+(+P5&OlF&bM+RJVl_CCruKXXSo+YW@f5Om%;F+9@;pvb zyMaBxF@FS$h&>A%FmDv^?~)gfGOisKE(Rkz-O14F`vhCPOH;|Aj!#vAHn_U|-)c7R zoLp*Xv?#saxT(-maD1c;t~d;2W!_)8FU2DKgPN9K`7SW+*j+R8i3yRI%OL=Vrz>Jr zv7a66e3wK-X&E)T#cs^)JVWmTB6*;KH$fjOcmbqq?;1DBQ~>Em1Lvqy4~zOe1h1># zc|C;`Z4D6*_M#yKMhdPh(B;I;5^~f<`pF_|>t7iBFUdPV+AjJPu9KjpV?Rd!)y1_*n$95j(cOe;wb#+p=gUNSfy$4 ztfXyk5m1Xkb=&Ga)X~Eg#p$^OFbl?40x)5)zc2tpT$>E2LE?Qu!Sp z8S*#y590uTEMy^``vc{s9yb~dcsM&$`&H+HclnuWhW~shO)~*Sa_<{TsE~qdZL$CS z)1N#Mc|OD>LYg^w=iCO#n%}5fL%`uv9xqd$ICyi2!QlirBD|OrTlEh~81-fcib@}q zRRQiK(@Z&~gRc@uhZ_}#g?}nKckNyBRxh~DuQ;zgoG5WB6`c}aMqVu!T(L(qm7S?# zU1Rc*p?xLpdTwdx#)K`a9I|rsW3OlESbIwUkkDkq9!TR0``)%uY2^N7r%YzG^ZRlG z`YFE;7()gw^Vp8-1aFgVMZ1;putLWby|-=q43>k?qYXwW8TKvzmH;}uAW<3Dt&LQ?TEP;%j(@*9`)A0;I_{21U=7U@#?oPL%Uc@Sehw9w)s<2Fmt-p4 zt(rEQrjr4JIV`BRfDb_n^flqo=hcfQ39k17*+s%UHSW&Np=Xg^Imhx5h2c@&o=dEP zoH%&Qb&z#V!qXPdyrc8`hxwNa**)hrzi-1Gc~`;*;+iDxPh1o+MWVfpj|M3#ju;gS zLk5RJl7v3H1Tf5(1Kmfr#9y;7jjYccwi-A6w)u_v4G7&miTl!>ws+HVX!c<71#sm2 z+dX;Cxw}*fLDNqnbg;i>QRovuV$OLCq8o5*ulx52!X2AB>PEZwHmgei%mBj36mL>& zPZaJN_8)AX8P`;0Dakx{tpx*JXq4&QMH>{eUlwp+3&qrlD89I-DbBT5;0W&*DDt7E z@&Z+IZj{%ACazmGL6D=Dm(5Gmtb9>8Y92w9xgQ76AQA`Az@$MB0;>me=TGgyd5_wt zqrDoyXZntgGwGNbXHqK@8!#|IKjjE(P3^C%506?sW0e44v(=+tX~}if@bSFq+-xO; zRpfjK`f)n!#t7r582HRm!4+?Wx2a%e5CA=Bz6WbON+SOeQ&7vvq>E$S*aUyuk;oM3 z2@Mm~cnG3~1PQAF1auEHr#^D5w+?nyc!04pl7a8Uflm`DMi^@8fohx$;B^V=&YuCC zyZ3EwO`HM6m6GYO=gHt@_O>}NDBZIlzF2U@6wy@bu4>Y|*SL9cw@`w#=dSv0x`vm< zY)0!S55)7VdF3um{q-!J=fFspKS99H{5t`SCmwohS-zoZhx}6wT}b_2)UQ92lFel} z1isSkazmqltqq#h^|kR;L!ZmWp+_Kob)3Xk?dY?ND!;lA?X)!=UjPizuIl(Bm4%I|mnP_+ViM33n`R@$0GU!_^kk^n{EkGM)GxlDzgS4@TALcbi@u#_4(pK6FrhaP=w zExYC_t<{3}>LQYMWNyO>bKc2x0<2m8GO0$u@3ne3=*(^HmkFgPr0SXIwNo93z`t7} zmzo{_6He%Hx3c)f&<7Tvm|*<2@2@ED{~DlZderNVZ=*-4o-h1_Jz_B9tc?$Doz!_} z5!sSmTyZ3Q^i$p6)0qiwHa+Uj;dK#WGRF|%ZS0!ygW>sfqm6}=p#v$wPpV$aU6q;k zvxf8E!c&Ge^UepkcM?~_&zZ%WG6kN-x%DE`1GWzN#dG>`D>XcByVb0*#HCKyJ*gHy z9BhsKZ4el7A{^leK); z##js;-+dsKYH2)5^YxW-7Lm0$1X4KC;oFY$OmuZy9=tER@sn&4?TqC%^bQBT*smV0 z=U;FmaLw(;uS{mHW>Ci*b!2+8iYeSq>;aUll8(sJvs`A96VLh?L`mw3RiPYC>}mU$ zlfGCsRN`&z{0c?tbsPdM>nPurd1!}MKl{m@^I~njhAwD9^p@vL)mr=8p}AV@{&p}e zq$}h|MmgcTax?|2b-orNuS^AvL+d-wF4t^OxUhJ_ z%*6Tf^Y{e?R#Ns)MDBIPbiZ)8#BVrb*&1t|va@ly1!DZl83V zzDI!w5YPMZB=qEAO9VH{`^WXYM5xP)GY_OU%MYE( z(LVPl+F^Lblsh#k!>dN{vpRp{=he)$KJ91iZgh8iV<%cfwt731@(ozoQYMa0go|wO zO)cFEi9@$NioP86y6sGJ*HAlLi4R$mi?PWB5H2)vheEA%yL3ifJ#4FMCi7p7&pEyh39=NNOtg?RhXa9)&V4#Eixfb{;WhB3=5^~BL-W#N03r|T|{ zYVG)url)4LqAyOeM)Oujj=h(Vy_+J|HB%no&6N|W8{FDqz;vVT`aN|!N|UN*6N~MN zD{an?V&oqYsh#K(kc4mrW|q-P1dQyN9CJZD^;eGgh!V?%Jc(X)Eh-6(z%CsiVUAnaSOyd{DbllHk$mOL)~ zzQ`vCI3n+^1-iXq@8o*}q<1tAYE?MxNC2noRMlpCFv;o5`Mg_lCtYw+ZknZl2hwJ8 zY#x`J1O$hRFwAIUWs(FvjIES&C|LWA)p+p@X~BA-9>%e(Ysxj#OIy6P3$6+8=D{L4 z&m-+&U1d`MFnI($*Ka(nf}S(h-Sp^JEu|2GVM`yC3$m-qmE!Eh6hkN!h_uob=F&DV zR9%PV>vXVdN-$$uu!aO+{;+~A{R#12{5H^GkF8M!`buc+(d>TsYAyqO z^_kbsbb-WER;0|zD2KZ<}-Z4 zscwF`S7(;ZVRNqV;`=i|8*pclS9duGP*(_$oG%sJAc9^52hBwK@OX2A? z42F9^^}ATFt>lWtl)v*&W|&wqY1j3Om-Ja;g9M;lc6;WKBpPn#0sf|ub1_VMPzS#J z&4p!!!aZHMWb>0gX15DQKB3H_oQ4ZkUSy#FvZl`e$ZF?6n17IR_j~w5L z#~OAoFm(H4T@_4Wjclvq8$O*E;OUMdJCF`NAY(TDy6`R`fAHDG9!#gS|Qg`IR`cVblZLHpBfpp$Z0w+&m z-8q%UxV&1Gj%JoauRg=Y%kMXLfDv#5<-3MYLz-W3ubozKA;n(BN1C;-M6=^~rcyvC z@&}Tf%dP9UgNtN0UH==Cqyk>Ib>3r*Ku1_D1Q!C6@1tuirElEEnoe%@-DzXY8ssc+ z13;+oy^a-yr)x?_5zyziW`PN0EthL!LI4yWCbn^UU;s3U^pV$bjU;2AGM8}d|E_Y9 zJti7n%{{)E)4Fac&VBITG+asPBCm?9^4>AJ-_d1~2TeoIG?P|AW0In46|XAkmTP72 z%t>FTbLG5G^{nLwpCB+L2PB<4d&5nN!#CwxM!I%Rw83h)+Nl{Eqh4h=d-)!58{=BI zw6otC;PaX4{f6F1rgUK2h4a?-flX2AX3PZ5Z0I^ItA{api>7ENV=X1ulwu|@b31VU zZAjGYJO=SPwV$TxRYg>o!#gC2gQWM@Z3#>t$ia_Ksqi99ktBxagUPGAh7BP->a~>9 z5S-?eWs&v5Us4h6?9Oap?Mm`<<_&eg|s0vtYLjJ zU1jCotYZ;)8gTfZLYCbA^yV_-mZ!KXFHBDpUx%&%hYloWcTnSuEt+OiV&}3RS-a$o zxQZ>g+yIOx{aVaE16>YaJl(9ZoDHV2rxs*f^BCUH z;1t*nHFB-Pmq`8dY7i_}PrYk*%MFi zv#Yl|Ud<0VHbncIx^8y8d@L0qdNcBeO1GgOjN&njY?46ksPLO&XXgGLM*%J#>3L60 z%|ZUU42MyEf#hmlDp!l&us3u9`?rd{ZR-)c$E$DLr?TMjjm#VCpOV|FjnYCN;;P2p zy-ul8vooQuDDkGYF8?aoaUBIsITB_X9QN$ldFr_yeXuo?=${ZK82fC*>_EuQyQQma z^Uo}^#NZw^>n5LPGxcEy@36INSO;I{nlPKTn_c3t^#hu#kC&y$^!aA??U^|qRJyiq zzE%ELmJviuM0>Z1!{&YgR!u*Rx2tL#U`0TV40-l-SLQ?$gtu? z!UkKK4~z}KTMJ&x5pA~e2QW@iv^(iz_mxcR9JparwSPzg%*a9>E5Afsc6?5WitQJf z#H|pSTQ#YHm%WoE~!K zz1!VvFXAsF(^MG_#F& z^*j36B-&};pYJ?vxR6?t%F3>th+$^wSa_q=@H#K(D;G`j^`icy-eCeoo zQ%B!QCf{V|`oP6$nvo^ z$Jw?t%C?#-uHLdZ58v(@ocP5v7-rAks&FsIz1P2B-%+bY-Mq=ng39ti;W^!YQfIN> z>#BjBGv5xn*eokeh{y0(BWHuhNF6P(6d9UA3Bybb6 zqn*CctzO)CF>Fd^mfv@Ez%5*B- zRI>{3%1NV^iPo3#usmC#(rqeu_Z2z(L}?+W+yy>jzbE`wN!-Bkcivod+*KRlX5MdmU+qrJ5G#8xbvj<1xi1_gemIOS)_VF2G(PP1NWAY( zIacd9aZug8odr#H)A(&iVu21f4sS5__jYHiLlM$@qec2*8^b8Ld_%A3G2>_v+ha_R zt1zhEg|)D~`%!j;-(b4E^Jb_?b)dc8X){R{&y}1cHEQ|}PNK_)+s4a_|IG=NEq)CD9 za_7+X2Hi@%L^KhIpN~Y%<^9at-87lu&1-+W;!M~XEMxfgiFJIX{#>9pe`Tcms;Td(;MDrHr~+9-Em)7-f}2( zPN+O?X{ul>$K%13>MvQQrLYEfgE#h83@LIrSjO>yHMb)O}kgHqx)<@Xlkv--U2#*K(!Tb5G!>3 zhVnFW50-7>s5fkQAswe^C)l{B!F{|llX`0Dw*0y2EAc(AonDeP^LVDKsO%0$?46S+kA~i@&@7jNsQlT(YWfMYmYdY zJRK${E85P~x~TSxXp9z=uUadKNw`Kx8j%&d`=3h!c07^2t%DF@vPMC#rVP_p1S~mR zfv^6)zYY7ocaQa@Z?qCd0B;aZt_?hcMkV7aueIXWvMU4`;r#-m@6yA1KDlxYL#Y!s z_m1mpvzR_Z$l>>LWKJ1Vr+cZvJJjcPb9LkpVsW=!-s`$LYr5qM zf5wA=xS+HxLh(ezW9lt%I1!5F)=M68PU|P&I1w!3HU|X3_C|`E5>5O<(t1c@@NMM9 zj?Iv9exUBT19hBevgE4Umf4^_o}ha1O)HTn952E>yw?lneu|%P&;9K6-~yxCgDFkT zUelFWW&WudffAJ;hk(8ghXBRsyrh6QBd)Sn+#qjk5T>GBn3S{*c;k?pT~! za9C*M`xYT`B;0q7;xcDl$~E)$2ayq(%L`q$N)qcgufMuq;< zjoZu-Gz0}k%mLWaIZ{U9D_mVTggLtP9YFH2Lblzzg?ydBtrCA&G1PW68FyG7n3cjE zp6EV5)*sH&5tKsMx?JGh|E|M^Gmx<9K=!OP>I%wmKKlbsVY-$yfoG2KHUQV~pqnaV zX1{$tDLm^9et#BhOY&HOo?FOCD`+e!KJGaX@?X}|Ze{#M_u?t!0-=$trB{)?KJP~CW zk%~eHi|+_c-xm~wmu4q3dkqTIwPr&$2{*WFyX>$WPth9LD2V)OzfU4=E4SD_PE+E# z@K-VVYMtdlutGJJogNvgX5Tm1(6A*@_jMrj#^c|PO*kJvsA&lxLOm)0PtEacAK91R zpglSZb3mOLS;X1(d>^+BNc6TSVG7fMITbs-u6g@@RkxaTO=LXlKNkfmLYM@%^jD9g z%~*uu_F#h%v$oK1%K>*FnU&ETR)cVzbGxIt)})6;_=;eIsiTQFAgH?ev2m`b+Vqtc z``_XhF)(CI$-AKPhiaoOc2`YjtySX4|3}qVhDE)#Un3TvfRZAO(kV!%a1^AaJ48B% z?m7%8N(j;=(%oGO!XPQ#Idpf&yJtqv^MCztuIqeY|8}f<-Rq9MC#40K!c+DIk&S-+ z)(|p<3wG`+x7&S!(7V)3j`uc#6AE3YKlMVU z(~I~A9~C&iys(>8k@z98Du5S3EPB#5(%dInEdgsLt`pJ2mHfLM26HG1o6r^x^(*=O zVs#8<-ot5D%Ws0X?*HD?TTNb~FaT2X_xkBU1{=me*5HvLJJ-KaUgJ9hs&B===H^5LK z5Klo7Pt(u(PlmTfANQQh+CsTzu2YK(*F8;%pu>>GflDK6Qq0=)U`rS5JsbwH}h4m(CPy`4BjFE(NSD@>r{VK zYIj6A#Z8{;?;I6xEX{0t6)pifU0vmKF+NoNYQGEt|H1l>yLJ@3t zjZdD*!p|gzcls!KBCAD3EOzU6&1D*;d6jW^kn;QSjXyM=I4VGk)}CPm8Ega3&gr$4 z=@se_@&Xywu<}x1GjJrJXKf-g#^bi0nKQ|Pvd_|Tb6V9TUh?UD%S+LP9=<#M6$5{V zB1-{=a_ZngrVAVx2v8V9%gcAO#DBB|Afz(id8A8V%*DP7CVsQfuu~Fc<@UA4=$W=H zCcX70!`a+4UeUk0Ca4}mwAm9!hFk4n|3dD63b{{Var6Nkn%HDLUHOY}A=bJ0+QJ*% zV}mZbqbCFEfwnlIZAnpq&E;p$o^uFA@ra`ETkFM|2AdYe7DBSoG zYo*;niz^omEi`EfIJ7sGAxjCvy?$E$BGwLg!MXgfX*OAm&fW70Ymd>!Yoh_bDRd@o zfofzz5)Exf2%PX(L~ZdF1&D;(AmzyH#OG=^me9_LR7LWl1{SO970G5`3l78IIi>xk z%0%6k9et||EEof<1OHgV)imO)1Ki0f07dj=rf3w_^V`^bG*T8MSW7P;pF41A;s$;S zrg@Xusb$pZ%F*|D_qd4{L5eT=^LzLL;o0$qf7U}HK!Bd#=mpuZ;E~l}S7!@_&z=94%#3}bp)0OzL4?0Wx>z1vEPp10 zAm}+evN~`q=UN!}jl4FE4`F!{;PQkrefi)+R;O3^bOBaUGE^ytez@{?z3zP#)YCri zVnY47n1~=jD6G$V&6iAo$V31yl;`{>FlyDwiy4rS$nr0kD&bCaIzP{BkrAuxV)LuN zx~OY5YC1i46HT0RO(JKPlPBXY`!kl{KGqzfBCAv4x2P zHz`5AkNVwYX5%`_ttFU+M$JFsNci*Db@&XoH1zJm`CP2taX8UgF<<_k>J@aZ$2+Yqf~PV`*mcN6*K=Q_2?`F9*a?W|j9o!=-Vovhy3 z)vqF0Av9s;6a0%a%Vgz*4Gy{b7Ua&0p8Tmw;Y6?{fv}nD>7Acb$BVlU1mM{jmQ`ba zoYS}}0Dh1xX}BS5K074jJ`OK(Z(dGL;=@)qHLA;X){goF!Fy2r(Y3)gUdtZMwVO@E zp;!8pzu?pxAtUpBAc0c--)L{ZXp(ZI;cw32X2#}lrAbdA0N@Hc^SD8-nd9wD!NeP< zBW{}$&A&vQoln~2s+J(f%2b{)nt!L$_m~z5w%J*!)3{AFBv33(=eUK=u#1OiAtj)&^W8f1fnlw{*WTF&C=`eKI| z*-R*|4{bT5+5-W)xb5rT2b?)#yXmxD14>JlE>_(Wg}5-w10t?J?x{Rs5~_g^@0r*d z#<2NOVpt)<=XMJr2!RaNvO8xDv9egENp^0rY#w%LgpJ+0%}dU7u_Bon!_)g2ux1wp z>cD^73^U#kQ$B*dAtfbAv{-r_BCpQqIa&}vjr3-Vihf7owc(i=-IO(R*Y|rpcjxLi zj#zich1bF~Q6u;|M~oQr;3fzdK*>KD9gnsfQ~+fqHR<^T179AXo~L1TW0VYV?dXBh zGC!x)(*OPp2OI@?v3UPp+fAb<|E3}N3_%DXMjtr??fOBGb5=LQB#U8fW0JG)L!MlH z-YB%OHMFZWtqdeSufreA3W8@v`AWPX_rpT&^Eh?q#4u-6pDI9fz6Nk9#oC>(kQ?fb zoCe=mU{&FJEE%Kuho8q^=*0+$yj$}%cC;m14h^|K3=H=QGATq5fSK|{7K2Oyi*V`P zDO{<(5;yi4>Qi6I^Oezkm&tIrI!VKjK-}A^u4#2Tgn}2#Y8JAU4O#V;C1oPrl_3P9W zg-LTjH=DIia6%XiQiInR%R&A7Q(^#?t(7CA(7QG&N)Yr^0vVXUrO&Ke@~f+6C=xy; zI=JDHqM8d)pLAR-Z&NO5Q}$Ly%-*-Y!}9CzyEWh6l7C7Y@3<=rJlYe;ejA^>QX4To z-7T%pk&1?E?@wmNqkw{f`gMBeFgFSr;7dx7jGazn(} z3Vy-mW9aBZ;|0FHJmfn9zVhffDI!aH6XHoPauG!(>#EZn@cnJN;#{Vsv#A?#9eiCZ zCC+~`+t~9uHxmWt_IPgo>Eo76T(jm9z9V4c&aIHINdLk-@5(1q&l(yc?e=zJ`kc*B z+~AG<112&anQ*TVT5}naneK)VV_lSY;e68TC$5X-2OX!B#KIencT;N6VQhUyL3#C> zOtkQm!-Vrhw;`NaKjXpoZ^l9@v3jHyCYYSIRP^VX+{im}XkyU|{EB=wXTEM^pWqR| zqAbYdpy9wkgXWih@YO~^ycuOjA&V4Hv}jo@!el^uno|vaCjx!O(NqbKb*~G1er@6W z!3vcO5D>xH6tF`8ZD9zR#oC9(js&=?kjOmHMmkXbJw#K=4!#d}=QM=+uco!IiH z=0M;pV;H#Es0i{X1YwPAjmEc5O_1vIx_9Bdld-*4tPNZ59J3keLfsw@Cq##ez1hI! zdeN+P4*}&u#$`r$GeYgH(!RbCeQ3fB#j^F$=L1m@kxP+DAeBR|Vdh7(@T=Jt0F|#c(`8MgF7L) z-9BIF1RMFBOX4p}QappQtOD)G{Yz0ZtW@ia#gHOOQW9BROjv^3+XkH)rcfBXWhU4t zNr{J^RX9GHN=~;0O-&k$&<5yjmaTC+Pr0$Tfd+?%ptz9{D#Jf@nFqgeiF=70*q~xj zSdq^dE0uNgW11!KML@U=bw(+X=?CMA5(e!^-s_J@GhYRd0vI5a2fdXWETObg+uU3= zsg;WK&W0#*l<)Hx1n6!NwGqjJV3i-!a*TTcDuf~XiezR3s>0;(eg47DqC)3 zqi5V^3Qym(whJde`ISr4RwCsqZi*mLcN9?7RHtLk$im=60~k>Cu(a`w{x*a_RLY8( z34s$B>m(L59Y!OfY9$PVp@xMKpC>|2*PjtN&z^o8Y*?SEv%%0HmfigvxOoLc7&WE_ zq1(2iYde8>D)~ovCV(zVEcD}~&gDB({$ZS>)7xbG3D zL$2@Q4klXvO14o&Zaj?=BNQFYcCpRFvBi7LNo<__VLwrJNM|aIh=C+V9w+?H}2>)JQx1E5IRVYfkvEgn$dRnS<0*lch=n;U^ zvidX~66^9bkews+k|zI8B~RAJ407@sg#>EDVmHDh|2!(KLHH5_Bq#jD8T{R2G4XgV z=ghHee|ED%tlqR`RBSeE$8VumJ_r*!bFDs)yypHy-l#w4i|@XdlhyK5oidSv@?_8} z)nyr)x~&?uQ(Cym>eWs5A{#GZJnT*TNxA zt>^QMTz|H_{2SnmJ!V#I5g`CGOJvXy%u!aq;MP%nfQSVV^u@hBk`Wr<K)Ea%t22*uIkGQ6D;r*p~tK8P#H=2X9i4LkQ zZj10Bx1OqHm|*r1h})ZvV0}Z##DUcZBS27A59oA=NZr})Cyg{(PZxO@tXa-2O>7a+ z-6CSum9%l)_K9sQeC{QIfKiF<%mL>n3B<@VX^k_lxE>s@Gj!fLZ-z8X;n*5wgRP4q ziW>2Wu~=N>J-!!wk6>FUZe_*PWLNj52U^)gL`dW{V~ifwg4Oxd#L{K$D#skT3-n2t zw{o8^?e3w`OnhTuIUbYFh?6(p<&>qoZ)=rG*fB;BALTR|LT!DSccl`!&>)NU z(l*JO+KDf|`qsI05l^3dVRKUDLivF1R&=*j_ReQXO53v<_S3@|41}EPNWGsRr@^!N zyP;@sLR5u&2)T&w(x6{3dIIF&q%Ij%Fs(el;6Pju^N6C6>FTFJRKM@2M#mQ+9slt#gv}nZ#fSXxjS|A%BXal7s6gm6x zC4ayjQ2I>IxVR?>2%ZlFqo?$TS5M}WdwpL2@Qre#&bhXJbiP3x?>!B2ZY{u5F){~z z)*Emfh0+eLjH_EjwrqyNCX9xW+~XUi^j^%++@8isSkOv<8Xw5H5un5}3j^RNLHbZq zYi6&KZ2bH#bDoLsPh{q;43KS_xjMDmKF2#^khe2{&%Z6)Ow|c!)mS028Gjssb(}aC9N~p zd7q2k=wy$j^Q$SxyTkWKN%fcAyXaW-(zNwm63CCHT_<%-ko{q38kjG!2tpI&HMmh> zj~!KL8gk;E3^o+OaC>7NH%2puv)SMNjYmzg7r3`?sKx0oj9wJ8iEZIat z;dZbaotz+0oGG7A@|#@$IM)PUYhYZ7LX&2>?|Xto zFgbujb2s>|Niv9l3*RFm;7UForo}bQ6(!hftUb@m(JBIL4Q~Cb&S8n}9DK`H(ISPd zBVvA3X1Wz2VGUfNB3a0x(TXRB2{v5Y44F{M?0iRM*C9 zuJQ5j?aVMeF3t_T=$@&#n7q&Gr4wxQ-AcT3{PH+!>CY3d(G#`)^lhRqS*j|Wvp8TMajpf(AUhED= zE6;G!T@GU1c2y>FUR*m77&CfhV5p?tSUr}5$2AhB5yti>mmA#~k4`ZFeBZ}PQ6!60 zn52NCYlEXKedG0uc!KO3X6hhqZ>v#fWJ%;~JZkmbUo~p>wuLn6q*!8T?f@!cF5fUf z<{R=LvfTS1U9h)>uKIal*m&9) z=w-7Hza+Lp;Z+mh^|jXY35qu3&&a{StGDjDdY))Q@|8O_q;JF%q~9=e0ac&eVi4uB zY)G><24da*F_|m9LzA?I$`G-{G~dPok&k4HYz()7;ZbWJHxDP@cvmE-RxpOLw+5vI zOFW{1E7{TAHbaYroXsi{IX{8=TbSGoE$V|R(D%ZXiT}(epK%33&QZZp6cfSv zkN_9KL!=>ONcIiao9Zef7KEOd^0$XD3NfnUGaDh+(B*Pkj1 zOT{-f1pyn2xd0s4v)@2$y>8HXouuf^8>;-}nW%i-95*69#JaKb1d))p+i1DlTo&eK zTH#{+Jd}b9on1-UpFhLH!ABMaGnO3jJBWn_frSYRKX2Vh{2SLjyAsiT$cv2YmO5W+ z3yPZ_T4TM~65d^KY4JIoByRqxe^Io0-2jL6V0aYP(Y?$2%4}tma&6b!=E_nsf1uFS$4810xkJ7grG?*N(IVjueFQD#$i{}T zjF5q#>T~#tTC?ZcMMdN4_xlTDHIdH0uWuPS91~H+jKQ-UiO%w7a?yGIlm2k`=)aGg1xMmG~;Akn~RiVDyf*+ zrcmv}?J?5zikGO2y>EunsF;*b+h zi=G_ntKPD=ZseF_Nfjru1Vwl#$Yq%rs^6KnEH-$sBA<|1K-==uM zXoN6VE(Gxq!*&q{%g4iL+?cTyU{0dYo(B!YG{qz=)zH2g6YXraaRjXbVX4;KwVSG{ z-x$|W2|^YUAP7SdglJe0N1q`t1^EOE3YzCrAo*xG;c-t>Eb;w+a=*W=%$w*n_JLW`dZ^bZ}*H z(o3JnIfF>Q3ik^l4?sBnp3UQ9f~G_z`ql(QM2p_6@uG6`u_SR&xL}6AV>8 z2!;|M><_;0#3<`?i%RF^1Oo2C2{ut0Um8LLP^?mJC6_>!WgKR%4MN^2h!7!Vqd;h+ zN_w>;{nM~$X5`+hfoYIDd!XM^*M(vIE%TPlw_H>j<@;OkGafHMw8uPu@(RIbiQi~i zv%t3pEMj*C54&~y1F+210&5AJ+n!myVyFd2#R?h7!jpOX^^F3`(xS^>e*IkvE007b zQulY~Qb5)wxI8G5U0BjZ+&%*w6T+IR!To%Ir44}@J6$z|2dJGkdc{y}5sUe|^j1s} zRU%4{u{6i?2&m~|G1eoqyF!m(i@AMzl{)8lUQ+QA;nQsZqYQ{~vLB_Z3$SE2Z6ti>p~xln2;`! zR;69K{kPJdAgF>2eNy}f&|yx3-fP0IVi_(J^-yJ$t~PF7ahl;|(L@8{fWN?Y?4#cJ_$nUUKQPHj29;p)Vgz5{ZR zt3smR@HXc_8sAW2Q5s6cdv(B0X|}emK!BW%edq6py`&$UnFO++QLM0fRqcZ@#3|o4 z<`2zuTU3e7-B6ZKaMdO2Ws=ynz7o|V9Cox)!>~__+X@hrG3_Nm4J0kok(IH?HQ(l5 z({HTC@Idz0iJ95oa zzul6e@p}Y|ur!gU>1s%GJn0RW_(Md0ByIgvJ4;dHt0YulO=KlUO0l19o`r!3T$fHG z4z_31bLQ7Tmxm#6KUgmcFD5gHxepJGg0{iA4Ow4h$Qmctf_)FuOht2~*YP&bIrz%} z-DY)2rOfJ~!+S{ZT@0}k{5^+hko_CROoap9i;rJYx#*3Y56snb$=j~+i$n9zupK}J zV>NzCb3EfsnfMKrCi7FX;^^yyzOq*nva#35}VeL#Ks<#cYH*4l5Mv_QSN?3lXDrR@k=pZ#nk3 z;G$`QP^g^Qrq-cy;*bw#xq4$Nq!z^?U6yjBl;_#2SycU;CKYatOT_c8YB|A%h-7(u zXgAgiwhWHBZkI`rl3QDiZVCbe8%r#9{bzciM0?sV{2Hx@4T_xQcmHy0 z1WrDS5abHpW_a(1Fnx!_?NuF0<;=IE2OqtH#n@QEE1CfQdG_$h5;uXtq-V7R5u_pK zLk;|shX)?f)1nlekkjLyLvPes7!1erhxk7?X4mjt}Lsp)q$Y`I>c_O z7EE+$Ta^=*%npxo9B1F|p0`0Dg{@2?aIrzq@XfR+7A5Lj9W2ff=J1xu=17o`AzTZX zLo|QSX>V(H3ZwE>V&yUuU?MGS{Km^eb*OOBRh-?r+{it`2u;SxB=KpINP(tP2@+1m z>4c-7-92PmXDjH}+Q}PMT_~aU9kQy(XQYD35DsPWI0@uSR?YQXT^Z7e@YIlvEj~8Y ze|G6QkV6W^`c?iZu)e4;hFRDo=zi{!T?#G;Un$)C*1Otg`@*k9rSb%5QCo`tkm3FS z`2>*bU!@hPZi3Cybx7PltUkSI+z}Q!qDO=zEKXa-X2e1NDh(0!GFEeuixKC{fGE`R z2Gxm-Z1WEpSkY$Ln!(R`2S6i<#Vq{aNY>NXHu+zr{+&FBbHb~$E1kE{9uoTNDP zo$9L?VR>iJ4xURwBeCZyLh%v)9aG=v)g2f-GkM>Q#Lfcc9AMxrs^2S08tRx>3u^$= z$-q|h8I?t>vUPGW-i@Px6QN3?d&EZq)~nt1faEi5cHFP6zs=z)dC^lZcelZR9pycD zXwaeTD$3|sCXH;+)tPIpxKeA-`4BBgH}D5IEFC+gupH%*Eu(yd@3eq>vJ474wt)4~ zcgv)dp~Arv@Z%D_h;j`!kSi>Lq?8Lii*gX5Je4u0wLEXNhwb@M82CMSOiwL2>o*|w zG7UPboHF^PS~$lwB0QYn)uNH;IGn)qZd9H}@i~~38e&qeg^OLl>!f^4=w(uNHm<%& zWNkx>w69f$j%7yO zNwF0I9^r@*&e>O8^S!NE_87J6zj8$P9W`WF$uDkpq1Rb(D9v2@f|`;ECPA`>9415TpcjHX+&*#|!$3fi zDX9NK?Hwc1>RDD+uIS^Mia@4SyNiICdfGPAp&-6Rw+LWAl=nRSBr&JDhIe^jcqEn= z#q55~_#toW$=+mIf$oQE#fR;mSWr@Wd|3ZNS#zcp)aknknA3|OJ%no_ zZX_uQQI`J<$+cJTCgG?70APQXFz$?Qp=$8Fy9v{d2l2(lv#)2ZB;l zR5ORFyq6b)zvp&J#M-G1OF)CRN6;$Uc`j7gO5JM-O7rK1O%!UAy5JDJ;m0~yL9l{}-is07CR4tHWD{Vt5*_8Gh}I`iDJXo9UrhtWG-Z{{E*=ut9J>rldUv~O zE@Cp^hoFRa-LKDbLv3zRymv`u2-!3G{OgZV7$5&4Hir(bkM#E!wJ^QtlivgsG%lnc zS=$ZBh1t|-@$E~4R>Egs+Kdv=k2wQ9_j(<^5l#YE+&Qqd1z<015<(j>f;PUw>DgPNa6S|p^-x`i0|vCde(Sm{(W>fknp;zSEy?%5B0ur~$wV&b`p7>l;Ua9H5S6x>Cq z)6xU4&*dQGi>p939zoaie^fuvp)rynLEX8%b1B)Zd+eFHDQ?fGGU}mOv$`zjdH4bh z{jMcLG^$FM0*NrPg7?Q^x{!gSfhqk;oTQMRAeg;KgCOzF74k!93}2n3e@6@!%*k|! z{TQ5I;qMa#+D>V9HfeWGQ@X;_zEXvbfA?qP40ceH{Zr3VA7O&Vi&_03khm{f+>Sa9 zpF|XgM0Rq{qc4{+@GFayd+Mlc>n6vls|w$*X(9-7=JQKOnoYsXZS=Nkv<>~#ysV3gmtq>&KZps>8>{JS*k1+%B^ z#Z~>LvrB!|=7-~IxJmf!+$CVah{iPC)D-S`@2WkKUZ6!P2=h^ymEw+D{zRz&9|GDd z!>{@#IY;_UcIy?A__tLu7fV2o;mg^VYTs_o>1@TFh`WO7PECFi2X)>Y;^s?HoruMe z#(UQy4Qwu-_wzz2>)dAVI4<*z_*1q9p{2c=C`Y_05!Bq8*9Ku&aHwJD`#i@1yBh>Z zju&tssH|2o4E2xwP7|c&LZr5L7Mq0d8ZXq9=h|n4YiUw{)Cipp4$LhHZedk59@Zkt zWi>fzU;>K^WqR==y6hP^leN@K}ut{++4hYbS7VG^Y??>Epnsb3Cz~g3}yA z-sm#~bb^Tl)NNH|R2q>9$O_P&*;8n7Gfixq1t_(jTz6}%>mFWP3>JMf8!#_CUl=}P z>=`t%V9ItnF@3xKk3+-)&jDjTudOCi!vuQd#`5JuV=aO8%IjE~@(<+!yC2X4^hJ*f zy0%>O#G$MJjPAOlZgXykXiQ%*TQN4g2H7|@bvWo$XL2X5MJ)8#%du5I^3@$&!i9!! z>whvJn10O}2!=N?MYxXT!*xXL6Wtal!=@g<6W(kyGHlbU;>!VDC1P4nZ3fpNOk*6T z9oO32B%sX!UCRreE=dibP_@DN@gm#RHnvuM$?0_Z5*VGN`l=~$Do6<>Ub8i!I91UT z9V(NK0FHtNdh|CE8&5mWZ~IrBkbO;Kt;g#8zd^ za}4gsCE^SaQma%G0P$3o+ALgDzi_*PzihwUuG6UqG#~3_)nCxt``{W7@rMez&SpjUX%8YHOtuYauWk)1*4ej|PIac^}p%k;SQ1Yzo| z_w{sd{zNZ-tnKq;aN1q28q~#M*!Vs1*n~?Z%5y-7hpAHg6HP0mtRB~ z3RnDu`_Vwn1tbJ}h4l9oqH>%N2St!N8_iT-`Q=74{-!CwZ*#!ADWD*O?H=7cw9n|Y zs;p}Bn(ay4x@rJOQYIOTp>bqn;jUFhf%Z17Vzp;31mW-BITj)Xu}ZJ{mmF?O-jqxX z>#r`RmNMGxroKC_dk>fh=$_&Z$JBwL%|8spj~z^BW*-+bZ-~8^exi1XDtd@PFbUWq z+xlVLQD3F9oDhZ}lXxi(5=E-}^4?>&gMZ(d-fPppznlbdALyMZH*6`rgAqpGnJR85 zZtsw$o4B}f40xAo-9+xW?^`?u zomYTb%F~BlM12*r0OYE@#J9W-aVX;Cw%w#7$C~(&smvRwZCl&8syH_~ciwU?a`Az1 zW`-pY3yS$G(x25-pDWEeFCOki**zWe@ zf&g|vdMFW8k(k939xzz}I(AboUfifdwNo21-C}Y&4hha7;ly=HiyNCaMap3dADx&O-+bC*I(`y?zrWE&!qFc1PedHh<@AE z?IG@ZG(s*qe`KFtKLjB=fKLf9xxV}@oyn@fR&Sph@{!=8iwtr?U&X+qL26hA=nB#U zy}Tg2*7F@OHcd;QMGmU?03FJV(1f+dlZAK$KgHlTm+_OfS0(yI73siRgLC_#uVcf} z$3W=v2G@FUgXo-s9d_>Ril~6fA5}0+iR5`3qgKZUR`&uF&Ar526locJd-yE&6SqOE zNB?E*r~Fk(8KIv!!s^1aw8!0R-sTLTgMeuqZ;$*zr?Ql7IaOBobj?-!pZ+=U#`Dks zG3=S~SF|HGgSfI8l}vnXk6{@?zvXfV;AtM%XMrS9C4&U@OqS!4>ox9zu_RbsSX=j% ze+kb1zLvX){(ApbJtZzKMr^y1X7Zsd8n#+w4?&_PsqW^xdr|Z5mQ>N>&6G>Im z!~V|5J}|=2D;Lz6Q7wa2y+LM_b&Bu^#1O3F1me#~Fi3@R_D-RP$P-y9h;1dWE!!FO zG^l(!$F06Kb4+4eJbCbHzY?JZ;Sgo^@2ipMZC>6VkrSi327_vXG|_Vc(>UgFZR^6q zr@K1WI+J87nZ$x_uueihxJ8WO+|;q$^FxOKOlpU~sIMG*^;Xsm4|_V(`o7gu?)>=$ z-BWtS*c-sEEhd4_-t%S(Iu#9&*OdX$ov1za)@VLPqM~2`RsyvwM%?DGzohJ54+Y`1d~0_@&lA+z?%l%T`u4C~dWtMG=MF!wP;bpN zC9h-@X8gbwg1q+D7GiXN^Hh}q(lu$@&#Ce;lbKX;@WeO<4XaC+SM~r2%EvlB1}NqF zE=9~R5Y%CCpq@VAcYh55-V)yT?Cu5=vK&!N<)B_@69Wkrr9d6&?H2+L_19@`DZao0 zZ>I8*uUmR&pyt#B=9Gy5f>f;@ZF|jWErB{-9>63ZiGw{* z0nk2{$okxxN5ysD+9eP~3=ThfWm21WfL^P*9$!~?^f8ry{;gjUUjQ;|&uG{caehAC z8_N7rd(a!L4K{x5(A**jr2^gKbet-Whc!Gw=f(qIwu$c8Elgl4{@vxaqSdy`O={@~_gG z2KePZ#J_M+a7$J#K2a)Z%H3a)`qaq?{_k4NhBBR*Zn8KOSJun=#T2Fo`)Lp`d7lbU zXn9FeDyks;oimpSRKb@LI&Kwo%4FN^-8O`NwR`KY`dwEBBA2u5tCj3N=Ji}%#K?TdT^%xoz z*Au@p|3;hT+sknpSj#~Qo-z;h%v3PdR8=uGUaG^6(f-13j80MG0kFl*FWCDo46i$P zFBNlUTmRmYT?cT<>0_85CdQha1&9S*Hesnlb zE^1?+>UE#_M{YLLkL%v)OEH+!P*CFxj$imKnvCC1vCFxre`iiID4l&(>{AV~eX$+o z;uhht%4|1bQID0^%{sd0en3cc&m-TSnApCKrtU(wYVgTy#zBL>u|`INjlFPf#z4UD ze>&1}Y95%*%LS7RNYChbq0gZ~q0D&y73 z4qES4LB0usz+{A^c&e!s(?7wEs|DpRyrwfnU3x-*d4o)e2IXqN zIMp8m#1-PwoBe5W9&9mNE5=;Q`UpvbSEdZ<@p)Q5tMDoNylvUmnQ)4#-HOj+G;>`Y z+x8Bl%9l4`G+Ko}x^F|;xgPeA=0=Q-x>LiuXz{=EBYP>H3bU_RD@Tk=`Uosya#paZ zus-M-`ebShOH%WL17Zw`h2Q`*oX52Po8^Z9 z@!pE3ggo!>knxCATjzTuxnF#};#*n&k>WP1XMv_h=)`HLtnwO)>pfy)->0n?lU3}u z!Sv9-{d4tgV(i#4E))ZUD-=dj3|(lEDx+CdB;RgG{wfV372fcUnNEJ9(jHtLftm14 zipg`w!zphD(kaKQG`llg0}Fc?aESff{p@1^^?ZxNX&$0d8Wi1(UQ4dy}#S&=(>LKS<}C`L@6z+DS_i3 z|1r$D(z!zuMpV@cY~nAfR1S%!)H1fWFMBb;JVR_1gGAFsmplbmxMq?w7a(txeg()G z(h`b*G|bxQ#RqJkLf&f&EOIgV&nHB+Kdg=Vx3_8W>@p$?bi2ElP?G38)y^}xT0=K}$N_(~i|HHN zgac9DyhRdv+?9g4AE}2pHl!0~t7X&;tY=Em%VAwgsa!o3i0?&U8BerV=&I+e7uLjM zq1h(+>|y#p=u!lGEcQtJ&C^?78&UEGpujm?F&0vz<)r0vz}Rcz*%|S#hl|>QTK`1x z^xm21YiD`O2*bynmCsRFWBw4Sf1E?c&XSnUY5FA(?EEGj;rL2-6ZlL~()k=NhPYjq z(kS&n&|_LWsn0ibZO<`KtPNIWQ~l)um3rQAw-twRh0s2!(#Gs`lEYcr+scn!61DYO z@>Jbb7j>OKTEe}Ua-1jv*WCs4*EcmUdt*)%ZI(kl)pecsNprrFz}*4FxpH=Jw@0%m zyw)UBFQ*YW$NO$cEFZx#9jNE!#U3A5-j{4BdG7MvJ$*+1DN471IXn#u5NG){lp7I1 zhp9z-S@ZGz$ugJ)CuxRNB=fb;zY}d(ODPcs3}xdB-E?>G#uqo;o7xV&M@CM2C=*x@ z*%Ad_&~}46D)>AvKdZPR4)oz{g(;=?7pihoo#WT^a@PbufJrEs{d0UNLwbNJfrIag z-Irqcd2hjD2m$oM=$Hf}lgLlu#*y8k% zWk*Ji*M0O7jyPS9JaF?mXD;$o+CE8XgZ0P0hK@MvW)#*fSuowKcfR_`=>Izz+mxxT z)~kZ-vzRm?A!Z5qF7n^g&;ap^N1AU7`v%xQ=o&X1;yrv`@sxr`PM}zOdvIbVOSrO^ z{F0iGE6lb2!m0Y}y&mwk{a_5%E1F{$Q%#@B5>aQq!PJYNRm zhFWBvDs5`<*Q%R)-Od_Uxy;jN&LIzau_8o zMiMhVx(C?sbVG6)JgE{k-LQ`Vf4Ri>-Dk!3+=~`N{0-RaK3i0e?Ivy4HpSy{XRQk4 z8QptgP-bOYv-*nZcKy+03FU);9t1Pp3;6N44By}VzXKO|B~H!rB~KN02rF>6e1OfT zC7EFpJ>Yr&W7$1|beO!b!+0JT%sCRPXtP#Y*QI+%<-Y6Y>H09` zOPU+F2Ww|KRR4|pG$=q^!hIcVdzkD={Y4Z2k0yL@@PYIeI%TPVJt)$D_D|&j1 zb|ptLg^y_E-26426;?a|+xnggmh1Ld5ZJpp66=El7yLjcfHfgbV4R=+UV=wM!=z4q z;0InICo4?w$rQb!Ejaht3hFXh%wuM6>N=}4mjS~?ps>q0+fY^^@B$;OtDaZ>ks5Ec zwXELBuZNhfftsk_7z5`lg@Fk*lj+23_W(X;f4jdHZrUg;x9Yamrhffp=oo>RcabU# zFxM$=b!EocihK9yhvqlX{V+zP0Mx@c@`HaepJelaDaO72d%RL;V9W$fbWI8ao4yI5 zhkj2E_g-8+i)!;Sks|wxI`8F?2EJDYPXHan zQ2q0-CYCx!XEV+NOjjurI##?pe{nhQ!bmNa1$@!Fv@Fr0Yt7+_2M_&jzFTe5j4Dkx zkL#8;si<8Dj#2hc{`|B~x9Qv`cS`@nXQ<>9pZ(-o*KJH@@UGAPOioq(oJmpW@xM*y z@`o=jx8qvEaxL{w@w=qo%kuAkZ@5MM_-1Xd=Ac(EXKHo!!Y@dBClz5Z7eSqn0Fj$p z$UkrYPUkHEp*VFJ;ab?-D}^SZ1IC<15?bG%YiWfatJZz4-i#mVf(dOb@hZKiaJRm4 zKODooOwPR;5Y4;Wq^_RTllZ2gs5__Xa*s3`Z^v`!{EU>*^IcEb%q7K(_wujx;q-Dd zTmxP9#Vw6r)_*hzGOKJWuD#Ur`E=4>FaqPsUvGO#i>g=yBJz}C_$M3eOtsL0YG)Z? z>BZxS{Ew9VPP@kkqICo1Gn{#Zn9!?;pD#4`VvK!mG$nbhV7z$0gI5%7%F6sOIAhVk z%k@Ln3r@rA1re*CY4FecNde;N-G#u@)?r9I<#Fi#v+8dynABpUz|HC+d^fp!0zHk? zZ)@`jLz4=0U&qLd^;<;cE2r{6p85IoC&x9ji(F7c643V8-v> zRf=DJ9+|lD%`aeWR{a3hVm@yAcc6#-aZ&68ugyyD=W8Vw^xf?eA#TKj{pWYaUy4)H zMCGY2AJE|nRWX<_|HC<*QPo<$k>6$EbanDZ8XHYXI=WDjCGfHECGzcvuBJmac4--=c*pWnIsuV2XRXlL}w9-lwQy|DYNIhAEwdIC{!dvnr+ke{*6} z&r>^~sn%0{~pYIYzE<;#Bcf%M9{7in-XLEQ{M(gi_^p!VmTL>|26j| zr?C&{i{L<7`*{=hj1+?oT47 zl(k{~=)Pb0FBN8}F@B9sudMc(E@*sACWHBb#SQ1wevYKg@QQ!vZD2Q{;rA9*&Ot+? z(gGBTX+pB#f3>XE!2KSBCf9UEg;Dctv+HJ;{cSXJhuSl$7gF+L$2XsMHY^G8C%(?L zI>6bQ;$Jq*|M-RF?_TxMq&|=b8o>CnFRN;*qiF2;Y$$buuALWsuct?R1iCK)zt2Us zfX};Vv8SeALG-b6@dz=w3t(EEDr2Kxgang61Gu{dquQPj8x^jIu^ek0E_0UR)*$!b zVSrB4yE|P?j$Wd@Dmcm-XyG)brm>pCH;naht?VXZ>+QJ2_^BDn$`8Jn2C1h14-Pn( zP`l{YEG%n2(_f`n{;76a7e0f3T0eI?ORi&NL6iDmild6*e>3LfmBMv)7Hnw%5~8eX zq`u_&cWr2uQVaq6!04u)64|I-brOHrR+O%~eU}HUtc-IRN^oC)p!x6M9zP|ZaRTdke@Az^#^G4B zu!mAqn(>A&Jo)sz=?R7n35a%%-5iA)pAz&W#O;OOpTV;h&Eeox^+hM!Y+*Xm$HfG2%ADjx@?7PsHVkul}F~A|3P^NKzZ;< z$piAUnFHq^LrES>icmasv>;s#zMAK0EQwsdS5_&HX8AO>9vSXgM>nVb{|9XMSrNmw z{3+3NZWSgCdZ#;AmZ*)3(=}0}Hzo8I8jhfH!$+!W%lf{7nWW()-p-mMcO@dOyT@CJ znbLGJF+b9;%v_x{{%06s?ckVP5MkkT@L39P_TtZGw{QN~Yh6JHFh=#`?E_&KWht~- z29pSjX4XQ5UwwWXYFANFGoj?H24S+ z>nc_DiqJ}xOqO_bG)^r6)kT`A@s}kr086%~>l6rBM@uwn*$NVLQXAFi31yQdl|a)8 z(d=-!+B2msBE|ttzox9{jI8b@12c+WTPD(_i=>#o=8&QOJ0EJWS#c)O5){0h)V6A! zWkfyrak9*7o0=oe!HA(gc9Ai2wbO7T3HQr}<*&5rm>l;naTd9H>2;7#BN0V=ro-P_)*|RyZ7c z?(D=$|L^#c0I)TFbZKb}Yo}bL@UB>gy?woqKjeqI8b(=DssFSH&6kvoB~c}-(eu4n zdKk--jkD5h5!ML-h~0J%e{NEO?3gUT))QbZwT!y|%}+v{x;Rv)Kx?hq;4m|ZiCP5v8+IgBOglQLTiAP{ z1Y5E`j&hbWjp*wW!0-!IM`xftT{Vb_|9IXMVD+t~inUWl49XblqAa7Y!4;TB&Njhv zuh0T4QpEi4v~Qa{XDt_PHRl)vHX~YtHr{EtKQlTcxAKlJRwhHL40{wEQ<@VN`3p=KVM@YIa?Bq?9B7_R{&|J z%TdV{2Lpgq#HwO{0t%RGt0rjK+ zTp`yJh!$)`ZYy$mN9sK9SQg5n>L2_1wy^NhK7_Fa(Vmkm>ynabn-wur66WQhDAT0H ztpB$uyaNpEBcxcnD0ekj|LCCJfSrkykDmy4mZKqZlRE~t+0JN!8UhuB4a{yG0n&-R z|1qyP4A$8q1?UJE0UjrfjK6&gH2jrPW34Pg7G2`5j&d`oIx2xlD=Wvw4q0jg(j4FN zq-gg?v23_lwGhMKBuU~Ks-1!av~6fQbM9r|QJ2oP!>75k4Tn%H!A*X{DTK!SFP}pH z{T^11oVy^o5%-b}<&%9AU4cAh*l&V6i)qn7m@-lK_XW<6u|KZ&!vSusr&T{hMY8ES zaQ-(v31{#_bUfW?Q{BO7#?^6qUa{Dr`Rz!I;+=uE@qG<|X@>9aH}Uy5)gP57s1ky} zYw<~riYNPNS6i4t3ngIUJo(g)az%G}c7&emT!s0P-`{Tz-~)oX03h!u6W>|yvC#E@ zZaA5y;eT1Bdpl>~=))cdlAv=0XzfqDR<2umV9dyHCxV+Xmk z^G?6k&fngDpWUfe51>*cz+ek4>P}GDu-89YnO>=5ofL>K)-9Ff|)GsASVGs6(<^DUB7onVp{iJ$Q5!l0Q!ATCeL z55VX-v8#<%!Mp&FVN==s+h>^q+#F=B0uekab>BNqi#*-GHfiBi+{F0Uv5()xJ2Z(s zKrNpC$Ht=B=>Z=t8SCE)=6m4vup;jd02MN3dE=${Z@3O%0G!~L2Y?q;?%@A4RByqe z4Dks;+8vJ<;5Q%dYNCUpr;gSPMh)hi-O?|ex13sDzOc|336W(OSklVw@#^ffjQsS+ zLH~0q`8)ux5O4;BkVC-8)ov^JH(`j&W0eA(t0Di;-Tm(2PJF+TV!VEeY&sBDOZX@EnG3l# z^Jwdx$cT@@UdQ?y$p9@?!%%191t3CXihD3LQA7IFC%_>=m-`p-`C$ zV|+$*8lPlbFn6Xr>g$3;?-j16a_^dbbL9!NAMXI$wZ>GBEiLJIwYfb~RdR)G1*JC(nZ&+s zHr#f24tN!!za0(Ga?l-cViIxy{gk;bhWrCKwJp|k^pfFFJ6)AVvp0q7yRvtwb{2EBL2Eh{pS%6IcCfN0zixl z)lYrEqJ}ya-+;mJKk=E)FX_C~q(uBrdIv=F81-^rhNeEICSvg?(+~(K;0~k}>d`h5 zv%TlobPBwNd-3m~fIw3y4;aV>sKTe6Uc;9z*{-^P;2;K!o{t9E&LeD=PpOIw31o=p z|3}*a389{mmViGQIV%4Yl6ZNw&Vmcl?F4lC`1lanr+pg^70qjSNfv(p{tP8R`9SGB zi6?knKp$Su-5fB;^g)HctMCgsYP~rxV9s%R*sOv7NCqG*eC!|)K{S9l^Te~^bCLen zKgD%OA1SNTfJ)mo-|QL7H@?i@0rD#*=E(o-1{MG+KmvN2m$#1)7G0WOS%tsr2N0_3;WFpvt?O)(r<^F zQ0k)-ZKl6!iBZ#%s6=hbM%)Qs+KKX&wLKH?JBeW30e>^NM6FVBQiEGnKAV#v?SGyS zK|S#ikwvBeoKr*(@I^U*Qxbp>=OxQ&s(>1b*=9uS6C>S5^zZQoEdkOAs2vdPBMMe6 z$%%O{4_(~N7QuJWX)3^FK8xGugjoDao4eYuKDI=Qe@Q5a3&0$pB@IQNd;Qc1t>aY{ zH6jo`8xg;AK&re$|&6o3?)sJxWdm{Xr=kkr=@ELI(V#M0NZF(b1v=;i~K* zqs}>UC)EZ{u!|>pjQ^R`5o%Ii;q|4&D!WrM5Kz}cNM?MXt$?{CI}=g7?%Y2ql|MNZ z5J3h2iFJ?_Tnt-)wq5X>v+RT`V237pij2o+y09cz#!cy;TpMVC0Ak47b%2tjr`|?8d)gXWR|``L7u|Qvmi9l?=Zx{B5`s4+=jkgDH6oQ`2J;5kzRgth=YSvKNwwRXx_K^Q^&uS=py;bZ6EAi4vR17uwB67lTQ zvAC0FL-%RnkiZ=%;&0nBT#Aygpqb6~j=F8$T+c4Q@s!S;7U4-9$TpZ4dC&p&N z{x^gyL;~pzkbv1pP>OD{k!t6y^}Q^@7|y+xPZp8`A{Qf(BYOk05E`N*!)wCRj7V6b zgtVQpQ^V<)EH~{x&S!!8&!8At%zJ`hf*OK_Yb_@%I1I1KM-FecJh1|U!I0N$2N{@j zP}Gv;c7 z_?{@?lK@DdG8YW`WG;K^I-UZEK-T&Xze*uX!6kS%|kYQ-foQ2l2$K$9;3+ZO=q6!dRK!dWlf0FMu- zk&znDBqV@@mJGownu=Vpy6g7}*n?Pov2%`t;~DTe#(zCU7wS~Dasi_^{Da7e0hBT9 znhx4A8ozdZM}UElIR#S6=!AH6d-h}FU*64a{bvcg7*P)^$s~BZj3lp##LF)&df z@a33K@{gyHR)ng=>1hs?c{l9>idT7~OVEG9;3VqM3UqTmZ;g-IYXTr z0dld`*Ayz@wb6q#PMSiDP_UCV>)Q`xnT&IQS@{K)KS25}6xDB=rDQk#7_RFCV>Ey(_U>S!9}5%r<_h{X@rXYzFrH|0SIqs3zgp{2&ifK-s_HjyL`{tX|BM`!*+-+fqp02$Xm?p9(C{VV(QQDcBq&<9 z)~-e9aMFnY>rKOH{P=j-f8F{ZUcAWNfLLIF>!-nNu$k4V=B5qa64mkBj+8AptFiJ6 zG^`*nF%`<&l)6bcWdO^i{Ga7|{SH_zITY!I`0u#7%o|_b7tbiZ-0^1s5P7E@V`U55 zMp4%=s>JnRHd`qP=R&GltjRs;R>V}y>V(hgh8?zC@5{l#F&09hu3B@hfGfHWdWd=7Ars)40~)sBu1ZbFQJuIW{?YqgPS)ua|l<5se92fk})t#-|Ewj*+E z@A3Sw;wkt~Mza6xG6{AQuDI=Y%}n^-jsqxViJ}6`8I_8r487*qo3Fd1)=`-|6NBTR zRR&o|vc4H{}WhT617i+KL{8^?(X{=wjv!|n|RPL|gL!Y5-SOC_mve<*%HE(+YSSit2O4d-Nm z2~N-;oRyA!ri-R88xe>{A7i}-1#@aTKn92|G>I?O+E5eLlHob!BnIQRg{kD~x90`U zF-7rg930x}6bYf0X@%14t~HTG;+c36(g)j+_&WjC(+BV773*c|+>T>^SNns1z;c+39C}<)?a+ zb7xx)X%MzdBbl-O@OS#gO9D7N(vPu|o;&zH`~x;Gm@Kbd#Ljm9RS(EjY7macUpWjQOS0At`L4}+WZL#2#Fk&u? zjhU{3)xZMfi-Dr)o&*j^tt9-kA#=*4t)VR4u_T^C;redC#*|YM!cz9cQ#J$u4hKM5 z5)}2SPG5_PkJoTs1!XTnfMx|IZjP#4HgHce9yADYrg2;|^J~Bt{v=HR;?u-Am?iwl zfuX0W{S7J?ANWnwr@`ID#U*AAhm{Ypcx>C`poXmQ`&si+{d!yMx{P%p60)?Qo%g+- zV_j!tS1j*X{rt3As=45nANXwoze{#l2o2;NLps1kd{M|a(}e4RFhrdeROUSHcRF$@ zCeOWi?6F!-M5(e8{6xrP{>k z$-5y>)WLEv_VU+U0c>X&`b1dU7viGXH83A?weA*OjT{su{i7dHJ%g|`{48%NAISrg zq@w5_epkSxiZ2Ru_ZJ4gfTm7-o!%DXsbE_Oef8LgF#Z=>0Ph|wo>*oLJl6R} zt;z5bj+5qlewM|ciuP->tqtOiLYF`o>G5Gc+h9)v>PGTNoI|`91RKANXF3g){&JV0 z!vSs26-?SjnZCn)Tzxf%26d9^d5G>?BV7x!=%Vx?A2vup3g=x?x?gjVEXXqU4gmA? zLPf#@X@Fao zCTbo|8~90>GN4wRPoF+9TyvQpj(S_BS>MTo?{TigUE51r(tU_!(e+b_KPo2R067nr zmM7J|xH!)U1k|2ip{O5~1K0v(n#kLV=dPEJIj!KzIxs+BV&lE-YZ!9Yy6Rn~en?z8W#t3F@e=pv1)5FowIg0?_v<}Ztp@=}>BPeTT%ejvJ zS!t@O158>F^u^~2S+r92#5g%vB=zwQx z@Wy6B&H9Jo>V58{Nn)q@^kPnbeWS!%K^$T^ghq^sW(D=Np*FtBoFdf5JQFgs zB+S!?nV^)!*NxjnE=ofg_00qR3S{cAwz!;0hrpb)Oj7N#Pmv>ud7Yj+4_eTgZz}e3 zAp(Ic9x6~_1f*HSxBb?f%N*dy&FBzUMQIHW-^NC*_E=6Hg)tKM^s*!YfH>MhGE))w zrMP(i1gWbgARaf=v$x33Ke36axu)YW_yczhQL3vK^IzJC{<#SpCvv~m=nq)_9IyIICj<9422lvJJZ!lbVCW>74ZuRWB3m4`y`zB&ya4wJ#)re59|q3h zfH8tV0^umjNB}>~_K7-bw`%zs0b6KNs7OO|9stA*ZSh~>^LqNlB2pG-97$tuanYO3KO60ljhfS<;0FlmYz2=m`iJZ}o-GRi zbSKs8@{#-5*Gp#2R@!5ei&&p67`0KN+H)zqTmYvLA<5~B_#CFd73dEPeBFnST8gse zGJ*L=an1gvSuNa0?W8&MtT3;VIvC6*`g?Cz0b3CSXW;qJ)B6>WCV3QsMKfyp>Xj(@ z6nZdh7`p?F9?cuDdzac@@+&UU{h2Z9kNTQAfCJ~qN2f8wC_aLprTjjtyy#Y$`HJF) zv1|l92$qwHOY!6^J#0(wWwm&|4_2Xk?*1+=hn++?mXpgUn<($Bd*EkG^oYTQ%WXX( zLdDp>ixE4n2sx+KyIvWwoSf5H*lX2Xb;;Oie`7S>X}tbWTKcqfAlZ|3psC)+&jzyJ z-xAn=-1cN$;4qwTuTN<1xx*6Y_&UxII*fVnr673@G{rF-_!7!6K~i{sUu}e0O0!%= zvG?dp=g6SPdr8`<3Pt0FHfh-*WA~>Hs-~#=U$kqH-nTO(PIMn?IIdp(taJ9FQOvP$ z>ZZ7HJa-U3^bc#S1CZS8#%9{!EydxYSE2%q$z@(-0n*Wr{tj;Y%I1KtVal%^{_@vd zZ%%;zDF?eRMM}i)vNa3UeY~n{m~Y`#Q2&|HBOZ9H zUbJp;eC?(eCET3tvw2(*|H1&WQxkcp;%012K~ojo^?sNHKP7oWwn9Zl1OFcSRdOTS zvlMAcT-C7<{Evyh@RE*UU;5m9g&Z7`F?PQ>E0B{v-wP*kZ%B2%#x~Dh5_7YEaUelE z>H1(exZJ@ic~h#$E4M1KakXR%&+A&Hl6t{_d~~yFit%mnD%IE87yjNFa+j1-qRPl- z(%$O5>u{ENGPiwF{mI1RBI`10TN!d%2Pr|Nb28dFbEPEiT(C3HY5!Tzwtp@R^yzF4 zQ9ZdNpg+AAzBN`%a%u~enHF8&;BwT|9>bPp=dz19Tw(KrHj~loEC-I9HXN@tRuu=c z_8a6y##PQlCM_Fzv+v*>9Oqe=P&m@3)E3{_cwg+^lwWxg#C-b<`SM;YGbmskK;6SO zx%V~$v%YE7G)WmxH8`o5`3+E3urgmDo{i2n2dp7G)w<9sDS#@y_J#gH7OdYx`8La>=NVKflAEPPC8BsrtE33+uZ#$?gLx8ZEL?wN#W* z^FQ}sU%WpXs;9I(<~%0@^f1f4vg9Xrax&C3rf*s%v-}(tBn=;rPmT8PLlXK~k>$1< zuzrK9)N~vx^$6^rHzOodHcX9M12rz<#<+*Zw#}-_*LSw)D9IND67f83}2eeKZSwQAS3nN|W_MB(#?x`WM4llWu} zer%bK<^7>)oAaxbA-w0hA1@3Y=XiqLi;Q-!=BzGGJ01ox)qL~T<8m<7-97(ka}Rm) zjj(i{WK7rG&fmKsFLIexXjCz!89 z$8Cny0~G|iJ{lH1<&S+ZAE7{$;&%{HWDBH{(xcu+B?F0m%FEU_qdT(VIghMJeZ>vE zHJoZ3>5qL1R4)$VAWNkXB9`D2O0^SaKW+g|46b0!+HO821^f<6RrYSQ=J{r!EqfiVD-!;_219cYv;2H4lvw z&8f7G*6Mc)=<+~^%;3R(Im(vrB5W&PVOZz5*p+HWt)4=C4zZi!`x=fVxIdeIh`utC ziNUmiIhkMF4$n_hsogpXy%l>7tX@S#+{iJfNrzfXB6h4nly^)k|D0jk z2dW%tFu7S5@%@%PM-V-WY!tdm-i8EN#YhrhU5sn^Q8 z4)HmLU*~6{CU7M;@beZdZSzgXP(Q9kB|agPoWbF>Cg%XAO$MCpI}38#YWtR>V8x1| za6XktH|WuG}FG7)yU=f1>pc_`6-svP0s|E?P|W5%2|f7?Ec)1-*es=o+qwUp#Y z@7U+(O@Vx^P)E_D&rMvoWybQ>Pv_W2SAHSYG)+D`a#Cau24!4f8!7q%`#Ax}HrNyN z=y(iQjC~V};+(i#6;4XOeSCYhu_`&uD~RNUy$T$OKS3d0t>tXI*R*{eTR`m68%v++Ql+x9iZP!W)-&&6L z*=i-|FQFO52ufE|&&=MOpCU=P>RS8Dj-C99#%=WuvG)~D`H-a>&Rs|BW#pbjO2 z;Zg(V<}vr$HNhoOB1_mCHt8V`QsRw)5=(I2h2@VN`x5t}^A|4X$n$8;k5J0z$i_W+ zXMPVvW^rx5#->%j0Kv^}GZRUDvf{)rQ)tEhOxN(I?`m}@^?prmeSs?tUlq#gyH@O! zXXml6ov`K~L3EFgrh@2Z`xj|NlpHg%IROy)5$du( zzi*7t4b_swZ!i%i}weWB-JoAI(qYDDsN;R`st%nY=U}NTlY385J zPAX;yq~HFh-vd3@5_wfl2np`Rj+@lF&fLz^d#g@CuOmktE~-*y%psSbb6|?FY4&xE zoNl?48U7;oRQ{3{4o0WfOegAEtaa5&F*EQ@x0+cY=7HzgqsKh1$_48tANrI0?_H;G z8w=-bFIy`HbHeI{(W%^;EU(h@OvR>+p6bAHSj{5L1T|aU^?#%|=kDZY98u=>G>?0u zHr-56LHr>A<+DBmnyS;u$-Wv3#R*Qjpur+4KD(35|CK`!D}BdCD0azIV+FLd_jWse z)*lww#XSA(ZTT)|?At%2q0W+LLZoo!bKJK?vK3YXs`;L`P1<)xZ)kM-ccf-2t78uC9ee|U=Mog2(CVN(FB$9#Ctj`XX6<+;4Kb+g&e4>*tf7O`Q!7{aGW*AO}D@i$Um?Dv)Fer)gVLNRR!BdPQ6uu3&%P6!X0mDm&df3RD|C z6Qc2v4xV~d{0%?DST1{f+>*HO|Jr!_DWays z{5*ZUQL<_2dj_hu>1hzYH7dyxW0Inq*vEsh0)4Fs&tvhnIQrn^Ezu2yE@52EISVZ8 zQ`J!5MrFlbQiKHKa!{22%I`yxZa>&5x@wTA5@{Xnmn!M2es4qDA$Jj?Qgg*brkGYK zI`+V?X5@W24qw2N>BA&TEBhYnnzEwr*5CUTooo_;i+6)`bK z)Op4QZm_srC+=i6S#RQFBIS>ny-=?{nx+f5z zgSx1Aq;#NjBVs^~UG%g$GT)rqt3=5EikSq2@Bp&fqeGr8O!vD3;TENOHJ;zW#B|H{ zPRU4zE63@yO+mpFO0naQ-67nji6m=JdUjc7z%Gu( z{NYf-*e@8uk%Dz#o`suI@*<+@@nQ4C%}sX{^~1@zj4t=)T?D5^=?vR0y~9sD>VyWp z97Bf~?EAZ76mded_-{`b4`=h$1wLsoq4y7tfkSp+yNo#e=Ed>z{b|_0mSb2NmA@NF zk&n1->V!>I&Sx7ZC>fMzgu~22SjDGu%gr4$Yh?T_e;D2t$G-t-tShiqOQK&kkG(g9 zJesw?xFY#?LGIKbYF-W7Lj`FU$Mv;ap)O^?pKTUf6AYKet?k(LS~2 z6tDPx7HHi;C+vK??^vN&hyU|3o2O9r$S^h5uy*(_XR2yISm%9f(MH2F;92RPEq02i zmoQ-hqsm!c0;L}1IAR|YY|9I`iEqzRQ1DMb@#}*wB1m>zraq)N4r~zFt+cE?`{Q)8 zNU|G4yIb&h7yR>cFf|YDp6t^pQZiL1PtyV1(1)6(-+mb%AJ8Z>R9!_CJnSe$hb~+P ztd8I7U5GjFeV_U=pmrr=0>47!s{{PS1V(ftAJn1!I~vVGQ(jz z^*ah9leTcBw%niYZYkJsO@ehPu!RN>BzM=xacZlzTS0GA3R6?LJluA9m1uk_7p^Z( ztGqOtkiMNZGjLiJ<7{z(^LF2X5AcS23hRoJvu|lG7dJ)Yck5&>KhrT1igeWt`3hQ# zWY#n6N(}}c81t|uY@dDRYalYWe>amrd%a#Vf9&#C`xviJL3E_J}8 zxa;qouRbWx!ZA+U_~tas@CFfR_DPt*Ap-HshX;u?_;e2OjF^GJFjqu_*>HW>}Kh9iQxGA-XH51ioOPY zg{PF~P17GAT*vG@##g2O-prYdr#de0VsxMW<~j!L;&WaTULx16{4mT%At-8qq#ae| z)%A?$&G=h3s)pV{4?qJ~;HgFx5drri!4)oIhiB|tfW)BMQLtj&mL^$a+YN&V1 zehJ4p`ob~H*8?WGYm*_*RjGE$=g(0XSS!G*8QdL6IWt{o&lGCbCsOo|Vbg0|?DON; za-XZz>e)LaY=n&jqlJpueh&&vE8z8sH08p*cF7@1T&vwXvFJpfz6PDGKEIw}1MB26MS$lx?Rb&c4zC$-6WN!-atUHwgs;-6lSrA%#_KPR^O zc}DPBUWI|0viK*8`Cx?qx2s&X-eZ zG!^@ga_5p;)}{ovTK$+!_U`oEr@H4Wg8wsKvFrXPKUDUeyC1x0;rG-G!WPS5W+i?| zNOV71pL=?3DB6>~ubHL7*+XyE}83x?Y!FTL5jUthi_O>P;!?VU`fRy(Ix zEHCJrNK%2Vg*vT&C3A~v*4BqI;tmg(QuLSsUKy0gFZ7`8%b!en89^R>NFX;dySLgK z$OFIt7XDL%2)mQT+tjG~y#ytvgAMQ@AAIi?L$%Of=Ls0T>@aNN4E7+W3OzX{8hmIkCaN%l`Hqn)!`845*J3j>9eeE<9x5Rf%%tV z*d~1bojF;x4kpQ6IhmaE4#m5Cko@6$Q%e%uUvWy8R5&Xb8-Vr=pa3A@nUhYvqeKF1 z>D{6~b6a2O8wa#;t>+Gp0(d`@AJBd(p~7;ZsRFF_d7tld>eGc!-uDH!c}^y)H}&2a z@X9Uu>aJ@I9lcQ#4J^A(k%Cs^Nxy!RyNqOW|8@uQn{|xT9GxSJ1Gtiy^8}5Gs>+~b zmG-Lx(GwnHOyQCk>x>>kN`(=5mcsk@(&e*lmK!6v$$om69FaMZ%Q3%>g;0@|U0rC7 zjL-7+pVL%4tRMF1bO|rOWVY&m;Xg1>E4yVrXje??&MlGjSpforFPA|i;}1Vb%A0tZ zp%coY5?8u=yrcQ60sL7VXyz_$8w^Zr>T{yQ?xqC5}D5 zhci4DM?BS8dCEy{#(7R9DU1~SynD=FoymF3{kCtXDGZ}_JhCL$v^%g^RH|Z^ya)N6 zL4H;gWs&+WWyTmGLC)XWj4nrkLuknmt?C%?R~A)S{X&9kFNAyb1J@D3dVELZ63S$25h{fU<(6d&yn$8K6~wEmo1%O|Hgp z=xwE~=HW!Hzpjg{b9_;&SgL3Ov#Pldo>@d(~rXURdtEBZE7dD6w@c7g({e4N{=I(==Nv0?nIkX}^6!Y%4qqeqZO_M1nnAZ@)e7+opi8MJK>$%Ih-wo$EcUYu&BQK{Kjl zS2C>@{-cLL>Zq-f{_xnAEie2{q11C2!lnTq{JdhPqs8gX{#xsO)>DBSxmFK86|I4t z-`n=$Z54U3KI(tB4e*Uby70@R^O} z2?@eORV0G6qk^N4B+5Ri7Air5-;|ddHT@#>d+C0A=Jclw=nIV|H}P6t%XMPnZCfUA zi?FN53|T0h!B|HoaXIcnt_Hpncj^(`aqfEb47_+R9_y_)jq0-`2X5APESnB#tKM4H zR%a4C5>~=;WSics6^5Tzs%V@#ke-%&O&@Hgd$7ed-Qq{^DRZ!qN6or5nRtnlI7r53 z_w?WB)%db8R15gSVk<1#uLsB4wZ?6>zdK}P6U`0CV|K1ciOMFeW?XXEv~+Iq7;rKq z)?M!GfzHP`PB??nPdIIa7cYOao;D(f+;sYNlFH)6;!N8zY%uP#kGHY)C+e#Efr&Z# zq#dj8^9Te0acfNPn3~6C4YW)0BdqzRPMjuU=BYq? z=xuOCH+}Cqve)I{cVu1~7Ay}#C~TD{7i<$;H@|nn{!%i;xw(r7lZu?t;bg2L5RoEQK_wy zdppNZY9EQM4|(+7i3l&03;V@Y-kV!)V*Fs3Hql5|{ss;%U8bHaR-Sj`fAN-AZ|v8| z3TZ+-uiw!u09?oVqh?(*il$SzdC@k)$ec^P@AHu^F2wF8ZPJ6&KZ{A@K-FW)xfPT~ zMvuOv;;uzWJT|#zmi0Ae5Ukj=?PNPG{4%BRwDIx%yaFh8Z8-+gu1-5Al=6qpl#h2q zBcf5S>!6CCdSROb_hedZX64BP3zgZouu=!6W9)sV3iEe>#K*#WjSS#f`Pm}<=pojN zFLqxWbD4L$%xwh`&HT%UQDp@*Jio(ptbLni6AY4klqkJ>+1!m*a6^G>i=cp0jy>z| zsH;dSP5T)N-DbV|A5WiK(O)I-UzgaPWuyUDlzO3?2didl58$o#(3f`x>&(tb&1}V1 z)RFj~S9f)(oEhll=uEgw$=2g$19smEB-HUb>5^(%L7dW09iVq9Odgw%<>HxtUxPRbVJYO>mg*&b-WItGf#XTVe+eiN@_ZhM9& z=c?;A)`lY9z)hpuVa3;7UQGgiuUUqkXH*d4Ag*6mIGW=C_>n%q`E%x%ud%y)oSd~r zIp9T&DI4#uQ5k8tIni%MI-wH#78B)#+9ZgR*=)@7dlcJxscKF#O>hAUtWBLY{Uw8y zUiU{p9fEa+P^Uul0@Zzv&z=_Ix_7C(s6r~XP3|2-UrIv2;IB79KGQqoUA7E)kIfS@ zC7Vb_kMb?eHn?H=LLoszD1j9;MoW6ui6ylTR2OnOBpDytxUZHpL5#Q#oJLojZnV&2 z2;p+|&dx1DWw%MF`dKSpT<$Igpe!1ggj}Xeb&D~XT&UX^k9{w0vz&n_(Nc+n5>`P0 zX@5Wr{%@z>`SxT?dc(fDX*w~q)s*3&7EaB+Onx2#ef32VuecX>2sIU=%}|!4QTm-< zJ!SYw!PCo6lhLlBufEPGL?pJW=C&%d$pnEPgDUYQxX<;fuf6xmOp$SZeB<~F-r(m& zFv?gibTUsY)~|lqS=R`IET!0%gt*KO;g!|?%pZI2hA&yV{bRDl0=c9tdCF9w%@j%s z6vL>J!q!BKvr^#3IMwyQt~7mS#PCkraM%>7ZP29nCf({Wg8mm!uJkrfdq^1PDFnsv)z znC=WN4dtm7+G{}m$q2vSM2{p(y9sU>;qknU%pzkcJE^V&6LIrAAfScC%;XFU zOVGW)JG0QB7C8kQWWcYrSg_(o@-%g17VZ$?1hZM)Rw7cIjTuU34)bB_s}~GTT#-p0 z8IL8Wna%~c4Hm}=nVNW)?Oqu^=h}Nf+stiXM*Y4HIc|6F?m2=BqgDTCD~Klz`n9v*B?5fQ6WMWJN;|F8-1;Y!-O-T zzGJCht;Q#eLpeA2%fVGW!jy8H zvo8wp9}0SDlk8(ROH1jE4s6%1N#Oc)ePuj@IPY4U7}X8K^o`K=nsLUU z4nxQ8O6)DObK^ww%5eUgq*c!!JXHFm@Za{-BK!ixAc2{eeZ)^Q+XaAxSQIO<#Kaua z2n+W@`Vd>?DfFz4raN%BlND8uTgdcIY2)$3zS{Axpxv@CJCb3K(ly^GEsi5wZ`}M` zp*P)H#$6n&4P`2YKzj{P>krW=eRtG-`pphh%zeb7ACg&Ah1*wn( zlvYI8twaduBcV2M)z_rZ{l)9sgZWxV9{E7@k8!)aEXGwAS`!HbY*-$v$b1{c^^g)t+2fZClkjhux%yqnW$rpm{e-_e zNM`f~hoSu4r^Z~hbonY&g?#|Kx7X{B&)QX>O)^lV1F<8)~2`(!0bU-Z#)1TXk?D8agCSQox z4NBX6&kCX7wD7Hq&Wsy%O<#Dqe*Ln-{}KDmm}X*6y(J?f@o-8pj%kmOaT!`%6$NAC3(HSkSou zLHooy$6pWPZ1SG1!7#@<8m$@@>wEW=8qPE5v*m#P>ZBm?s%jOpFIsB6QhB4(~ppu!jeP-w|RK9VPMY)C=9osfjZT) z1&Gx&&oF#5z{HFSL{648y+<=o^R?g!z2wb&_&Z3J@aRf+?upuymlZgCJYkL9My80m z(`{1)Uxqwqov3MfGR(8hw{?O{>+I7*lJHC0c=>V}1N>i7@y}|!58_{wlYm)*_;8eY z@S7Z{z4mDk1gs+%zB*HT;C&+;LQ2ww%iq)MZUnubK31EzX@H51D~(^R=H~HNb7JrP zTva`{^hKV;^<+mfEUUdRc4lX3AT20EDS@CDa=}1#a_4hm+ytvi3)sbZd9r9F>?G5a*#=oe2i4`d5aN8D2_>h)F7}I)|Y8U#%KT z25KJfr<&}*_1OrD1}(b&y9&__!%hU?ZBV6}3+r7G#@5j{g#c;L1rxtw*NAsGS5=4E zX^cC>3|TR&TddsFzCsX)EG)uYgwIon!|s?ow^D1_V$3^;THTN3N`zDpbtcQMPa2|D zB2A%L9Kw4aRaGIO$~_2JEt0P_2ybjh5~<$J&VR!=bTF)AQozx4lQW`Ve15zIPcVw> z(R*N7{p~nMIoEC8UpGbETT=9Lc0Ui*VAAE8Vfjv7uY8vh)xzCrJ$y!Disw;&-pikms`Lw4cl*-wF)35TgHp`%tAM&+2<>$5%n~W%GFP;m)euKPmxwG+8%r z^cgH~OR+dS>lM?#suwWLb{}T@8NW-fHb*uca>si$qc%%ASuZtWCB?twzdG`y1ZQ^d)+w@ZOrki)r#p!!~50rCJTfW$X9iv&nvtP+@9XR;l&jb$R4B?%n8JMKo)b zj`F7SA&UTW&OB3|`q%9ugV^dEmigQZmgH5lF?^wK?gYGH+rOiDH=!&xy>WrqeA-C! ztXqh?V&{2QXxM4PUgs54&ByqDYxw?kH zwGEKNQ$lq6%(~tqAM)-k%_3d+pJI9yOQ>A?THA_2EweXfn#8Y$|o<-nmWctB^_<8-lEk z&g!P$OS%&+Gj+3IqF+*&YP>Fq>pNMlmFv7Qt{03l(wmJAZ-!hNl~s^1=pD`>mPS8_ znAw@jA2?LgTS|5Y?QuE1^VIWeNY}4b`pyfw zjid&=HOUyVKC#QT7xpjDsuw9#P0+i2?2m}SM&jnyPhxDRY&%L;W!?95_3MXy;3t{m zF6aLm=P=Uvio7Z&Y1&3RHPm5bVlGpr+BHYK=@A}WHX2q)htUb zSE*Ogzql-@w8=A~bUH?^AnyHNdsqDz<da~!a!n4l}1_wk#c)Y2s-xvVrUA+;hUx%ARV$I|)EO}+2=58fZ1`H2s6&&+jwue@hxuDfCTROZDD za5SWzx6qPGHJN7H_`05kJ{4Y_nWL!oo2Dh|U0izW^*t0FQ@4I^%&v22GNTsJG81Py zHS*lP=tI|+I2}DfJt`CD(4+~T_Iz%CTcsfm6InLFRr9Y|!HQ=qdX{GN^zXY>p@PnE z4lhQdc*`ir4GJ7FIyTn>%-zdAgzi@Ujva_#b9Xc~%EnDbBWx_{*IB#lSn3YtiNnpO#(`_YpPdPZD!^zVM znenYvs;!3wf0o*ZZ72G*AFe~XX2cLL)2wJPp5!8pU={bkyENpMBNJz7lgf35IPm(+ zqd&>mu&V)fpYw-5WDeW?if?Le@{Ta|&-zjm9>C@=ZoD&;K*Q$H6m_N(q&}{GR`1%B zYJ4@~`CA)5!cGJppypFYs&3@qPGJ8(Ih+&H1Y3|iZrLQHe zQYt-P19^N<+6XcuZ0&8dxnF*Lb?GvS7#=*KmMJ1^Isc^HA~Bc!;+S37RG zYm4gN4W*T>L+=l21jSnltl+Mw6)$gP^koq08p6|rO^u>O@G)wy{=}I>Y3s8)-w|+~rvBm|*KB`oFueN8}9bg!`{FQ|(5i zuvv?5H0POGPX*E@3y$C1s++9FtE`+P#q2V3skTUk7KIRIyk+)Q!XmZou^+3>8; zNznMx#GH$(Xj#6cOr^S#Vq%zxG&D|RGRjkWAvyGWb6gLN(f&k6N7{hXhi7+21ilZ$ z>w$uxPUT}~{K&K9le?V0t`~5t42`8Y?M9ys+VrNoTV`$0X}hj}Ii1nIv<^S>isnWm zZdLcy!$?oV5w;Pz-tG2*tQPtF{5YMlp#+fv^)oLdFxxRrjMhxo$Kfgku}-UIt(&nI zPQapM>XWXQI?E`2V7&d?U~<@+GN2oySTU5Aqxr72nz5nHmQA22`1C`DYjYBCW-Ql6 z4=?K^<7^z8EU1`&vr4m5>!Yj)5!X668AUezqpY{6u-G|i>Hej&sq>Gt*Yuy$W>Twt z6?wmk&ER?Q54CRxKLZW%y+m{dHK8uj#qtrAe0PcMjh>jC4Y~BE_jzty>*-XbvJIZz z(d!t)jchep3Mf9hKiIi(-Cx#_h=^Kr_{!rhYxrun$)vZ$q}{-8H=4)i>0DHpP_1@y zQop0$Bf{dNPR`!g_U7V@1+6_9lh9AYF2lSHh$pH&Ez1?nz03Dn;FkhMy*xSbUv=mN zGb>LwQGvkxw%=&l+3IAz}vod_zWcgC7Ds`)*_a;ekx*6@W za^4k%Gpaa*1>M7%uhYLbCN!fqPpJmoRrx~LxE@@gcFuRca_Mn$bFs6gYH3PQA)yVU zq=yyPc|!t{=dbmKOr-G6X_J zP_#bbqZY(Ox29-5+t}&;D6;aWz1jG(fO$hvTk8RqtAt>OMJKgIfzO>Z_k=4D79rDb zLT@HJ1)Z)TOc2KkotD%a=K3$oY}v$6v7|NfiJ+WLZfE$FG?++r*f_@U80P5jiBZ+u zk;Tg-p2v$$-Yl-etB){mzR7Uz8ib)S*=z*@8J{9eo!aMjXSu_29kt-_k00`7WYSwx zQiO+b)Hv0XxVOG5cN^L6UWt~&o^P~^&nvhe6g?X1pC)p*Y$f4-V3exE`fp5hz)7pd4X#d zFK&i7_N%&^VpWUX7I?TVQ)*t+dB-LynxG=*&pav@!6luj|HjLYBsx2rni?DTGiujv z?GE!x&q~(0^!M3d>(L5PUQ+{1?n87M!avtny%8nP9O=HFQ%YC(VEnM5njr8K<5*bR zJ3k)~vS6R9Pqh_pl{lUyM#DEHtQ*|RQuiru==NQ$ay4n%Vi&)a9dp%u^s7&XxeRl^ z4E%T|#Mk#m-Eq;hN~Q2i9l__-@G2RbD~dXM+{Urt#K#Zr2!7a3c{$@kHr=>c4EH4T zn?5CCu$X8mp6<$KTF}9TNeFpF{Zq}RTiQPPTnJfVovKr&eRiQ${EWwzMkxo zD>gXmd55f6>A&d+{X(^CJk*naA?3TINrt2Pek7PSfg{^})^GA0 z&U|TxM#IceOEo$sEqf&Wr_*$B&`xz5n=FUv&0uas2M-eipU@YeWo%f@T%<`NI*vlr z6UFbz96&HH;{iGr`I0N2HJBq`ZPIaBuTcaey|;v?o>$@UOb}^l{ zbtbxL+Q$mHbXkuq7F^S{u9Hpk7dyGiW*y=ZfE$T;`@|sh+YMsPiH00iz0MznHaks8 zqd~ni&I3CU?|zZbMR&_7q{B%xRW|3`g`8LKGjPs4w4br$TY8zY)bJTywRC!~41Lu( z@Yz{MyoBG#_{8aaB@zdZ>ohuP!Y)0)alO=jV6VL-lqx`8Z-B$cm}IKySoUuU6~=7k zm1NWPK2)4qmzyQNER#q26u#On4bF*XHN)@JsYX2?*R*SGEm?gu-^(`PZENp2<~LTJ zi>~u}F_A)*)ie(F^At1Yj3bdKO8OHHfw4!eB(X3^Awg_-ZWZzC*+UiwaJ7{+DzXSp{woMm zgrq>`yzPdivkW-%JwtlCQ{t2frKFW(vQef^furAwVx?%r^jxk`H6rx6x*C07_Z57W zJv%E3U4^8$pt95^UWQr{6)?8h)~>O*jjFTRl3~Y^34}ks$9%1`d(OqBKMKVs?4tY3 zSbKPfnFIZ9GSo{+<>7EKfn7)i}oyG04qsO4)ASc9%@Ew8`BeK4*kBSxD-ShBkz(=FdgB@{Lh#$LX?!n9XF?_xBk+FOvUp%xm7< zRq1MiZS9mGbzo7^!7i6pn&}G+6L%%4Y659qsXfv`IB}Z$siu@W6r|cYHhP;`GqhqA z%dZDhH^)`k&@#xZa5T=vZif9`cUy6KQJ~4&V{(0mvLMRUIxJv9hnT&ebB zH`SG3@iyYUS}BJ8hV+g&JB!DHAuIvsj`D@V!c2NTT& zIiJ_v#KtnDFm9e#edP9C8d)_jD?imcv}!K<6iXMWJK(zLo;zRrL_2HoU2FJHyaSFXSGCDCSxCl~x0ON2E14^0 zUA&dtLo``E6=~t8xY$>aZ>c#jPYC-qk!m#;M-W(3IN2-mf|2rM4u?2e#3yQv{zSfv zLZ5oiP1jO8L)b6fChax39xj(qyZ0rJtv>wphKrQEPa^Mp5k~v{gbhL>NMgq8EmEIL zlS1u*yJY%4~ za0<5Y4_2%zPH*#?Z_V)Y>iuW!ptt9O>_02a34O+_73Esh7Vqs5u)+KG-g)*cym!~p zzBO;y4fmfF9522M%OOcrFFQ*TvdWKoZOan_&H722CG43UM+ErY4qP%#4F~T)F)SZ; z%=@rc&#fo)g2kgjt+#~``BatUfF0r+OwpA``x81>U-W>@{9-{BCiKm)ni z;D3@|epIF(0@YAlzrkagr29T2>{W(LZtPc|fWf|2LR4zDOe+kJhf0~foqJO})0Fae z0t}Y^#|l32I^vnSUdnuL%^P#35BrlpUj*ezy4({ko4pmTv0wwZe5G%c2@m7j1y!}K zrrZi_7C(S0y#a_-o3eCQuWy@)BCpSyy=u<*!`}dam-hA}uQ|Ec%7hwy9Lv<%mqqqx z{`w63vg@Fkg@4w@Y_MX18VjFvXj>JtV&*yQ+*0;VY?;#~RpGcd*6REqPy>dsp1 z*C$YX`(@~KsX$><(Z2!6(tQn2^BD(p->Gt=9s|_{I+?=}BhS3zy|?D(QsOsefl>b~ zzW-$}R}(1xMSwwnA%%TN$t-Tve-((*J`YA*03#Ze*O}OZ&$hV)0KzrjLPWv3P64X8 zWm=|FlY4=Zk}a@am!%?s33NzHG@M2p0{e3#BX*bP6NBZUO5lb#UEj> z0Fa6|_tAEt=A9S&sHIj4J#k!m(7)y39ay}Z1g7sCi9pa)k2w3gLj?tQ#Py+}fnX>l z@4Wa&jx9=v-X@RzWr0m1n1M}7uzlDGE-Lm;zI)sFhlx6;sUTSWpsDY(d#!Z0w`fC! z;j5Cy03RcT|Iu*Rt{hN1zZc7!!A16r6zzlbsQNkMO;wh|(SzmC%P|@6=P|C=x+Hka zMUebz9(S5Uq%9tCm9EOn4hnZ2*N{{I8$qHYU}Yd3g#De{A!f%=AIu)I22{YH1g!T6 zDMA1ET-0KVLTJ?vZir6Pa5t{6Gf<0@mFM9s4>zQ5cdhqXreIzq~hk@Aq{ zoLk^%kC`GMi-H#|HvFf*CJ6PYtU>n8Q9(DV2T$+7y!0>E6DrNxg=-#6HKkYrlizQy zn~;xlS6SXnEVAdKkeq=3i>DQ6bL$5ygmwgO=?S<06O92oKp5_r>s}&YijTX$d)cV( zqWCb2X7#?CIGDYl8CU>y<`;aqcT^aCi%eXV#f!>fVBq_$0x5U{6ysKpOkVK4mnUgIoAC5AEw<-Yq@PCoppD zmg#`u@lq-3=xKFNK%T^r7(cWV~Dl2QKag!s$JoM;?LXV~vwS-=Qt^q(V z^U{fbX=!b359{x?VmJc)w;%{g27X`gan+4?0}YZC_9;77llTZohd3TUI%E(6*#;Cd z4jOmVEig3nkg2ZkdDz7qkis0RCwUcQ8G5bk9=4S^YBw;^G6${)D1Ox&z?DYopA|`S zqBeugoTYk`4Bw3-wddQXDH+g&lT%}`8Rq$y7`>)JxcjXPX;0OQ&U=|0LFA zK37U`iXQdwm?twV6sqn^=TgQ3jb_;m3}W{J(gl&>pqcf-eQTaj!uW05NJ5Kop@W!I#h9-4VN+2I5gPEdqLw8qGl*Ear4Mu&pSrY;`uYJku$L{ir_%F}Q#{h|xV&4glW3cN#>a!8Iz;1u> ze~zH?f(j3YJl_utd%r~>4=EOM$gJyAMsP5K4KQopXGPo_*pDe;^hW`Qx&$a9Ec7CA zRfRqEII_DIKAkw78<;Aj}=zjbkiT7H_ diff --git a/doc/screenshots/DashboardAIL.png b/doc/screenshots/DashboardAIL.png deleted file mode 100644 index 753863872658a86927563678ac20a21fe873d5a7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 281189 zcmd43byQqU(=SSdkc8mDf+a(6cZY-w?(S~E-7SP*!GgQHJA*@TcelZPkU<6*U@p&_ zJn#C>y6gOP*SYKT$e!xnySuBpYj@SJYfp&0tQgwscdwC~-FTNomJ;#0d z^f8A7Erb8@@WN0^42XpI=at!75c8Px%1&Iv5eW$u=bz6Lq~uhB$4nF_2^kTT)t9Jf zOwYb)uQwwhy+@J&epYr{JX$exdpq+E<@!@ za5Udj5}f@u3Lx`DY%LjRZO2*&LAZ$kuFPr3gsKq41LLiv}_2mYUGWWyIaK!s76#OVsYJDEG?b6_2- z^8OY6%l#ufn@O?Ph3?@+qu+L!T~+#ezPs@&AU zICh2dIDF~zzOt0F+T4`p@kLBKLpw*7Euzvw)1*%;sZh1-4Hil>#)cLHcvF_=@Cq7Q zpd~$2iJ5j73!5+D9E4gs!%Ox-IE_)A{%~ywsQceBYKt<6_ESzY>G8HjWiBJWrNDmA@m$B83P> z;>if+mN7A~F%|6$8F}?eV@z)Ekgkhg{cl@29+~Vpl)gATji`=nikqw|K5~*z`#=!R zn;4s3h`$P6f<^y_e7VRgC>2e^HC69J1Ah^RZ}Ll$rM`%98o-#Tz0%F>LlRZ{usX|{-%8_QI_ z=;E|slSB%9E>fsZ^0wv1!i-Kd$@^^Wx6)kAPh8%t;ZZ*R+az({#&6J(pv$@O#MEc?>vL|@yyi%{DC({aWI+JE#QJ+x~rAl zWw*^yf@j+!NhxK|3?4Ki*gqoQ*ic&)ygT`&3a~Z-X-8dS(gvIO^48NOI!W zYkSl%AbidOtHb$zF_~(B0?~&>(aTmrIfgYNu?$ddkp3))S z^Wf?I`2^PgTe}Qj)3Fq>*s&&F+AfOl&BN zwZQ3(-|u|jtWIF0XL*Uv{a)#QE2~$SW_(*w4MxSC`BV{on^yDj3ST`dQ=^w1WP@R@ zcpi_~#;_y?R=(&avB{?2`3nvxfLr?bs-zqjE^u)11JaY!Ea?<;3u5;}f@>*g8Z`m5 zl7lMt!0w`RY}Akw?7fdR`x+C=iu{IQ{i9!Gm6J8{tq=4$+&OU59=suEtcRI4t)4&T zRSSA-G+kRxNB5OIe7chFi?*igjT#~~+$W2K@XJEVid;4WEN&}jh0vxKP}JyG>_5mD z;bTr@)x|*%j`cvgY8zUle-xgy8Bfl=cmVvPOjui*08qe5+F24pW6wjYM>en$_fx57 zOqK}WVt{Hxc;)h3O;2*TG@=@mSv)PZkgCz$aCWaP0t8w$tc;x_2F01?Jd>waaQ$RH zWo}pDB*f72iuSt$7rzgw(e+al*CEU4CEI zmrN!Q>Xz~NbFx3DVv3}Hg=KWI4>A8u?tHu6I)_{Ri={$fNwSxGqsf)K(PF?a$Fuy0 z?8}-PoE}U%FK>v1l%T$56stuvofprBHWIMwlFluX+>fP5C#3k4kpR zrNjr6TtPT?;ju%j%L_v*lF}cYPp__ivF28^HJ=|0Q^E$6^r>bX&9N3`zkeQFGwPE`S`%|L%p z@>!UUEcA*R?;aX?Y*{JR0|sZqN(Ln>jHMZFRBl27k~z~2btXo%NqC+&CNqx|`nKMy z6|h&ljeIxOz5JsvSFs1HDx8EqKdPi?=f{NV0!Gr6S1@v9Fsi)~era25Y+i6RTM406 z9Q8)tfVTj}8>}UAgQbnj1auPd*MyKU+ZK753?b*C{`mBpd*BB%@MhdAg;B#maWeP) zu^ugK@!%8pKQq8>Sd;fz`QsYY5Y&*w3=9jgB*G^xiecI*JSe9?3wvFf*4hKs*qo|3pg3{|lI-Oe9t#r-lLr8b z2gQe}C+vV9E5lRSje6-6VBHoY+9U73d}#m?&&9koo}-v7xRQwu8)u|n)EnN`9#mc> zjit^s6Ellx;?mI!s4Zz-iK42j!Heo(>l_PHn@WiDC&cP=V4$6QcBFFtVxGHxV<+!v z^{bOL`sqU_-Za?)eBL@+TQL=LbKKjVBcXy(firmW=FJwTK9EXG0d*Bw9#lL+qkL|$ zdL8_u{(;nxwjrtjAe-iTj<-3T7ux(OE={wT+}mAKlw$$8X~+%3MU}neZK-Hc!M%DgS+ z3A(f)Y&-}XYW#?PguAZHHW7P`=O+LtJ*B|-GXjx%sAe5A%daG5SMiFg6*;h~@JKmh zRTGI5a-e6NjjhQ=tv4%yNloogWPl%lYG2gUM%TwiR#nv{e4MJku|voPLlk|*|DI<>?6z{--2&a5_!`&+O-+L zB|B)9ArZ7t+ThqAyfI#Lz8J(_ZiJaSBtr_gwvES-UcnH3qwn4bgHB_QOCS_~&u_JX%f-*coA z(y2@Chst3oG33)Q#(l$?BO)~olYpP%R;Hbn+5e2e&ZS(z?QTp!v;UUO^jJ(|L$ZR6 zM5CzLYs7W7&t#WlNU_x24uS(u(Wu#?HbShALxP3H(;IepLQ{@-0-hEfFTLMw z2DbC(t3WHH(8<6g_)tYpcgLxmCKc~B;W{Va8!$z`pW>vVl%#vo>yVP5xPXwzRN0mO zl2|G{yXs8U*{1-Ae;V!xH8Pmjt6OTdPA=N0o70rPZNF7YlUY^9z=I~LH6<^z=pCo~ z1PcRiA0c4lERj-ndWU$H*Tnc8e-_~jF!awKkV^`DbZvSLG|#h3KixX6T$&0(ZLP2P%aSgP4~h=O+n2O{W^0^9%aZdph8sD5tF|fnzS%L&A1E>L_K`LyE5duF>Th@2E>m24@%EBI2#)g@f+{K4!)I*g&h&OR z<_w4umZhfU9OC#L%~YbytL^Rh|jgomR3cg7nQ4bp-4vD_n%% zYq@Wp?i+dy{53c?=e{@UbKInib9K(u>IQePU0R>~9ut@OIi}*-$hDXh-X3|;nb*#F ztP<2L9xFm~W`;EvX_Rh6Po9gvTSi^mbN0n5c{!}+=b9{OK z%d+ zrE@gx(Yx?gNSWqcqu2T9eJfLv956K>s;aB%A)6FY z%J+D;pfX7=HCXB`r!_71{GnJtR?&*ZH6y2J%t z{%I7Hz%cse%oSTLU$-fmsuqg|_d5L#T=1>0^yZ1S%mXEQ*l8>?jPivbiPw*!5H6;j z;*~O@P`rvPTRvyeJLH1dbKc8unEhFyd6swlx1O z?2{)kRD2My>+Q}qPDttWAWtp0Fdee;XxCbJA>gZ8{hasB84vqwXFB*_bR#M^gqnnT zhO|^KMMZ(YkJYGlIK3-tsa|EsKp;OG7n37xVI`_lyW`Ma*9_u=6wsu2XZJxO71l~FbUmH2dbAiQc?C+#E>Mc|*rgA;u?x_R&%qQZUZfJES zyS4lm4(h-Ofqgt7*76bH6rr*v+4+8f_UE>Ygp@A(oyrl!?L1oKZ>FeJZfvg_MWO@b zmjAc>)X78m8Mh&>`>TW)j(Tg9h&@xL4*P#Vw13jKg6Ww1H~+#{kN%oCmI?WRaI4o zM(y{QnJ#uHef9$b0|o{Lh);T&W%UgW+zbpOOG`w%AO6aK|3>qd-q!A}mc~En)$^x; z2J(Cjn;EHCG;d#%5EEO-isR$sza{&7db%CM%QaeBS}!jzAOI_4ObiJnGNG={*~drl zzcQ9jh&}BP)%R_ zcjQjruAjMKhX5H9i1y)$rd3IsC75{9-ZC@6Zb%_EV}*RZtnB=Z#C zcX&Ml%KxNq1&N>D%hSg7y6y2l<^C4bV^2I(od|#Nr}UKR+Rrx3teO0Og0VTv?0x%^ zHpDBQr$-~p4jgh8+Z*6U7+YG)$h9b;-^Sn=_~<<8N;NEP{i<>BJhXi^*4foh{g@P+ ztwAp_r$OYz$)rTlXe|EP2^ECDG3}m+1B~-_=o+kBitII0-MiVb`JmvR?t9Z%UQX%O z#rUTyNc_6J4Lu`|7;%B*QjOOo&(!F%bfP`_sf2tz?grPM*E6+qp1cC1^FOg}1^C($ zYJPF6>ZCi{BaUTCxJz6E-ReGG7(7BWB$BM*FKh4tv>x)e*+!0(?KL#_R|BOL{VkQ} z1*m+k==VPRC@a=Cg@5rxq`PKv}JDp`mV+)i3sd#tHdi-AEo4@4mz1H+Jw)IrM)wk zimlrAhjAHieEZ8_M|(Jn&F5|I2qhKTOJfC`lQa6oh{UfsbvT$UakYCGRep2!Ba%?mm&={5SrtR8KO zQ>Y;pTVTQ#RjZR_))(WS*Ue-PRM<`@m2TJT-pny;I` zYhEpwyo;x*I)}Q%pNXrvS@l$R4bDE(_3=J9SC}>DODK!1&<|lJ^z>OtH(qeR?k}aUcQj5zi8f_JHUi- zN+(q1)toqxlNww+N-D09 zCH*g}75}gr*6le?Wb2k8=y-Op0KvyNkkH@8nQ+w4aJiB`9#EnX?4rDS6Uz!@jL6S3 zmokbe78aI~Zt-}7YKyw_sFUGa9)la%P&_2I>1^E_zBen` z`D-Z?i{d96g-R(M*4c6f;o`_xsu;Qez-Wx5HBVo{<)o&Y+hiHav~helP!mZYz(1cl zLHF(`nHH4}S$6CPihIggOR8?Ig$g4+X4k13uzT~2K-5y7-e4!O%_Omnzv6rFk(bp; zS~hj<&dMJ;fq|NqM5f&%`|O!3C%o_!QyK1?x8jN+v?}D z&qX&@d|SA%XrLjs>j%_jNke`3P_*azuDYF{x?koxXwfBo`vCCi(g|G#@GX{Vb6H-6 zWWGnt`dNQ(DHVCMe_Y#33nd~&O+vL(B@|qtCmsnId>!+;u9ZQ=S!e=}5*Irvfc)U! zts(?#q#$H`hxFn6r8C(?mxOpXj+T2M%n??n!|DkSNY#DluGwStad3q zyyaF7&JJK$UkoRD1y?;_a;P#Rgg7ek9 zy$tYWT1k{LUKCuE3S<4dXmqp>=J>uKj;t7KCH7?0@`mwPrOM`!XH9oc&#vaumLwWK zE|#XI`PmydL|hp(raU5p6QKv=3Hfc@g`VnkYEtQF#u1_x>xcTxWpm;o$F) zgg6V(#_R9|nbttU8r7&L&9gSkTihQfl@iICDP*^KR&~WyFsf?8#y`!m4z^dGc-4LV zitqYa1G-P7Qa#IQ7^r~RMH~8a5Z4fOju9;!*-Tl9Ds`#X-ll__JT|i$EJGG(Q9-JH z5YI?=d;MSAF$$ly8zWI#9ku6f&plrWm8v%e-s8 zPpYF6omcC&T}UG0tj;KS#Yp!@W^SE?A6Pb$P=sK-h*Q6LDU^q0y)3LjtL5)6WRGSh zK8tTHAI^kyGVo!P3TpL$F5ZywR8=%5et;fvwM;cxANuk$RzlxEcy`?PchLCe`aEP_ zRm?FPy<;{U>|aU2xT|sl>F!LstVuod&I^NPxu{xY9u}>TR&w6z_88@gv0K_2TZHrVeu*_;Sd3TaFSCzEsG z#yIXWx0@g}ZxNYN-}TOb%*@Okke;UI!P=UB*!tn+ZCF?s9+$57MmQFc*u#yHTGBf1 ztg<-%hynE%Y$l8X+jVX(J3d?_SRUF$ZTI~j6M~*%Gei)xXy}XII+*b8+0VY zxOqP_QY!p#Ojgr8(?+Il@3Z5WQIJx^1~EPUdh@2CnSH;#h5hXA$B0mm`uo3T);BQ< zAtCU>f(E@$cVyviR`uzRwI!fWrQxwSQ&dREX}g=QYK9v*D=RBB9A{=kQzxd)Oe%Y3 z4I~DZ00##%oiH1CVEhia3_8EKZ4v5&rB;?@?)?5?wSE=Ss=FFpK?N|RtFTGbIHThG zHx_`$T)m>dKk-Aw=%Pc;Pm1jLa=BE1^Dam!@bYI^n3w6amXz%BBu8ZmAfeDEUk%Zu zsRDm4(lc^z-x-|9d$Ksbm7xy3ntvhy$U%wlv|T6?X28NyAx)Y!o9vy^YER-VmNSMu zY_=~ZWl;jIX3Nz)!VzqA){tv_pMhEJBp$nQkXr1Eck6@bvJf_RvAYN+J%ZD_YGbr= zmcP0njoI-V=Z7uM%|ove)ojZVgD}gxKGo%xL?*O|i}-21(ZpL<+nzE01OqqI63Y9? z{i!)+ClBk<>K+6U#OtPSpABQxV(b3Hi`lu46%VIyCFpN5#%uG@=_?SY7Y9t*ne2#~9cEBRvoqyD*4|A0xly9E=DuFd;o+&wG>cEIJOs&9SNhL<@+Hyy&p7s zAS`qmb*BP1P`SIk$2B>x&l9TP?{uvT}!AA!#U6em0??(x3`6Sj% z21CF3(9fBtj;LR3pBvCA*}hew@?ojvYBxnwEt}=tR^AhztiF_aDk^jcy&BFU=M)oh zIhZcd^lxu}7E=$pLf zWz-h7P9jg#IAuq{&a?D!U_YipS5o$GQDGCHFxq82LW(5W1BV^(@DHySF*I!OZ?^2_~{Fd2?4WZEl$B@N%<8I5Yn0ZE1#sl zZg0{K^ChV3!gqqI5Ak3BIM!8VxD|RqDnb6IhmT=4;6fz1{klWmtts;YJq_mVo*xS2 zIZI712XzGEh(M&2a0R`8b~*D$ucQ~R!uy29JXF$jYObSyn#ln_UXrnl8*D7Yb|)b(hPQe)X{R>vpNsd}%wYHR;1tgyOhoedb)2B0gcR5Y|KjABjw zi`$$U-v!$~6!VkH6->Pf5LQMbfHC-Kv7y$FUmw$_IedCW~c`t2|22}(V5!hiBCFw_Fcd*ht>R%$R-*{@uXykk)Kw7 ziAf1usNQrN9!Ruqm-ct|;6x)m!eyrA$!Tc}QR2iTB!n-t z&(BLrN_4cfliMZ!0|fcAxQJHFnHd|`HrVT|9?MbrdREu+91b>*s^6Lj_3b?0u~LCH zzBxSo#G@eJ8YYad9y>H-FQmuB8tViIBT#8?z*GD5zZLO+Cj4}+h8qH0U0r>w5*ZoUqm{JI-&4E6 zZk=o|U~D%fDaqa4eck5k-wWJmYmX@2JoU>I;ky58>mniD^_|K8i`yX~ZGQg$Bf{%{ z3ulRc!$#uAeq@IF-!3gN6c0Q}4LjG~fK_yoPpXXL**NX&poT#lyrUlK|G zACz3yI zkvf!6{%6?UtA~kIbx3*mt5DmOhZrh5NA_^@6ZUSE@r`@=gmw^Z`wDeCi>mZW&tU5C znP+08o`W?x`v$kNyT+!gJz2{h9-x!vFJy*<6z+aXXvZ61=xa3#x2!v?na_@(^p*DdEXoB^?P_;8L`d@i>2ne;JMl!jd1bmKKU)Q;9 zV|F~hl&m-9d4!P8wy%A37G00won~6mCpQe`L)ro^dl+*wFi?Sn9$XldHKfSl9Iujo zdlY9v^UmM8+5~^ji@8ay?=(0u>ZF`ktJGG1sTa=CA{1Vl)5y~9>qbJ7Pg-K(;+;HY zySc=vT>__xv?3fFK5~TndYA3U?Ke91nNz^mP}1l$t6DJF9JlB+4m=PKefGt}_l~-R zZ)XJg0jAwrL$WKAE6u_S53kl?2e+$)$mPp$f%!MwoOB<|c79K40|4tCzsp1w)-S9@ zg2vRy1byl93;GTp46D||0~B$DJVO$LkMaxzcXT~%MtL+@PkH#%1kwNF<97ItCZ&dg z)3kxWC-pvv!?Pz?Mb3j{SM4dPPMbFoQ#p4pvVo`Iyb*QS6(3y*yj|#UMfLW&uBOdh z=L7gcrWQ9YW^pcbwr0rarGjBOWI|4~yv&FakYMedG`VIvs8;{R6pm;+@#BOjvTNkh(ozV^prtIMx->KyMX`lBs4T)Wm3=E zJ{U~i&vWh2n36KNEYV-T!@ys+tE?v`T^Xoku`W}GjYA`Y zpiBVKr5}RTdiKGZD~DD0+nsP9iptsIYz#VZzPY z#J0Kc15RM0ByQI5q>T?>jt+=1vSj3UZ9s_eZ5!Q`y@x@l4|<~h%r`vn_u>lbG|z_b zySq@?utTDU$9REJ>$wcD?uQr_2j2u&Lv8SBPEAk^x+*7EHn!Fvu|$meK(uO=gPCkg z(q+mQFH=~(^XLfTDpm1IZlenGxqLuN_|VaMI?hV1*41umMUq?5logNa_QZtaZ|1pU z?fvK+TwLH|(Ayw>`jm47^sXLMX5{k_#24zIvB5qggh*-=vt==>-T|$3jA!XAtyt9&(UU6f;gdP`H!~Gc2lrFfS;;xcxG5rK- zO=4=6@qH&n?J+h^*7V}^lQ=&Fw`yJ;+QH;Rqv`4YqT@@w4hTY_T{&(gc% znH6iIE_GfE9RdTn+-3X(^(-k5PhB$8ovoWZk4L{0=A5_?bmKD0!k_>VixK2*u7%&3 zx$8wjDox<-m);740-(%l3STk=tJ7-D@bCj?sNg+!RVF3bL~Pof-3lsVyBH7ybE4>lkG&$vXb^ed1 zj#`8{Qv^QW3A7xn9;%~%vxzpK+V6Ab<~<=!aVT2*nLdb9?R(8%_~u%ADP}9zKP6iu#0%sTUQsCc;aQW6 zr0*Y`sgMzY7F$j4`1mj!6Wq85GAyY~if6Vx0Dy?+t)9(q&ZFzjsysm%bt0e7vJ}f{ zcV|D;?OS?7n)##_YpXM`Xy}?1VhdU3X|S+{k^>sr=O&s$(JIzVCrq+tf?6R>yhFT! z4PN7_pAw&;Af49U(A~&q=oqHJuij{=hofyHikDlFnsxLjhwu3 zB#Glm6qCH564t*sp{ycbs2@V`h#tx@$zga72(({u5c-a47ZIM{sAt@|qF7W%=s5?Q zQ&4=dhFbGoVH7W)549-=$VT13Dzs3&Gxm3=OF?DpK4`~sF^6^sTKn4vM_^sD`7&={ zq#2cW4S5kK!$uPF8{%^>Cbg*lIZ0C9L!qbaABq_z2dB8)!`RxF;A&Me;`rthjdY=W zh$^bGss+KHIPldY_dVm(#qyfawBo&^(o3?DVWk*B%ZETFGb)p!5iM8Osxn$$KF>{8 zOS=A|%O4djc|hYsA$x_wm_&wK>rG3eF3Ux*0;^XlQ@_N(+DCiQLE%9`oJO_bL`jBeNfy&~^z7 zaK0)gTJoX?@DDe;C-J@tnFf0KwWGCjhTtxTV-x!2wmMt)VOu&1eKvQ%)l$u!P134s zx>qqYx3wr?kW+H_fQRNP3GBwNcYb>s5adR%4S?50~9z5EH-2#(9Og z2FZUmE9FS)p}qAdf{Ar%b7oh$?_X&ZPqG@a#ViT=yBo5xBRr%{j;?9$la$BL(da( zttB9<_i${ub4|cxZ}df}7h&7cO|lIn-HeaCV#O4yrl5+qnLtw~FdymKw)iCVj9cfY zB)46q?T8lsPYXF=34GMcs?Ox|;;ShTjM`?ui04UaQz>sGP-ob!G}~}JKMdz-%~zq& z#L-!nImPxlOe_}4>X0$yi;#}fS9XUMWQqduu-MuUteQP;b!REOoVOYC!W4SMQ|0x zFl2~-1#C^n6)!pFsXOQW#7QRbl5A)6JWQZ_n-d>)j-zx>qjB3gRbVplY?xq!&?1IG zko|l{WV;od&c~{(YwtePauXwDug%q5L9((iOZ~&f$ma&m6VfRC2(E}*YAj~L)JPb^ z-tBZ)RtsV%Xm!B<7k=-*-JKYEP>y>A?P@bDM#Rqd=wH?W6I8Y=7OU^o{Hy@979^CK))bj|3d(k2e;;CDAi6O2!U0h4__j`UbqYO$IP zMe3ZC%MJ~p+ovy6GG=p+1}DTk-&sXZ(Tw39^6{R9h-+v^72=?=>}nkJd`FE7q1B?I z!Lm0&Rtb1o74)M>+O~4a6UEw$Xx1IMHf{2OFHJ|mY+&~$i+AI9V{>2&XG@a}n@j=X}`WAVEjkWh2 z>@e{d&j(WxXkL|00$A6<(RdeZho zV5WO(q9Iu{VV zaLo%n6rm&`Cl%I+?|5n0K8VB6{k4=PgGI zk5o5zTP01?Oj^}TW5K}Q9$E+N=wgy*_ob!UEoIs*0zC=nG7ETWf|V6Bt?rSGyW_LU z2~Px4pEnqzSsK`13p3$BZms8BS9SFvmQxeaeN^U5w!hPdnvz{vk3Iq%JuRPA)#Pg? z{SpIg6w}3?&7sztqP}=j<+2{5NXe=|Cu>Sw21z=~{f;c*5#Zkhq^r?aMv+TU4%U9b z!yKSNVTOBmh1ViFDG=y{g1*%N)Pz^hXhq_wH8+VSvE^egk!Ht=z+YcunL-DD&WM#T zJo9?m-A%YR-RKi@C0PHvpr&nsDS1(rvPtFcoP%UN?azY)y;ZOSGCDn83!{UzUCc0x z4qrkokBgLw2;TPk+XDP&2W?@Wj#ejvlANitie1gv049qAz*)26px*;xl%G#ZAB+@7 zycoULS24i1CHF@s8K|e238VspqHTJPQ<;98JZb6M+*hN@xDiGnn-|1d9%sk(y#yW; z50UxS(DX*T=#0c*ZV}Z^hBp6%x=fx+_u$2M#VR#sF814vOJE*7;K%)y@;DycmxKV? zR^6-*jS}{wJzJi0O!Ia;H{?M`5;sK@NgjZ?V@^t7@rx9^6u8GZb?e22UY^n7Jaf_R zD(Yg)8fMj+)mjuJ4v3E95<#$)-*|Vh4!;W7d|a#+2Gcesr#);VPxbv61q|+6YH15* zR|Vdv6kd-HOAs}eFWiY*Vqw|yiqyqZVROox=^RI;5z2b!5J$Chs776`&g{PCm7DVTf?qy5JEX2s8cnq2 zmOWLFw}cbkU!z{_`nWDQ+3@XR@EJ4k#9Y-UOH1b|Or+xwyBlBQVhA@v4z24_(DNAQ zPQ6>x+D{lk>sTQsF=7o;7~4#Xnwgj(U%m_Y?7fe}jPjH9eOV}E!*Mqp^u{s%g_vFWJclYo%f7mn{l-|>>GKe%ci zz=tSpYjzn;cBuz816)XVygZ4rv+pRP=py-4b$OIzl=(6um{_oI6gC1fgsPISR1Q~d zo9=i7;IMMK7q*8Gd7R`54M; z1_Gy30S;!bR(Aq+@NiyeNq@B|*dWXAKP;!N0lC}8j(CC)#A^XD|3ORE6b4KC7hH^N z--A)|kO~!2qe~{t%I_{(d51Jd=IdM<{N00KcrEu3SE+nKHPd6V%bo9pI$B;#)N1aR zC)HOt$HSV2Z)*3M*ADX*$=VT@H4^6~6zJB5s~WuC4K>i|=%&Xp2AlV{%(OIu%WjN? z^E;R64EHdGr~>;5{@J5D(T1g{%y*j&=zG6;VHat*^x_%!WP(W?4wjMmI^n_0$)WDE zyKA8FYQ>4?6mDp|FVp^>B=p^( z9oAkHIPaO9Yt}pqc8Z&GErX9#ulz_xl1*~DpgiPQCK>a!{AbHn9{HMynt{RDj(HFf zSv+^h#>(2#@(_@9biC3Ghn*m{j^q>-qbn+yWKCoD{6++;PP^LMAH4TASgZrH-*Jb^ z1Ex_ZJRHJS`FWOE-1JLU!RdDr-T18#F8gcVc1Q&k_C%jnU+WSah5s448P>rwgkV!H z2)|9RqSJAeC^UcQ>{khugoQl%2EC6NcSQee z34dLz3R92Rl>Zy1;)e83TxGI+UXkh{N8z7+ZzO(#e_~Z-+$_On4md+iQVz7|*r7`D zgpZXxrmyJ#DeW)n_ocXSFVpqpzBe9;-ka&tTNNaUdo(cJYJpx8Rblh20-=gX%U`_j ze}THxo?#T7&`)fw-d*=(qzRZN2QrXdt&CV%L!g5{q?9sr^!M47#t-@m3C|K}Q!a^d zb#@M_zW6qjbDmbUT@M#W_F^32kxn=co2A&&9$UvS=&@Q*;_%13O5bmMucuNY^AK>l zmK@+%pf^Q$lw79GV)|IR8|P-->I)(+7+eOxqB8N3DF_JKDru!*A-B2rI6G#*k%h7xA0WbmD(%Ii+*{ zM}U+#5`DFVw*y%VKz4&ZEv`_dQ}M@0;BR6!iN3x?@``11b2@v*`)3dN3e@O#1@;GQ z?L7svI@yB=aG{VXWJs-nr&(>=CJ8PEMn27Kur#4wZ1x*&m?Qf2kBD@T@C5i zJDUyS7?MzmRi%k^w;gTKMAgNJW^d>7>s|nN*3~5q2oaTho#gVWKW)Np7fBa(E5bzj z#P(3~ZN10MSxPSA<>RizpZ^7*mIP$8@!+@bJsh`fkyE20A$5?OkylC6l$JIdYYvy< z_O*+etO@;`*;ySHXjmVwRfz-M@rBYcbu&r=Ui<9amA?cXO_A_ILiL<)m#591MPz6z z0XGV>&JGv3RX;}TRr;Kl8>KjCweDC@Qz+_XaP8!Q+0+5mY?YGE%Y)z=KA$$orLHRf zYuGjWxj)I=Yj$e^5wwc*WREYn8`Lvqy}N20hwZf{T{|tDr7jH^3bfA9_ z2I_9^SkE^YELaM@AaA-bvX=z6890z)Vabu+oK>HQ5s8Df7n77!jt9I1><{R6r6f$r z-nYSvC}F)p6-?EyG6tUeny|U81O?I60>hVGiW$Ujy>1Pguh^2-G-Pu)M}C$SYvim~ zRfcKEM<#w6O16JFY;6)Io3j}_MN{{Ym7^$x$(?e?aFV)~KB}I(RrzS`y=SF*?lD8* zVmFR|QqpgmX4lar*1@w9{q@vc-3TeA+43UE7FNF70_xwx`U`gnlxgj{x@?}|4D;T` z9^VfhhfJ3CUfV)>wMEG?h1ZX2 z=Np)l6jU_-)EVhLAbr_$y|>D9F8q4F;@GhMqts_0UIgV+z(y#^NU(U4uBa{!Wf4@U zk$Xl{JeO%eAXNfr`oQV+_>^Wx2g?}Dp_PlCSXdkRnf3MM2~MWUN-?9K6U3cZ*v5-RP-xUR;~I zaAKta_q`hBqc6@0?K1NOo476nk88NW_SpC4_|~ zo(_1GR{56r}1GT@!y62UmUaO~NyO>Qan?iB5St^H*(@4Pts6E=ysB0>GZvBS4 z%@m3AJ&9nxZ3?^3?RJ)LGcS*YWbr~gI3dIavYNi9by;%`M{90wCbPe$s7{)`dx~q_ zYADrbBy!$4#FNn4(o~upt5^Q!yb85nXFnoD!*)smy!@^#;**taroKyHzr}IGp#1DJ z=JUsQkrMS$y-w#7`W=-#r4`N^RHU!c*jPv}KjPscrJ63auhIuW*MlmK1g)NcS#*MK zFp*x~)mMyIz~4~VErilvb(YY<&^>&QT~ByVZ1m19r;)#}=)tg6*Q#<*0(%vYKk6tG z8A2J&-*1?(aBbQrgs4-dK%P2B=-t@Vf7Z2S;Z~^1A=Zksb4_;u(!cX(Y zmY65&$x>6ks+b>P0?AcX@Xy0V{3%bSp8W{ZgGmzLB2oWl^1a%Lnm`qE)_93DZoHVW z*;Rz(A``f1Q&-bGa_7QZv$mX%iwd50bYa+Mbh~nwj9!3IhDZLWrLfe!1v>vfOuYqA zTwT*OI=CdbySoKnbV~Z|a`F>=F2JOptaf}Dqpjl65GM_Sk>_#)Jm{3~W0U%%ZYb&-T2y}aC z@<&Iaj>Ze3OqXEMsnuLV?66EZpql)I zE_i$0Rc_Wa#PP`oLBaVFmFGKyc1lJL;;-3y!o3gTj{~YV3%a-IY;K&M){D95uFvd% zHbc!n=y_4bvR{dL@{QP@)~N(kFC7$z6>UFkCj6D&3#65|?|RX6x!?8LU%O+)TK$D3 zs`%@7su!Py8heNqBOTkCy${_T{pnX9H2s}tR-%W8zqBzTLICun>gU2H-tLPEY$fk2 zhy%&0gJl7vF^BOYhfVDO!qrkvKwPtjcabc7XHZ^9Qh4USz`n2 z-oMrfS{~11%B`5N#3)o42yI~+Cs2<)>JCSFfK0H`WL^QKZe2qdg~`=AEDTz+#WmtN~n zt*dPC(p0AJr&!+X-8I-$3L^j%;QY!UHl9P5@MA!f-FRJ}2#CrATZjEF4@R51-RB=_ z}xoBSkVJSq1-hJ=S&iwt+jKJUJ!L01fE<4v4q@$|bS000Dw zd}+-n&Gn;1N!g5=)0Gtg&^BpS?XwrazE&tMmCF)3KPKhQ&vdtVyQf(G6p zvv-A=ZO1#F-b-I_wlV$ctoq*?NvM^sAnUT^7Sb4aw~B@8zmCD_lL&QOwiA_*^T*4P zb!;l7t^?^;Hz$E^f$KomP7Uef?wg|yjk7{C^#?(!QEmmD#*cBO>!@Q0k$mx)5@U~2 z;;kcj6pYxBdk5{MBZ^!l*)F_Uli7$9OU_zlc$gV;N})RKCHP69mAhdIu%53+sw(h$ zHrd;~ofVUv4-v@DgZgJbe|Tcc-!eh7-Q6yB5$pB5rReqb`{fhNzMHXumc;;6vtD*q zmVv(wxOWKX=1pX*k*{JE)x?(m84x6ju8>NJzVSdFwGOP%-s=*Hz(++4g#>(eaY>XN zErZpIx0H7k_dC+LVFq|y?F6RihCk;aEV_J#4Gt=bi#G>|IQP$PoVmAlBLJWwO*~|B zQDK#fr?yKLu}5A zAq?Q6iD^C45nSqITy;;HT83gYX}35C3=?wMo9X7xYexbGCba`I^z0BRuj9i#^?DRt^N5NQ z5YTu-Rc3}P2)Y9j4Xwi|g}9;eUl`3nsPMUAe*qX^Ery=2dwzYa;QZA4DQ8wVsHCe4 zcs9R~v)!_n;AuGAWZZ4_zBhv+)qKOYv2nlYX1H-~^sXU9mBa6UvO(3=PGM?wbDNn; zVu@nB+`LD$6ZwvRv(I~E>PSA|embifEW2neqPB7d)Mpg-aha{%PbByGZXO|vtp|eG zf22g7cJk71aMnMK$O-%wwWr1hV5oC}E>2iOOBXUhi!Bi6*ImYw$Z(hOSYe-DtjwjP zq0zRu%7Nh90dE4bFH~rXRLp(Nk&t%ljjhV~*HfwwO)iy2Hw^rw@S*))oRDg?N#Wt) zd_8446wGtFMQYZvi1+Bpjj1@IF=)SImBKFC?NlUQ<@Y|S98i7zv ztkEcasnh+=(ij-AzEFUbFBEI?$h0Ap@u<7Er2f5l#B!zR{j43yU$^ZGWB=_M0FUWAxO#A zt$5s`Oi0weE$&-D*~6B?qdOWhC*}Kcx+licmOtq{EYd|Q`C0BNjv|e6`2^Dn){*!D zw{EXr-QP?Xf9m4PP80>l(Us+A-|bX!NGA;)9JvvOZASJQFk0V=5x}4!t=WF%oX3Ec z0BnmI*VKR1?X^yq`e^^TvfsK5=46+Kje~eK(rb@+1TJR^XD6BO{cI+9pX?ZA(oJ3% zFJ~K(UsSb(`l>OFPqC+MOzJ22ws8&RuHOFIa(S2DSY(cx3p#(Ex9zAy&ycQ=|Bnn{vXj^Lgy1tHYhkbCgJUMAVRgb7s zb&&G&c^{X#cd(q&0{P9=V|S~mm4RF&_o5gT$6XC)K&}MrV2YQO?3P&f7`vm*cK5SX zh@H;y-sXE-p_ZB};>D>E4DRw7H@M3o#vv8J+)*&{_F>YRT#$w6Ql zR0OZ-6Ts!UCabr%hR6y2Mj)%J6=Q?+2)Ymx42{CW0$JkV=hDbp44O{w;bD1w-iKP< z?>xMZdTQ2V{NBjLpw94hze;3(KGhaxgz};fEQj#k=D?4d!rN z%GKR-<@J{6)_7gaCHa{_iWDOv+7@>b#`h0_5yOI?$0Aj6%0zg8$kl0w$G{)Mx6%pX zZO?g9$Ir2^X^pz6F40f(`!xBO#K9p{Ios|I>3@VTUa7e0RS5q;wPiTlc*PDvrr;Kf z{10Ah=RqNhf}0f!>ZTihm_8t73V;?5j9(BVF<238QUs!4J||2u&5z2V+jC_~6}GGc z`KnubV8jN~tTYQ%Xg^DtIxDQt(I(4P>*}Kntbw0I3FX1$4O7s8?6uIF`INMjLkmuO z))ZfhJqKCRUp(4~pb8CoB&kBRV%Lr=fQpVznF_r*Te?{O%-J1}N+A}fnD3YGm~$S# zY0Kb)IMBoQM7Q5L7;Pp9NIo(;OL!0p^GeijblR)t$W$N#ydvmP;pyZeadM5fFHmtY zs%`$@JNHPcmWV=#nDrcf>9~f#sV>Wxzl-%54_{J$Zt*Lp96bHb3}!1GPe_NXf4 zzk)_I?&s7j+y`S%!<*Q9Fxv?sEXm;iN*&MLozFb~dkgGz^hPCeYiD^8mRz6*J?N0T z#v;oGV-BfU1fS%Bd@-7#?@c8I7jeb830=)S6$$O}3Vfu)E8c`*IQ`0|@C9q%+< zNuLQtJe-7l|?KBzB&*w02q%M>TQsYSkXch_gr}$F4-9)@#8PI;c&2{_* z?v|Dvjz$t5vR{@~_5uyjLL5HbO}=^i{&x2)cy~0L_-wXRBd5HhVZ3%XnbAA7p*53n zm(AzDbw#Z(LzKO7wx6j?EPdJY{P)I9;T!=e``GFCDwB7t`6ZLnT<9yFuTOy$E%krp z#7I<##Gem+NrA+g_~2>5*B7lRN!Qg@ahS88O#e)^*H>v>e$=VyH)zIj`o5%|5h3*bb-ISmd83=GSq6Gq zfjf;0PwS*GsmflRvKe3WbZRyU!)7k%K(!Dv~GRzRat4BHM0i@kB>nY6SfHp$Vrz|s8ngF$ZO^Rk9R?nOi0HE$R zMUx8eh;$!Y0B8#wsAXEGQo$9}^9r4xdHLx9j|+oUnCFEH{#88^OodJ-0U$+%AJ=Ka zEE;|O1XI)W_b>Ns$V|v7)1az~_<&f9#gPjymITuJMEdR@t^B)==E_pYpCIn>o z2%QHT5h4OS;1oZKU#jWcIVB3GNI$48fGoUwp$)BDpD8+S5?~)MJNq2#X}op3BcH>t zg_&h{71w5)t@>sQxhj|!cWeZ&PK&u9C|kdsIWi4f#(PC8@S#&Qm3=&4{pR*J8(T2@ z2svW!Ib-kPjE3JCS-Ts%(hPD`dYemrbM!R8?kv+SeRv;K{|e8c3cw-8XJoE?o)dRJ z$ihX200b;HNpqY;eZosI$(o3qA6zt2K3OU;%Df#96$EkM-M;7A7I0NS!XbJ0{@q)) zywPMFeUu&02HjgE83plWx7(b?-Zsb`U)&BHac^VsGmPC@Ytnar>gA+h9VNUrI@=o! zknl@YUUH*T|B*_4-PxRBvfWFlU`;5vyf6Kj@u`2ubwhbM+wZFkr|>RV-);~5+Fj#@ zMUs?Yu`ip(SiRfR?7>q{c|yt)(pUCWfwtmfQG`M<7lw8rNr_4q_)Y5HtJGa>+eB^z z*CTl%3059n%>N>C$6+8B6hO(+Fu}TU?3HpCCSOPj!E$)IACMfBvR<1uh zAc#!-iAo7q?^Y?3xu^XDlp^{8Ol5t&ALTMjzk)`%XP?A0q6j{iI?85;_f@82nehkvoC{xBJs5noF{h+l{3XyBd5j+fOvF7ku3xu8en#aR?%YRn9NrLUbNy7WDbZKmz<8KQ65oB8wHA zqoig}Ipj;VZ4PBC=Ngyf%-2M5yshv0L~khu7_DHLa8m1dOppb*6`G7YKgAnVrpr0b zPJp^SYPb1WhH~*y zedZ|VgpYc-#gQG!(dV@L<$O@5;ruidLH?B#{9IWAvNyie*sqC_L7w;YM^74YW4W?;5E)hWFc2{uy0I#*t z_5-RiQyS6mnDbcqa(dN!@L3JzaQpaN1d(_?`^3lN;-wdeI(rBj0E*GYyvX|#^{~0z zdmS>!`;`4hYU*1LF`JI%)CwEiv=&}!t7otp=tX6V?9Y(kB*^T~z;h*TTG#Sy8g8EE zZn<3(i_Fw5J9ZQOaVhJ~?e?-51HxE(JrMlya+P>ERLyieV$ZbGnd^iL10dh02oqxm z?#vI?PF+#xvQJY^w0n5H+#d*S{vD)|xNC215T_JvJ^(J}njMo&IxE~V9S&1Ag!Me# zr~txFFIHMf0I2KA_#8v>FNTAj4|bxS_{lksgGOfcI7Wu;?lUhgz23{7?k|zGrOc@3 z?;WXE@mkE2tDK!g!CF7i(CZ025IGl3Q6EpgM-I90MOZJ|R@EjZ{5@BDPkA*F!Mck?efoG*Fv=fU%Oo-1kNr(CbIe zH{7zU=3-AJ00^5C4b)PB+2*6v8O z4qk0eoio^7#!CI{CbIbewkhrVE!|N3N19>^`|vWF1-;yE89%%Ds6^u*Xbe4RG*6tk z6=1TUCT<2P25?wdad063>`uj&*|Vn!UsVk7Qp09T~WaUVCh*JHa3_W)95Hk}H;R4wCo^V1pa)T~qmz5ACSBXV?9I@ic`nZn9Op6s{~@qF`-OyYgvmnM|myAt)tVL9GIY5#6YLe>TZBL2W* z6}ZMKnf!bXu>#ilsni2({H_o*jt2H$6^e8k?PV~!T0ttfa)JGy4-pt%Hs6Id)1U#* z0+zEyjTJAzU_C&<_@FhO08NS0Q0JZ_&M0a0zLEN*+H`VIQ8f-njA1)1)KY8kNvbvePXWHneh8f%OXz(0skk)K zrG%hfs7=qK`!uok?Ev3};H}j-V{7N5u{*CLdg!>v!TF6`xct{5Vtzl6K~$D&z~l0O zgaoZ?hXEjDX>*C?fh9dp5?i)ng<`Cn=Xte~gk=A7`bY07RMCVqOO1vJ>2gM4eMg`y z0MKb&o{3LXS zX58SK_{T;P%HA0{L^>Uf-UYH_Qrdr&i)9c7g6N@#WM3JwOklcHqZ|_&e>+pwocrFh zNtuv>R8FCdf}7`eDk30e1H-=<=JV-A*r$Dgh*BFx?V-tYM~5^-*@cJfAY)T)5JwG) z_TJ3a>^jm3m*^LD5z00w9;4Jd5aR0h;(6|bKB~4u`pRI2aGx%=jqxz1hO_y}3-B#K zr_vImh5)o-!f;iraNo^7+kE@_B{2mO@cRl4>7dy6-gYd<$*2?6;rL6-wtvmf4{rTd zRr@6!MPvJ}+YuU|n{Kv7`yi4X_#cgBYHS#em_gQ$eM=hceT{(mPH@#Nuk`s-Q37I4 z^DDgC%U8?=`hvV00CF+0zNoIC6kidm{6ZbLNVSs8p;ztTn*{!xs8UEArA!?+S6WSU zL_ZKmX?aSdF#Q9Z38|RITD|A$U-9mMV@v;4^u17`Slq z)G(Le?=H8G)G11(RO@LxTHs=s%eSELr^16SSL@DFDIAw8{=J25hJI$J$wM+3s*1dR0n_zQBXINM zOHWaOYIdbJap<^%ochKD!6cd|72t~yu~a0+*c4v&ARBPH_E{@o??M@ZzCY05t!>t5acIr_^I*gy zMtE0C49%Afa05UFDp4o#H}8ED@Kk}JNqHlQ7~TR|(g_DG2;sssN79yUNmd1l0;$lq zrI=L*APe&fuskayO0vK`itxyY7A34y=n)(TqatN$Cx$@JlsGMJsS8$WN>;<8)y6Vv zFijF&GmGc6GCrnG2(y^WkXJk(92_Jop$6VMfV}O}mzNh=Dz#rhYkAdwa}Tmt^>n4J zb7{AE7LMthl?nJP!QJ56^GB^uHfX6zY&v#;--s`Evb3=dwgl3l69wCy)jW$K`Er~Z z&53A%tA1+mz?p#qgMi1BO-g7$9#?ec-0rI-4tkjD#3j~Oo*$&f$|z4Z)Y;HGb4IHj z*Mu%BP4OllQn*e8j^{mQNF7fahnvL=C_cQ*l#BUw8%>CvGE;l;) zT`Y%l5`BzFuyxM>k7bk1<5GiRzlcD+&SjR>3nQi)TdmmCW~RW*in|TR^NIb))Ukx$ZU!2ACKm?WhX3# zTQIpBJb7_p?qxx$U=>RVwtDJOIu`UZj5~?jaN=R`LBiY>wcXKBdo=v61(F=lCoe&9 z77Zd{I&eEfM8q=FgD?CA7fJh&zk>DiPFlX&P&eHoiU6=t(GB$FcgMD zijRE2S7|5)nW=1^_p?f_ zRT&HV#85x&8=o~Y{5ZE&Lr29G)b75sfDA))%Z?V_J-(Q%n26ji)}+A)h(tx~_6&~q zmIcX%QTuZC<+8|cdaIl*b_2&gnHqhffe08+>+)Y;@^8pp`8$!Ha=4l`N+6WU-Wj6H zFlD_dvNJUO>lv^6@X}j262I~2s?Tdm}vG(FL;XHAEe}TX#d|GPy^@&ewJZAI# z-TE(TEsY=yW$ksMzC2*gjrd(rDIAyUqW=yX$ddt8s&1YY=We06#x5gLeRQAY=6k&% zYkt(@Lg$|?|HQ!h)X;MykgGWNZ|JDK_zwA4c zVQ9d58$4fLQ}_6F<70B5EE6I6&lPIMe$^luwh(fjyd8P5u2Q-vEqo~ipzcj#v%FzZ z8c_zw<5Orde?ngS_sR-C;3*=1n8sjX@7u#%;exd@V};t=xFK8|AV7pdC&q9Aw+uqu z2O-jK(j23lDOTt0$@DYH<$D%fCFod;#snAhg0Ex(IaO{1fOQGd(`9}4{)_q3vz0fs z`4!sKOP+c}^Q<9TgiOHy3GF}b5AanjN|lD4Wi6}iOSM+qS!0XA)0gUFlgO_zf;t#4U=gTNOb0eAVOov7v zRW28n-iThsoJW0LW6_xqB5j(aZiAJBO>G)&H95CChA2Pp;=6;(<0Fk%X5RoaGV#N~b>A0{=vvGfK-n9rI_`pKar5o^CL5 z6NZuXGP!3@oTdu!8{y%DR_Nte{-m)=#FD-90Y013Gh@`(Y^ai=zl`RFi)7(%M3TiU zgghTxdNPU8zZ{J+W$9uTcSh2hkDw6@nM zh!Ir3(#6i+qefgZMln?K_)O&U-ftbo| zjq5mR2LFZxwX7FKF2W${IF?3l*%8IdjWDV_4Nq!(bM~eJ5E>NI$O+&Pci! zoc9friKhS>(fOFl-!ShFMR#4wjdq( zG<;O&8bSD~e1n{{7dpe&HeTvY?FxTJgp(9t!@+oTj(hJgO~p1ylmsanH#9NNvm^5h z0ppvzV$H$ZL(OIHHYID=b3x7rM4z>wkDT))@v{X+1+VejIFY4>Ha6Uk|A^_Lqqb_9 z>*L2H*@?wseIg6|^uT9Qq$I0et{;v5!fFI}Hhf|wZHYB98<>VzqKk+cQn||6~fu|A*QD z;iPapgi~y8Q?MkCKaDdKEVScKMRILdOG6YQgt@B%xm|mVYFk(B{sV7N0i$+W^vU$T z?@Rl_%?YjpMq0tRpJ8ES`#AKfwB#+cN~8*%JBTI4!A9Wrc26IS%E!xJzdq~b1F&pG zw3^P(*C&0}a8TrgCh|J0MbFI_<7LSEmgkwtSDO@duUKZKqspx8C?+T8fz`_&a|y$SR5o$NIuy^W8U&mXB{OI2sWwO$BL0 zs?b(xQ;>#=MTCc!&AaQ#5i_JxCCkavsIjQRK$3NbD;?fY{tuRjRiPd0W18b*0wj*~ zkqg%wFF{~P%M?#n8t#M`6dVK*U{plDO_kj0@~o+srh!jZ^Mk*)ulg+kDJ%Zq5l0BGzWa4px^goc$k) z5fS-KG2g8W0)Z0ciU1y>=3Vza}y>VolQ6>A4 zcbg_0tj|-sy?nhXxtRVaI!&H}3z+j&sj*DN7&Y?giA{aPyOCz_(sO#E_#a0zFtl4C zjUk9=?^QVfwIAZpb7j9CM=aVO{#6u%qkZl7QJo!+V6XTiAgMpmsohYi72}790B*Og zi|~gOlX(sO1RMZuxVD6i5@U3H`a|R9$5T9LbI0L`q?g)x7iLw}ZgLIQ@;BMP$MI%w z)$jgTtTNvSOZGP=Qj~Y0Xj$vg0wg4A9Yw zPprOr`{+(e@ZWzG@Qt?2DN)#R{xSI{#s%W{G=tfD&**JfmH(8$cDT1Wu$hU}r4&XC35=3~VBS}8$R|6|XH+|@GWYDc+*Vf5^%ytSD9e+(t#DlFk z#k?w5n$+>VVGm_(O<{BI$Tm_iSsv*Fe5rovk3LQj^w04-3@TzTvjc#mFQ*1h-Wd4% zPVVy4cl~Z^R8+KhoIU1nmspv+IVKf~T0|WmSwQ2OW#~VaSg!Q;zS0FtPo1sct(vN& zU5xqR%nxd)__Cb|T%TcbQQKINbvA|gx_L6~`8htf?C`iW#WYMZ&eBvX7fZeM?#W}v z2CPWs1Xg_U|9bw&gws5mb|9i^xdx+F5Qi&Q~QH@lPo4F@CP(UQfN*dk+JM+I(v59I>ccWKfZ9 z5{vSCOQ)iB3x?e$Lj$j7?9y72!hfav11&72#S|hy{tc8g>3`IpR(66ZwV#Xy8s4%eZY#&|J}hS2&ZZgdFSe(m zPNJFgz5i+70`U6pf>eUE;p!GPmAv=4>lZ1)e{R8D85>I!YY8-jvaE_n+6lzJJzWd* zs+E+KYJ~3-#}D1SHzki{wM*L}zywYK8wN);_P_p*-ae3uN-FI}z7X0gh$FrrD|cRs?0gzHE%H!22VS|)tJ$k;^F zm-py3-AXsVy{R5Ld+fhSuy~>t^*Z2zeO94CKR0l0?!A2Lb+<-4h)1H2#OuV$7t-&f z=N*x+EQK1uyg};l*Dti9C*Rv6Cv^&Ec?_rK;4ryE;8rvXJN9$sx*JY*&9Q$B$K?@I zMKG7LQ&EM=?&b+ui6I#lbkm)6(|$`aOiInPWy{R{K(?*d$x^bE6~dX1VLIthNOmR%WI zXEb&B1Riz}{{QZx&a8iF2|Xy7K8<}(1@|u^U-EdOj1NJh1QK$+RFtCg?0Fc2fPB2S)3N42l{1==yz4hB&B%E?a4 zv(l?E(zw>>lDaM{;~EMZ%3j?c7l-<+=p`c~B`LPkbJ{<$;D zZ2oNxfu#O1eeh3xhrounLA;EF0u^>9YlI9UJUp$hh_kpyfFWlNf6tu3Lm; z!Q24pKUf3{L!7m2tqC;_iS*KHawfT(yU_$`714w=+e_>|p7q-5@qRt;cWG?K>0zYU z{g=?r0s2X~fdj?m$I0P*q0c_`Yev8`X%2LR$QZX7Nk=#EsS$tK=g(0lKmBWjLJ>4# zf9?l*q3O?ubv{^qV5rEFO>-G!kO%BSIEbapq7K@C9s+=_=X@C&^Z4C;DlRo$qgB_^ z>s-ery_uRjS(ZwjK}ot-0K826#*08cU!_v(c@*C}(7?XFk)Gl(t`kcTq3QdEsXRo0 zqZ-2gb$WTp#0GRG=^E%l!5(uz&a0QQTxp``qUiroMFx8)+6|q^F z@dcDC?1jWM_6iH342lXwfU^|>8dQyCHr~vur}!Q8_3I@a$%2iwaWoO{<>NQIW*o@> z=lfNC%ot_tsXO>I%TQv9>a19@Bnm=0&NPRy;I2aee4RhQW($T~{WCTa(~he}lM z;U=pxo4uqoc%nnV=g^0T?@K$EE3}9;Mf3fXD!FWHpjFe3piuE8QLmk63se&q&kCXLCWwFQ{!a zT`#njx8B{qxx~b58oa4X0kgidzUVEzEX%8boi7G27IN(5CokkV#Z-pTrXq*a@w$Kz?? z13NmmEfKVoD((|5p}Or^SkAwl(|ppOE#Etx^B85m(nW@ju8|Vsfqsk8yI!iY!lM|{ zL-YpKk?O<7h@H+wbYJcb{kaxb)$F=7R9OLaTG(VCH3_Di$x$-%`bNxrRNL5LXF1GL z%Xch3tW-u~?zjEET(47e-}<`;CA+eOa+MSfSz}E#zGFJ zy$0QK-K+0!#BE_cG4gL*{{*Y7R+*A}do}vpOb&4b7aX`OTgl8H|iN&4n1NO`%DV}SEn4=Aa;zdaHh>p6s{lvYmJkm5C?kCu_*_D^S<|Kz2 zv}L+4o_rT@X%Vt>M2_CH-4`KSAMRY(LYwbkdEHLVt)sIAg%W{pElSYunCvcQ=bMi2 z!l<71+v{G}-8Q8Zi}~fWdyN?5ULkHVm~v+>yin~zZy9$ii5^U6kS(NhJY;CHg!C@E z=_Epz0)_R9*#l)uQ@y}f{YkrzlBoig_lnt3VylL=#}@o{>W7UmEQO%)#p27J9zT_H zTl2Z#(0CWOU++(g1H`06(fxP3u-`zU|1Pg!;E<3Mm}1M=R;a+FlvFnuEeGw`aygUY zNZ8%!Yc(EWg`T6hOdYITaxSY*-HLFbSBZna=yuxw=7=2SovpYpsE;gw@9ugxvz^G* zig?O+^q5)%cQsH)WJ{2B{;wwa_1Tf{chpoPuRU0e<{_$Mu z-t>qeLUnCD5po6L7IyztGD;mBNBBZ{%LX6WP8@%{8I*KIeIu9I_g$n!g_U#_PWN=4 zx-NE!VIk^uUou&x1I(mrl&XWc(b0nF)Bl8c5ThG85g8Plf2Dzi;U=@MD!k-#TMPW1&>D;qlwd51Xl zA#CD(n&$Xi=0w4@G5>1e|3htu@nF=}**(RaTBGCwWoBWX@=-2g4EDa7v87L1B#xR< zE~;<%ezJf5Gtuo8)luvyrmniamNEi#v*ygWX?Efxl1pwdYIY>b`$s1ff~khY8V&kp z3wvNW88;mvviH*&q(}|dvK<%aQOtu(*oJ|rq@sGG6^Bi?Y;JmT`3$WBwfbUeB0*yn zmu;xxtr%qme(U%NnBNDvHqP&TZoIg=eR%W?Y9mD#Lt-C{g<=5Hb zVw0HPetMVuA_OFOm*82Tig47gT1uz{pWy+XR0_~*oHKFf*l?C&o?bq?XG>??PP1=Z zmpjlb*OV$`CG+U7@9a#t+%#9?K`Kc7&ti|B#GG2GV;L>A@5(DrqVnzQe^~Gk&kXeC zv>nSZK05~&G@1eDEgvf{Rj8V;F=obA5pDm)Y~gfA1@w#=`x$FPcjAsQ`FvRXQ<#*Z z6vj=OkwpW+w6MUl$+59qYg=C)_~?*n(-rI}$#M;BVLUzirSo&fuo#Wq?C0N4zUi zc65EN)1o3jr@Da&0w4EcNFCKBnd2MA5UWEm)k<45*WmpSG0{pWTr{h_~+(?NdU&j{PML$7yNNl&y1uk+3G)sa#oS zZ4@q{DgGa}JoNU8pOm0Pmk((k`!PW%wfG>lCXI-MbSSW;XrMgy?SS3rl4EzVyU~-mp4v^jEoGB$H6wTJc{I=6`#g@d#btba;YYN z;QM!M(xsx1B2yKePE}QPy6%&*AV)q9GKiSF@QONKgmx)if)WNAI!P$+&#wp?nQ7|I zK@-8uALY@;pIDxIkByXsApqC;g7{hA?l&U;mY($V^d$YF8KP1Wl4eAgEuJHkZWtS_ zUhK=ixWC5sI_6!(+8ikw9fA-t+I!~0-4nSJK*!R%KjX8&a;xv|s z7MD}8A1*=uGspV)6ibpfFi^A)3&`=-;CDKd85J`z;kEYKJ@-W3Qnn)JDNBf$j*X5j zMlNu3EjVCEQ2p1sj|y!3CH)aN=)k7aB%Euca_-AuaUU^#bFa(H4cjLH_9~aITi0QC zY&Q4gQT!rVF5X+^U!&m)N4R%bqi?%CyuO9ftKgn87f}4dwITnuJ4i66*%Dv2O&E1+D=>NuRNXci&H_y=9#ih#qPBr+vqerP` znjHN~@fK}BU$H0A{V9KsOD=E$gBW|Gi}WQ^h7u?%*mCsT!nxc^b+~;v(=8^ccq|-~ z7qkHTuV^8aF%$|^Y)|&x&3WYmfDP5|-ab!ftFbDe|mx6WzZt^0Ld_(*HC8m42E-4K+tC6A?dQY zRJguURAv{;y+?`=sDykF=r;T`bQgu^!e)+=p$-iE7vbVMf|V>7@p|r+F0it7u_&x5 z%wBp#;IdY&f3zDuk}Sv+hRRx;`#W`c%Xl+a*N3RzS-jXpVs5{(zXWeFa zF*L%$NkRn+2Iz~Vwl((po9=%5<&H@z&$rFn?i>6(HLG9j-a3x+so%hr;J*VFUH2|3 zg(Yu%u#Wm|pWJO8zPq@`oUFS3iK<}C93oizz9&9-t2LHN<;Ct0SjL_4|GWU70afzP zBpE830hfmVo^bwibeVbxV5h`q^q!f)()Z6tL5;l!xCblbEZF`rwR1)9e;-q6B{$i5 z9qx*|GNm%huQhZht}oZ^em$kTt-h4aFPz1;+F&>M=x|KM(g=g;Njl}!Su4eGagd%f zw4f^z0%F8Ov9p{IS7yDhjw9Nz^29Yu)2V7MzJwzKWPp|G;4T{P=h}lVTN(k|f@tAk z8nQZjEVwLhj};wj{r%vHeOA}CbYAK0*u<@L^6M<0ft%g-&%3Wb z(MjRwWIW1b9Xt3;9)v%E8!*em%BqgE3*cDkesy$}k?5a)@0@d8&8eUd7R%kTc%8J| zM10eFXm9QC*IgYqYaJn@s#iHf;f3`aMhbSLCZWx?YQXaAIld=R_KPG!qBA*8CYg*~ zhLRg($w?Q}XyF4a<|2FB8mdAYjr`V5^3 zUhnC&zP{J8I43n1L0W-VfYxsxIaYkb1epfQE7KC{4;MPNScYcF`S9l90tk2DH+7C?jV9DM=9HhSAL zypB-gbG2K0_Ixb!bJuYfbx_=gPuu0&@Y%g0A}ljXIk%IXy<;pXQaG6B`d~ef(;J3$ zEnAf-&mbR+skG$_1y!B{xRUR0f1C3sfTo3;ea!Ai0XjeSF!0lJl72i~WedM>5(Fb@ z2P?$-i#xJ1@`Lu{MtYFyUy0uIiF}-Htk%-w6+1dNm=mkxBA{;OO`HCLwc16x0{`u_ z3JbJ^2s4O*M386{DC+HCfjhpxjw=DNWtRj)_${$_sIYl~N`oz1!sp=nF9y5UadR_s z>)BJIp}$4MzzOZ>(_lh^KZQ`!@jP>Zdk0bS+kw++H8Okjgvd)dk`pXAvyL>16n??g z;0hEmQ(I-kZ#_kU$~{6W^5Vdd{8K}K`fQq-Fiik=RKR$Tow(U`x1p>_GGKmXyFaIt zS{TW6%gf;=rqY>EtIe+D=<{ax%Xr<=3Fhdmu`a(dK-Qw^!FG11B)-Eysd5t<@ENhY z+?@yD^w$XDwXVGl@&3fQ?flb2@2F*jREt-lk_ndu9Up9y&F(y=Vu7C+c|th!UQUs5 zu5cxPdul02hw*EKu}AUGrnI_;J95SNz4EC!Osw;ch77s#-^orb81w)7k;f>&{m2iND0hkZV_z~Sz5X%twMyM{i`EVS6IB!RG4<;}-PKhy@bO?C zv=cS}T6+2fFkMfgPZR@(LTr^D+DsY04xF>$IJ7lsNy9=69oQ|?Ry0@#ms`QJIz3oP z|Izw_bO+R-112o~o9Mqnhc-4g%+wm-T$?N@z)ZhGV>#z(YjNRLN({YhZ+W{v*M}F@ zoZu!#sZ=%h$+&Bnk8jVS4V^ziJL|Yjqm~sGxG(Z!=?hC~oR)@Aw*m}UIOr>=>|e`P zvh?!ei|X$xZ7bt0ce(wJ#PR=;^^IMYHBGRmd)l^b8`HKuZQHgr)3$Bf_Vl!E+qQYn z^WJqo-5+rFI=gmOR%JwFBr6~-8zXgGRlLlVnkQjlRGCEoi}KoMQ~@Tq|MbJ%Hb%a( zctwMKrT40`rjcp`PV6kT(=5vJ%ddTD^#4|e<7v8czkvlGA78csDJB)#BAwJNy$D5O zH8*E>yZ!N;S>dnBsQ-%G`HWWzv zt0$T9>Z-U}B|9f)ti92xi&jIlW~BQT_j0L65|DlwM6c=togDLe+&E}kZTa6*opasLV=G86 zI;A)6g(f-(U5r<1)1UM_Vp|#3Ycw=8!ob2_x@o%-_@xmUDR)`~Rz9dEYzTQ{v;tb{suX!tGtpOvk86 zvVwNm z7Y?k8sveWlN{Lquc2NlaUs-F)AjG7og6K{-ukTm6NUQK5_dmkKu&@g${0r5;7eGAT z3ECUdFHD=4YLRAzfef3iiiK-QNX^EUBx3yS`8?iJzfbzlD1uX&u^~N7)F{1$$W$33 zqJtAvEJsb3{xw{bBseH232(o6=iMXRdZz2R24{`lDlG0&ghFY|oc4dh$D&{5%5qe8 ze^n{@nRz0FMMM&te8iMOeo6xI&gvt25t?KaMpTFmgS4WXUjVy-rh$e*;{m7V@8d(xI6m61t zNf8>qpXo#Cy>AER(`M@r^ACA_L~bCNM#XCPyT|2vw5kbfvj%6~3+v0s z0>(MSmzT2yee&2*UDtvY3ZM-39=1A}120~1M#F`WK)dPbsD?sLPt(-Y4Q=Pkz<$_s%Qhz??r`V9`YmjX?cPMnJLGEg{8N-pys7S_0g65>Z zqa$CPGmM8YT*AcaR0pfg;D|B_^zfHWzdF%_5dcKHRK`BXH-kDqAu|N~m6l>YGJluK zy$rK9KWK^=X<6N*mXE1u#eB2tg_FL2Lp%h4zE_4=eIVlR<@u&vnk z4^`gtXokhMuTKq#+=_)ew9m6EfC*!^md@BmTK+O53?qb>NXg3CV*Tuan&m_oq20-ht+XNXLaHX># z@^`+6;7P(iR7gN4=kq5}=c=fZsmy5L+H*jToT-=0yQn*^-CFhWbJf8=bMpJ6`s)GT z*U3ysml;6R{IJS-OaV*Q#r0!J@kZ`PHQGY5%3WG@>Zv&@j)4Hw7)Ak)EF6|wiQKc& z9R`Wra>4U4TkdqxKA(yxL-CjX7I-FV`A?YB{TMvRQx>0Uk(`6A z_IR&?(3WqbyDu(cYYVdZkX%{2U|9xRa{rI@al;y8AeO_+(4OX9%g>okdDNGlN2=of zvN!Vrp1&Q+9j)VSsHL-{8Z2CsOk*LMD4(J@N$s8Gt9z+$tEJ|b#Y^SWVgGQ=%sRaU zb0%M^Yd3?ug2iKx=iE2gU+VewO-=yUd}S_OAT%rV08g9oM}>jw!^AuBA?ofxyS!95 zi-ehq4kp}*>6D|ji_VW6C)}<1=5o(V2?{NXKo_9WCqxt%8yRk=q=qJw4%)K|@)Hds zw12Yq4INEPhL^_c=vLt+r>Ia7El%%8w$=Q6H~VZkhXok~fT?WyCAZdJZ;;xWPaxVP z<{uJHj{>aDUrgtwV(8y&$LsgG?5l%Z_16WbLbScytY1n~^n8{kW|}CQGyDUd3!Rn3 zIJoL*apN?g8JcQ-O=(EczpL0DqYv$O4FkwU@*j=xWH+A%z8^Zs^3B0IdvR!*LM2hy<A+hwMEG!*nL8{k!u) zS!+HNy%=Np#u|#YcNO!`{4eF052gXeMSsGz&pAfCWHfaH&%~H=)6;cc9L5I{$Iv?R zFB`kO-JRswfujZsx8cGm;4=Q?Jb?e{&yP>*HD3OEUun)8A&d;%gd-BL1krzl1AKUs zgk4#II@I*P`sOLUCbyN^qt4`>gny1iIh)i=;_3z%-zGcaJUhD=Jy}4T4o|7eZ-j)u zbSLk(r+?}X*R~XJaY)rxF$T`(MZOkIw_O*0VV`in*$oX|RH*n`6a4GbaUH7R5>VZy zkb_^{*^C|dTh9B~8; z3y*x2nfQ>T(dX*AAbK2dS4v=XIRvt z=*1E$lj(IW3n8{F7emcyt40+D%Tq>?0x3dRfx~H@(__q{%on{V^o1HbBHDq^^~HbcgcB!jm@t7n|4KfeyP*@?%)G=6WjAcYQ9 z^7k+hwC&y-|6}8-i>=@PN;Ov~(#<~Ly!QO->mhED`jyi2O@ZH@=9mZ@B@!!(Ass9b z;?K3|taG1L0+Hgig^7j|40t=NCU;is^v75+ENpGD^Wd&Sdk$79GS11;eJ_WtA(~)On;l#r^E5*H5@Ks zw#tFyn|yo7Q?#HGjcjf3K|a1+KOZuQ49SK2asT(}$BxP}vb^}ytuw>5|BbQF(&08m z?pnNuhJEH3dVQgCtP+&C+}wU-4mBFgAsN@p38#X10DoCW5*QbpE?UnQ@S1{!=2o=K z#%GzVL%jWc7o|e1z2*+Vi_FhK@vMJ0-z>KOag^Yw(ruL{IFTqR)+nOw5{Ys8x*9_3 zOsX^7Rqd0loz1^uO9Hd{(0@D#^>O-EQ0Ax17Bw;RhsyM@IfuBMh#@!K(tZw7-l_V> z|B_~*asGFdaqTp<|J{I3yh>Kf14xqWvctuye!K7}+_lt@M}(D^?AvQ)ZA?=AqQ-@{ zHuz&BK9tyUXZ)DAvyBnzZzW>wSRH9!c9-|#;gAZ|zEgTvA2VS)S%J>55>^R{o_H^k zF>9|x7mC)Z3}tRE2UE&q*t?ulqb_YpM(?k~neS6iQtNhfJYs9Zf$56y#%_u%`Q9G> z>sScz4R=y^U4P$;r%YC_VxVm@oa?s=(a|TP8GN#V2?X9_z7#LB|9GN5t#Ky2PN97c|A|2TghuWk3&huLIjH%&w4DPVH&>=e&zb2-(Q*a{ax@4 z*uPnsknx%`2*3CT@ygue+*a_WdQ5SN zjpUpNLiBGuKd)E9-Gl&_qg$}w!{xcR zUzT1j3-^wmcUT*eW`_tVtj0sn!T(OL_b6BQ**~B^5U5|4=rq%N&0xe3R1(B-$kPnf zdNz%0Q$*|7ag`F5@n5{H@bulqiPUQALB@1z1N ztQu1%DX%+UM>WJfF|P4DUz*sdo$g(sOlYys&OKwc@_dLTJoeP;A9H1L&7J!efs;l< zX|~w$QlIY`mye>mE{SUZqr(Q&@tl^)GLW8z-a6^IzP)0;6C8{FYwnILuL6p@f8%zq zchT?9tn|MDm*Tb}9JYTMKLfYd(8Zb}Yl?zjMj>h^{7jp%H%Zx(@|rjcD9Gg=q+k|! z^5{pPibx62RwM>4K$tww{wNfbqm{|-M*&k$5&dv(#(l#{{F($T&g{Tqf2(JQ3dO4D zaU})gmZlMLqB6*^3}M;PqA;;w#ujIx0a4x5FUqRvicx2FH$}ORrtRa*8v3p}vS!aO z&SL0ta)qI1-2(M)3+e(qY>Dy;#`d(X-qD=YppBDi5b4OC?AeX&D%_O}3ul zt*xhSTFlU~_*z1P7SU-xRlwFbyTpAfHvJo`C;Jw1&*lpMibZ+@pV7|?W(E`i^i)++ zx~4W4$v{DfC@d+(_gh81BFv4sL?F#6T_RN}Ny${<(1~FBD2mUh61uVvo+pv8e@P4sySNubvfEQrXX$Vwfs;JGri&HPM>VB@xh2o zdJJ4Bv1YEx@BY>ue7rwHW=WJHGT4_+l$R=wliI#^hTUmD06URf^rlGTWTz7sA=jT5 zBpWBbxtIf|w(gP|XBCG2P=g=cw*vJ198KeA*_W*8^b%i*K6|wTA{uBR5v1;IWxzTM zn5Y)U8sC6#Bl>HXDOSPza5Pyro!hkbd%@TD_07GPd(PVn^;6{sSM%$;9-j3JT^Ca? z22z4`#M=tOX^mS(ViZpStxz74w0|g2-^?(QG4`RFG_(jE_zw5lplAj zUP@2!AOoqDsTx0row%~9hdNcVZ32DHLv4kIvgXkY(b3d|CJl1OEJ%jp?N2$V$uk5L zOA1&om*e-$N%Z>nm|!i_?LUAyi%wPt8{S{Re|#?JrvCPo;YZB1{(74Dsa(_*{6H|L zOVL_@xOrWfzn)n{i9XQ0_j7fjZ?3kU$DbJ#wxQz+A(Og{@AHUTW(1)TA*>EQ=$Hbm z0C8t##d`G_LgSg89Otp22ARo&kyX7K?a=Owci6k&EK$5nRi6{Jl;I>;us?N-Y#g#2 zF%4LMuc@v9hIgSonKjX=H!Z0D9i&iV^5GoSp!$O7PslOKI59ti0E$w{UF9aCALh&( zg?n3zxLhtA!e!|a?p&9o=Lx1DrJx-B!eht$w z6KnerJ|Y4PE*O>>Di`9dcB87kxqQe$WUM5msN^iCS@zhD*D`{*Fh?{{{>>1S!vC1N zs94xnaIXr#XztvyE6Q?4XgJIi#;AGZRorQ2sybhl6*Fcc^fnQoiW_N@G zd1QD-&T%UJ``5*C??&SUE~H=`B*Q9$)p_$>{XJ7_Yc6JN=gszVVH8px<5&CVNVWQy zIAfhjg+lA^*)11wu2!psQ}E_~Zb0APBxl+mN5biXNoe&lMNZ{+r9UPy#)`%@*4*E! zDl4tQUAq-PN_8_J$7aA3#sdy1CB$m!E?A<6?ms$hEg3rQ(}ieK=>)iF3VA(OV!2n% z(NbNTQ`9f*aYMunfm@qv#rF4?LiNSiGK2>yNlZcEowFwR4_in$_ox+r>%Y&&2<+1V zVut-y@7qgU`?ntdPYWO*A-|Y)$8jHbyC4$VIti~i4}Q7o?%&n=*LP%cpRo!}aWpJ#vRB{Z z%tkkmX3|4uw4Vc+=@kK5qt>^E{n}U@2oBh(Z1A4{6p*Qy8m}4U%BeF3-hRzl8MBwZ z)F)_ihlsm2wpIH=#KLBHuW9>S2 z_cSnR;3)NNIsd5XXlz`p$E(x;V)q}f?4B@y3==!Ab;@`&bj-r@wYJzQlm!4l18*rL zO~6{!$MrkeV}Ouy+9z|6#L`?r2o)IwfCj@=+0x?42qr899J0;VSDu_Kj~p%v2$-8= z!O2}XVC2t51dNPNQ&!sGRhCzakt%9gny9L(c6V?Uex*w&7V=gkHmBZK7X6Yse+3Xf-CNi$UtV0yN0L+Dcam2^?|eAf)cvG`^wq!y`i zl@zqtQMHCQ$?b+O^jm4^L<*Xd1aE9=$_$yOZHf8y@K1nBrU z?TXD){L3~3x4h+CFz|P&|Kvz}iSuq<=`h4)txQFMs|p$z5G^_uTy|8TcxjL+1_GdY zS*%3im~?w}Nm~>I^j$mZN0=A^0x(L06sc5F)U>pug>c7LH6YgHNa9cI_7<0y)$=Qy zX4?S(hSYIUR@nI9v8OWKV-NsPv&qVu7-nf=^%wK*!UDmDA{O2QzEP4QwVW;--Kf4z zXopUyS%t*bzH|)uPaiKrP_#1ERhw0Tv+B7fn>-#*ndl53W73-QIgB$DzA7qc(l4^Z zhT0i=ZtqlxY?ER#X6sn6?xX;Ko&bA=TUvT*WiAl#Q&szaF|uyOYnmvN0KgNN3>~l~ zISbt>_)fUijio6vZDYP29ovuKk95QsFu1H&CXT70$>@sDYfY`*ac$06_vh40`CmoY z>w8bbAi*&Ul~Z|)f3Q)v=h20R6}Kh)L1H<$=-%B9v823c2q8fU7v=2(2bAdsYI747 zqgW~##s~EM0E7+h-9_BtxWNlYaS&@A9Ue)it@*F#j!&UNi@`@%9XIkuk*G)nL4^I7 z_^7DZs#ed??=1DM4U#Jl4vp2$`l|Wt~v5?aB*qX`|jQvzIZ}kQ0mcCNY$Zqkl*M1a%-`= ziaovI?ky-FN`S*eD#FR<9&ostjbUja{y4h1eL3)u$S{|&TKwV%1O(GiG>%SImeKDS z=owK%K(uf~!$r~vVw1qm@(T0@g|;CA_x4(mN>p@nbD<+HkKGL(3%EIv*eIbYS$WBL z;72eN)(re+b(3?kB?)lI{2ok#naM&IW{?hOwmcqdoD8u{}vt@ zuw!6f`&RR_cnT_5)Ves{tl12`zv9SQ{1bZ6^LfL<%xr0NZPbQ_n_%9n8y6o}lcDVj z=CtNL_NcK8mWhT*SiN8|$8G_q0(ns5pOW?EER?EP6m)#DYe zijqmhOl&Mbef5cig}M1Lf_C2_YrF+pN3rG3$;nF#f4E7Mv%7ogJ!*LZ2?rwXf_a^( zG3y-jK|bxZP0p-+7v<(`U4)t4%38Uu_5g7gxnjABNP(lX64sL6$^W)C0U5r0xs<+% zsDrz^z>S4Waht(tUi3M`WJ=5elMYOfMC>3WZ#LIbN$~8Gd(Th0qQJ!Uawk zfPrY~#*{$LSac?4N6UX(`FPQ1pIU#=#;)h%t>aPK!zGSFBqfdTy@lbmF@TXbyBiag z1JCCJH&7_wCUd86-u%+%-0}a>}c};%&h_@tdNV8L_%a0m#$!+f%L4aANR4xCpksA6mH%oBkK5izf zbD~ZI0DwBt#MpTMYC|hJ@hZcV0lljdW@2?t1$$5Ev9>Xyw!c29mMMKoot{g%4I-`+B{ws;&b z#BZvW`)ow3kuBJP|u%MsfPhrku zK&qqvzER6=QKC74sU5u(#r>1zOGCki7b3Ws-A&KSXh2}zzWKZ_UT@yNi+ef}c_uI6 z+$|2Os4-n#%GUawTbJ%5Y{gx@16VKfxsNTvoMCLN6~#Glo>{{tMiM->z84ik?I8IA zfQK9K%Gz}B5Y>~LT{y{;)nQz6{GMIp0ZW-d)cPwAU;+jZ)X^RzU(-m8r!N-aIzzyN zc^{s+W=(6iAs(Jw7`@r|zOhRL?@^9|j`k7PDVy|FeLdRQuf|Cc98%&&bFL5r`u{PG z=~tHIAxY}PPTM8@%8uh3Gd(HjVNIsSLw28;m2&tx31cobQ*waFR_gQ2>>Qr>SqX^d zZHMFLE|-dk=_{yHkWQFSU%jH3dLx=XHf5}p<+CX_nsuYRJQ!-*Zluv;ZbzpuiHt$d zOe0)JT5JkRCf!NOkF$NjVj)R)bsT2wS}~V%PQ!h(6%qmg7znLAuiN|kI$CD9S?@-+ zgEHIejc5S?(Iz-&c&8bali#y**^@fz}}P zVFVWrF0gRVL`_1{WRFI^KF7s#zB$-$i5OTO@;P?ZS!^`*p&uu#Jsqj_TNSO}nSi~l z2kA<2U0CBA1#p7#C3c;;K+a!1-;*aqxOe}wGBsc>K0N0E4MdXypS3Inl7UWxU3+iR zFcA0aiTP!M&}Dhyh&>nT)16nVVtsB_eFY%Nu-m3t1nBw;6=L0Z{I!lvEul23Tp3 z%sALJsgAUbG`}LO*_|?Ik(_~nVx<8mdc}jW3J%%6+~V zH-ntVlnRJiAb=exP1H}!Oyx(07zg^=x7c`zWP$GL-`D24vzO8QDEsF4P6GE0HJxgj z`WNP`Gviv@d|ypmt(G1%C+R=a+Op+04hNyIjyV%L58`;8MhE?URZC`#PcU z&QR8$eH1PAfB>p%?Cq>vCqpT#Rzx8y*Q1(aiBfN%KbjK(YJ*tPa@4z~ItgDXC_olU z#hkZA3TwN5N@R-D8E)=z1^YQ4g&TJ?n-ZCq%F3P15L|I&jfFR~3plA8NVIl?gP+fC zVdWolatzk1Z})bIVZXyeH?!mSxWRDT-Pguuy>%#TU2%B_?=wW_7XliBAM~Ph$cr-6 zG`8%I!1>ztp1d|rea#VHbP}@`vl{;j@3qEo9tn@>t2c8cZRJ=lND;zXQ}-PQ*b2E&x#Nz2{xuumarn;wNEl17 z-%4{OR~m6R4-j^tO+Kcx&5V_Ht~F9Vz56}=@s0x^@~#!+7i=x|bI$nHX;92rdJp`H z&ouRSj&N<2IeKZtZ~U$PsB~;a(@E1D>7MG26;oGJ#xXopHu&7Y5rKT^xD2)%%;MqlSxfnN;vHafd0rzM{onDXz%5VJvO-I!@q% zlc8K??U6i|>n3_CN7T0U{!N}(OiFB>7)N|7L zd6c6>Aqmiuy)IT+O09cY6vYFdLI<)3mL3t#%@!3f@w5j&*@!1xZBp3_W2U1#opAt2 z7eAwDEBtj6X2?W-*{Lt*jfzY-ymN!W{{S!o;N?njD4O zJoN#=kbTXsZlDZ<0~T5Og}kZ@MNN;s*4yWpE3HlGrYUmteSJc6s+ysKk+y;{vDf`u zfYJm$Qa>9AGn!U6=WB?sKvekIlt*#I(v(d>IYrpiMPHhqX6xKiaC0K)`LLVlp3FI8 zb85e}M(7tl;wVWbg_g5g^m_W4iH!l(3Q6^t7jheZT`avB$`FwCT>rz4*$pEL$f>+qhq6speo}3sp zC9B{D9#>IeWbS=)^EhIHL=*0_z5|=S#XYAN$~*DObsFxk+<&rx0D#6a_o3sB)lqnl z=05OsCYYq}A=lzw*9}%t_dCI>=M7eOSP(}*yLcGm?|=HfRe!KUo;DZBKvRlNA4<6J zJlY0_Y1+<3kc^Zxz=SEi&&J!6c~r8XW3E}}<`+%}8nc0Xjg<&AZsG*K9Nz*JR2r9a zZPkn8PK9KtyWubY?)nSID_b}i+C-UB$m+C{Ud2TGY#@ql>I$xPibEjs&;nz{hnT25 zl^{u(^NklSnBrWijG<$HV`g9_i0Fj%rhf6%+MJ8_iBy^<4!)3=k;D|Ky4kh(wD9sh1N}Fr%;` zm;E0J%H?xVq|y^lZ6cwkfALmriuDbh7aRW7nKh^G^~Dln4ay1244x zWf2IdDSo+{71Nt_&8_cSh;#3DCFJNYW&N0=lYx0~IPA zR~d@^v6EeFGp;W8yLJ-;kF}8akPsUMsm{xoicN;+}WFM?6{$m^x7>YUhV4 z$48NKVfn4-bItk_>rVjBbiO?{!Yi)eph6cJYs$xPoi+CmRJlU@CBGxPLg7_iRUGfh*Rcf zLuPtsPvH`!Yd&-4Sx2vO@N^O$^7FbgTPMrIqBy-ABE*W+==OSQO+fh&<53T+(;NZc z{c4dv?O7zQnSJ8)Y7w7~l5glcRq}Wp)dkpc4A`tZqc=zX_T19FEbL1|g7TUntRo#x zXwbTnHbCotqAPre)d`Hy_SEk>OkQJmK47#7=xQ(2h>WcQ{pkMKGg5NG98e}jkh3u3 zz15g^jn}<{E0}vL%qYELkj*zgx9VmTQBY(T2YM=`=8&&WM41$QUp8?#MLcA^(S2`u z^uEGVLz6RZj6(d=EtEIe(7;~FaDO$;{rK1%JiVGioh;|bXDHPERco;+6-`OH)2gix zB+CEbFY>}lnWB)Ph$)wzva;7j-Q_50Bo-^jb7y$Mxn=fYESybKDc`V}J6}huAD?q` zSRem24$HvJbNzP4N12bV&R(GW^)Ji0i}LfyCH9Ei-w2as%@qrfmvt|1`bT7cl3H!e zs=Zb9CIy-@=Wu)7*Z=eDMLz{`4Q~%Ee3&g-+U@l4?@5y~o!;Zssg+GV;x&wYcZ#gp z)$W40J#v^u7bLQN-vpgGX0u7?IC4EC4_t303bb^1u3Q->DOEF@qhju@7Y}9dug3_c zY=}$Wm#4L=8CQ=gamPd6NgZH1^<3^H=Ol2H$N=`n_Hz2TGe7g+`#iT7@%?2mZtwS^ z6E7qp%ZUoeET{aZE8aQ(&hh=Mc1CUc$k}tEiS$psJhwa3&(G{;xth?(^kCy?E>vuH zttunn@3lv>|5-rz4M8(Ltn0bIjI}bum4F*1^`|X$s;Il;IgiO&ztpEJ)8?F-=VkR^ zA!q52rs)h;|JY_h5Ws-FyF#{i!<@m(RV1#D;lzRl)^8bkY~l%>U{OXwx5^& zUFHy_`P(j9Z#F_cD2&)WFetMUr3pcQz%s`|oQ414t;@2p%;zVZVc7ZrL)he-qZ5N= zEbyx_NT-aE4>qsDWf)(h;vk4XxnrYWvCwc|KGbriYR=*>EQMX&KPCfKib-gP~ukmQJBtS^A+1P{v1KcFAdvO)OJd84fv8= zl^;@^P4G^@c_BOD?J&VUZYSJwYfOF2cL)8)my6+?;k9cNu%9J#-H-diz0){o4&#uv zTR7bXa^P7lY3DqTxyPZq73&OI%6+31*I&HP!<|*`*yrP{xw_Ln7~gN9e*|Ll+zU^$ z9u~JgM{}#r6@>b3&%yRSyzAlZ<0Fjs#?98&-3lSDvPl-*Zp%p=VNOf!mq^=$X@qaP&>U7_i=@o zsw&1)2JkGwiBCqmPsQ5Pa-E(#mR)<)q4R+)Jlns>OP-kzx6SbOtmZOP*X<-vk^L|5 zg-?7m3fF_j@{X75>*d3LGP}by@tX`LyQ@Vmg!#{K?b1yi%F{zU#?qytqsoC2$y4Bk z6mi*NmBsNjPoH0a<6#+k*mX9wbhWkOOa`JO<{EsTM1ABCM)UpwD0usk6{0V_)w{Ql zHu!@x%cG^Spy%fRv(tSW=@W%rb`@LS9A((;6d$3&gg}BXa!_*xyJZdn+3nvTL#kz8S|PRUx!kVu)Rzi+t&xy9F#n(}DjwDjk)cheNa zfX{4yi4kIbPpla6+2{G+>&A<1@8;!mujIU46Q<$zny2I2U#=&Op8@#2_*89}?^`i> zR7Zw`F51>@gxGUpE`KXMBpLs0_Z#bO5c**d)C?gg;>6Yb{cSbB%<+-Qa~`b}QPtl` z7S7b!U^M+l&ZxYspvL1Npgg;_Ah%e-d)w6=f->VWt8$w+d))#N4I+54V^^b7kqQkE z2z+-g_S-rZ<`KgaM$VLIoQAg88M)nJB1`(-A0&HsY0+d5OEnJA3b z&5u&XX+}(rM7a=^e=ZsJiJNmgYcV_x_gGSKoJ zdAqE++unqA`>7>Z4SIalL1SRGT#$iRlGhWZP43Yv&jdH2y)9Bb9|lCdxfhqF7P@f{ zU(2ZCb=N~WYaqUFt%aWeGkci{_l!@;^|0|!?1)WhytY=~hxXP^#}f7gmw0{#ACA8` z2WzWGqr@65+BCJS7asf`!< zZ=koAOSKfd*_@qc<5osR!~GK3c(XQ^Lngr3X77$mFpbGWMq0ERAd)e)M>Nk9_% zu-a_qeY^n!Qj}Lo(qifU(Xi0a>8>TR5P&#Vdpam+XzW|(2%wYzt-pZB-;dch)Xj_7-(=BHUAT*4isx0(VjGBOGZ3bWDL86RS05dh$L)uSLH zEL`&CdUbsWluP-s*zMMGZkg$RT3(d}5*v~pe|lAFY}BlIL+}0We5L^~MP?9&K@xpW z6-I;*k!OU%j-0bH*lV(wih4+%)Is0(2&5@)I#_RvA<0C<$OHc{_P6&#x}39wL{N~# z{j@B z)Tb`e`lxv7yK3(SUd!``2`DVEOk?fyi=!mX%TT6JA91dj3;M;65WRdLST6fZNx#Z7 zQT%mc!_B+6Ca18dA2ay2u*)12?aSs(&iME4Rgtk$>9It1PA`+6@51S8;S`;s_fC>? zB!BN0P`~NF&vE5@Gmn^R-l}(c8-x!8SAR2hERZOVc;L|CJgJ<^-SQO5Pj933Hfpbp z9xOv#5i*~g3^1D?AA$ZWcIQy&Y%*FaM*kBdSG&U^aD<$7{Jpx__429U&Jez`Al(8C zO%zXM`|7D7gn32=F`EWKn}#$e<9%mn@{fbA5oPCvs&A}*gW6TR^o59{4e&M2z@XA4 zylcC+VAX%$0|nZRp(MLhn2FEoj?#}-MNd6HwSavEy!aqUl3*JHO9*NZ6Q0_c+hjmD z0p{bgGs$5Gd{wQ%%Fzm-BdG#=F18se+Xz1zAWkjA<8g790yR0Rl)$eGJ|-{YI1B^x z_Xj1A#XKL$cOw)HZzkZKBLFDguw`zK;c9C>X1m^V=t|bGUScBKOnUm-z8`Pv(Oa{_ z*m@&4rZu$N!cK(07fA00KtiS-Ggoj0=Z~ubEdA~~~ zpEYb(v>o(Ueac|i^nP`K9iE)aS5JC57%X1U$FAa>(p55iemt{+3?h+Mj(6OyGjfiZ zQYGGyM}kTM0RO^@E5)@%I>kQ^JL9GgoivdVgC` zMFn;|S`A+2hVTR_+%JUh3ot5Ug5tOz+D09PqAeZ)antzygy;ZIrkdU;$e5W0b zy<8&_cO;Uc&8?lnnn#AJqK51B4zJmM{o>#G^=jOM4`}G;OdbPu5Xzj&5pcKM z7BvO{#-88jF{kvn*%A=~ntnDGF&zzVgrE%xH~XC`QP9w)T_@yz4+`y?;dm4fl_ ze4YG%mb^0*eWN34B}VhJ2^Eo}s=cvhP1AIsDLR9e!wk4+DC|S#PeAVP2yNXClU+jN znd&&DI~;d&1_J<8{>X;0{fDr`c7yyI93cQ883=?KTI8z67Z}$6NUIw^)jJ|l>239!q3-_IXk@oK;MC1otH9TFj5V((ldYi zkd>9@T*hmF>2*Emvg=vb=!=ifa$4~n3GNr=SD~avfxu?r{b8D0r8}%|NwJ}bLAeSY zz)0S-8xD&U1OQ{`8XQDYxd#e}Gr`F>We+bSD+U87`gq)~{2LoMYh5%lSojjjT3i*` z%_^zF@^W;Y`0ER8czsPk4Wj=c+2c+AcGC*GRb-X~3;cx7o@O~Q5R!nAAu@oK>_#@6 z5Dm zyRN2hSI41lqF=Hz28lZnmHHo#cRFq1!F=_zB{`t{p)FRcW{F&!u@9jZ1yx z8laEzC)&=*HQ_NppTq*{q2&8OrFhYS7Bl2?#!Q}DRoRsoL?mOueYR$WFlNGgW9vx~ zTD;r-wmlQ`+IrXWZfi^&1g-U#!s4xU3AkVWpq9%nnd9>(yvpxeU<1tby|KU@EmXY> z##72PwNH20y?4KVyEXa21#P#dxt?8#`+M zR&OF(Okn*Dco(`!L7el{TFBhH{EgqYKe5N-4r@l|G9*22UzOS2Jmb&AK?pEF-4PY; zid22Pf{z@tRteTA0RU#rzm9^h#=ISy@@WZFTE62juFk1GF{j*npL-=BueuZ}6xKMr z2VE)tG8;{-Fb+9R*>EYXoIsKNdTgnSf2-?mes&ocd^R2fnhkf-hHk2sm~F^bVY6mb z1h#FJR&1C8V<7s6w}=<=JYy}LQ7`Mf16u($WEQ|+BPRf%rG~J ztdzBJQ!cA{F;(&0ud&w8*>CUaxw`tH^#;%An<2!wJ1kS}Gi9*y=&2@Eg5ijNTs|i1 zwW5evZ#{ZB^cL&a@@CSA=${RK)%RtGSn6`}>49IwIN>D*qh_M}hpqm+KPw<#lH$k6 zDN(;E_TX?Pwwr{ZX{WQdrsrvxs@aekcmM%E7!?@W-X0${dXdx&#gxkEyYytSsO1pPM#*zzUtp;AE#?1U zM+LQwv%LXc+W<}1tH*vz$?D_^rB+Poz>bwq57#fn$(Azr@&b@>H;(7vNL5VI_;6rq z5pRnL1j3qHO~xYW@E>$oh=RBA(LBa7V3NkU4vapp|I&m3Gd+o+%rj7AI6(2zWGw~= zk-0gqdPeLX-Nqz{qQfCvZ(d$rPfI^1v;qn%7hp9|M2kNkn^awv<;Tvaww99Nga`#- zN~Y!W<1mU?E$eBZW_Wg8+dEcVXD-kG-`vCm?*HNGoueaZzi8o(?TJ0HZBJ}xV%z4# zp4hf+TNB&1Z6{y9zkBcZSFc{HS5Pn#hv#I55k7Qn#H0s?Z1n!GWzBEhz5} zmzkFF_i5^VmB)uG+RpAXU-{?$aD-VW^2hKWql2q#5Sp}3_3Ik6NEWdMEPu z_$yhZ`cS=5##Yo-bvVo7h}B8vYdp$?l9alI`*Us_tjlE+O-l5m`DrL4_B1)f6$ne0 zY?h-tdJwQwt&{14aK_6C6l_ZJjG{SQ$h-jwl$@LCJ9)9CU&&iGmfMDN(;Sk(hTo^h zpqQKIHhTru9(&!f+-YlN#U$ZRqVll9JtaAlbB}WOYhHNd+CoWhO#DBcO*C2~Dg8u$ zFLMvG1|$mPyLE5{-1fTy#?MQ%A+f+?J3G#amFgU^d)3|BkcA-5*Xn1(mmn+mEDzHH zn{c^S?+2$Eg{3ij@b%TQUS4eYmV6hDnNb5RpNXqjvrhP(JVu%jVsPnynS-?x11>J` zr7^cTOuMDbfZR&vNnJ%F>vcV^X;E!Qf#M$k?Gbs3mJmgt$sG|LKa<&+z^k1tc|!dayRA{R(M_adWlwgsTW!3^KU|}!3LB2&bYtxGsr*Z?7WVqr3wN@KZB1BE*zNE>p zeApmB7=>4{3ubi>*fkLl`CR8+4>$9DU4gI7=W=e%M{imB&Tv-0-4ES4vM37I%f!)P zFuqK6q?+K8(&D=_1lI+MwM@@O(F1u16xRUKx^8yTzS4Ru@i4y!ed2Mu+>SQSYhuus zjGq%lWr%i@8X`_qaAlb@kS&0i|6_x_-6mpU5DT;mPMYHFuMXL&y+mFgFB{F9^s6Lw zp{8P~UL0Kynr}^PE)8t1^HJ7R7X3+}^ev?v?vgrTT&bc#kikT3XYBF!5iPu4s8Ao} zpEDrnsfCmJdB5jAfDaAW5t?l{_}j>}g_h_MvJy%gv2|lS1pyRU`9IXddKNfL+)kkx zV0Us~)zmp_ajK{Dvp2g6n{qo3ppe9TM{moGP0-hfD`92t#(iy6KC1{AE z`je*||8#O-0sso^Xx)10ZLc>2dmqgABk1E1Zr4(V6av(!kF&iz2I7|2WfD@?!-)XD zNBmQpn=e~j8x-c1x17(RvSqympk)O%c#G)!7t5#BahfJayXnaPNXfY-nY>B~zZMBy7la91T9Wxipc#&`AR>^z320%*nHfEAFCcAR$H0mX?p zZx$-+!b=Y5T^d;h*tze_VlWrSxc5qNcu(;ROW^h!u7o>g^Ue_pk809l-xa}T z!)J1Uv}v^CbctpJw*pwE3X?hECkuo9V3@$r|{KI3M<7cHGDTeRgn7KtcI?Bm%z z$P#wRc4>CFVe9)Y^K1PSsbZ>dddpYK`RexPq0Q)pa!eA4<7on-fKQJG!Lr(A7ewUe z$9&8&hiqep*W2xT-HeEKZ?E@Wd)L`SmKK3wh6aTIJO&Zc3(MEQFaKhsZF|obNcUgA zfQq(wDP5PEjkIgYhRCU?oa6g4wIBA+u-QRt4;W+^ zY!z*|LirmLps(&@90P1FVo&m}FL*9fFEkz#_hd8xz%G3OH`>|mdjyMLZY_me2vEAk ztY$m#Y(Y?PtgSbI;i!Uc58Q}=VsSTj!F8Zvy@&c!&BTH!RyRePn|i0EReFKruPZ6; zc7A6Mno!=U@aTt3OAGT79>sYCoA2(Hse$?)ML5?2e6gSC0u$}+o`pE*@x=aycIH)< zx~GB-Up%KLqm94|=?;SXp)bD1<2zWCiQ^mbq$=AN=Rk=E5F|zCA8qSm{D6YZ#yG-r z5x8e=Fu^xzK}+!AZ{hYXe1=Djp4;|8PMBX%s0h%40JtmP(Ka1O=I+~@w4-F7i8L%K zfKbH!9b!ho2tH3*9&QZ?y2ow-3a&NB7^m#E9)F8J7B;b~j5nUqlIO5(5LbeQD%%_T zMl9tk#oI$hKvoOfe-L&l_0~abqPeC~l-HCq%qi{~wl3H6jT|VExAX^DgNV*b=*)eQ zM835OwE+u=u3VX350#CDvG&|%WT|2;%40QYl}vC&2S{#ahkyt3Fv>`5NuGC0fomR{ zzNqWe4^34FZ8OGxWpQpq74Jp(St=N=*#xlPnpkTf0BEdEkmiU0fzC=9yHWdMWu(mf zH-zVwBFv4?#&1;WKJ~O{A%kC)TFwWoZ2GVDXe7V_PE)bB>`kwsn2W&suA{`{+rj5@ z1)Ps=XMAWF$f$jLFqd`O@U#lX?W$Q5ry@l)5vx=3`t^ynYW{muzXQbf-}Oco?`T`J z2+{CckJq&q;}?Y+Nk%l=wMajrv@nmBQHt9eA`k*a@_npByqcib$HUQ5*Yo6I;7{D8 zNRrIP+_<)TmK^^3i&$}$%>!+^+Y3auf6R!){JVHduy6~@=5mYoJHSd02ng$9@L}yS zEvkZH>Q4fNBs@3fB}&(63^x0ar@P@c^#B-y^im9d~4uDiT_-?bZLpzX}|gBR*(th zz3MCH6_d5k=U{WO1nI`LcBrVhr#JnyzON7=dt@-1u1R2^ppNEfwZ$=Z21`$RO?bdv zy<0*-LbTvfsJ_IXQs53>2lh_^HSNXX3P1Q4Q``3(kVVD4LxaC9gNQ({3<3yI{|Osj zU#!mlQK)avsUrT-vjAsl*TZ$*cVcIGQ(amHGLCj<__8PWs`|j<_RM~spcKbHNN)_3 z`S%$4a&p%MW^_tm?Zu!}s0X;s(&-WdNsf}aMbpKhzGbSt=u1g>I1T$_5x%wNys)cr zp=?$pCCDYaT`sSho4z>8wDiYnZmrOPJt4`cn1IU&qxr1>##C&*^=cB%1OUHQm+*k3%n$N;8}>x=79a|-*YDoH}H&Zlbx zARUD_;Scmpp(o{^|COligqt1b71ij8PC+ijawO?`&OiBzx28McuUpgq=L}OQuB4LT zbUx|91NPS3XQet1+}KQJ&-4Cpd)Wx)Ctjc73Ic+#M~!7in0bV)nnG1>tumu75^J#jZ+6O>uEae4faj`_fiZbhQy0 z-rEY&eD2CcbzNHMWv=1g|G$d$q~?M5x2MkL-LlrzI7`B zkxO82UPsNsYqU~EC~pQxi&Y`~04j+GJ* z+j0Gl0gd$i?Dq4*06^(Ji^hi~&g_qBX`D>BA#|7EfiBw6bPv@#%e{zfqgEYcm$xspqW9=AgUw{$Z|JV!61@ z#o#(0rB{@zaVUONtUCg1tua|#5eRiwwO#HDL~ zx>DoOK^M*g2|8XkvufT7y_*x^PHbvDDPyVnFU4;*D{-TO6~rYrQ9AwF`75ARmptsG zI5e_#&f;Po%)bZ8_xX7vM-S^M+GhQqgd4R`5M}M_ZY*BDI(NRXa#lUrT+js_xE)%E zldWZ#f6MAUjZSatSRWu&Bqr%i(F2H7YSy;5uM(`N#DTTb^H^8D=CRZrKZR3H$Y!wg zyIhWki7Hh|XBRAIcIj5EG5er!k1b)pj#!q@liRt+5+uXc=U%ENL?%yH4KQChhH%&{ z^_xe={d@Scr?~$&H&`&MS(vwktIh87bIUbhlW0{GE^PQ6lN-%+RA~Jf%KtIN0&ruH z{LCOr?DT_eh18bK)K2FE%kgj$wTi)Wz80>TuJA}yqLjc2SQDO*BtIfWTTJvsfqFX& z&{tdm0_b1Zx4`O?)DW6sOgSg17@YY}NA4}T2u{NEnJW`_hL?Zi72%{^0H%+R)8}vh zGo1`B+||qI4N?z6ZnX%x+#B-%wJ{H^SKtU#;*$;G8TDlg!x5hAXVEg4e?%=))Ke=hiCpzsScIH|A)4wUbimb7 zMnb&qGvhilTV7T<4g8zVj!+>^0@{qMJ8gCl8$wqE9AR=qEP`Mt+EhW*q#)i zb$}j_+f4Mrxp+ib$L}E0rrNKX5fF%VNa1wu=>ya=3LwKM4!%B(6@;5s^PyoIE zM#}C*4PsNNjxn>+8s#s0FjO**#9Vp9cZBE!uD4tlMZg6wu$uF9s!^9iXuk!#mhrb$ z7Qf^j8_Y6TE#{qPoms}S1FU~QfIHCSb|UuD*=h@L&@U|WxhpCrgTmH7Z#5PcvD(bD z%X&7rIf~!ClaR@>tQC4~TvO!u%q2Kw-)A24nr%2ubLo8}nrDnp5PMNaO0_lr5t@qH zoQbfuMau<)OhW%LRal1!|1PgZZ+lLpa(bW{adt#3&COl#{*Sk-#(4Z$e9Oaf4eGKU z5bDJ^^K4(%ZS1Xl2Tq;VPUNdr>3BuRi3G0ghsDRw^=MQBoZubcm*8G}*L%2@5NxsE z(U9GO&hP$uFs)M&*sq?2LSaJ=8iS%|R{nOn0u?b_MQR$pi98632LcKXjZ8EnYanWk z(UK${J1Q2En6YxjJS6+9$sV(P=xay2I%`P}4I@sv8}_6|a31L5R%`6!`Ab($$>5qY zdeUZ9v9r6YaFg$*+39SYiA;mZxbOC__xxSDcGvgR?0377vD^seA$*6=*lCAw1*u9b zUe2t|WVGQ3oKrVE0!6|In39-IIp6Lt{Je1|9XRr+gc0e>6KzW4v&b*EO1|#)F7T_? z#$+g_HR<#r8y;ORAM3H1=mx*x^D_3t789XA#&<+GB7BNod9&NRC$s5JRqB2P%I4B= z*J1oV!D?6Uj+IXvF%8=h%t>3tOgspGOqb`K^Epw`%pzh*PJ#$P24hJY3xkGCB$&IO%cw?J z#fAG&=B5cf4!n&&nfH{C)bPKX%vlWn+QKC@`AnuYkt9GpDty*pm$$2dfbi6jw4ND? z+Rnx5$@_Tzs_UTpRY{+)nrG*7N8@4!=7y!17vgpReWly=j?Dnw0CAL(kOTQeb4V(Y zmT4df1}eZBb`;ZVGU(m-o|f9k?dY;Tz|npAF84D|r_|{4l%YKa40AENx68dyZ%Dgr zH1FMfx}hpL2R)~*&RE<{i*T76UI_Ev4UyIK(6^dL7&q=;N~ARQQ^y}K+`}JX^%ypK z7I1D}w1H!^5_qp)Q^V^&bAA=`@>xpmn7!5_c2c4>FTQ%_!B+hlf4vShler6Spj52# ziXO}Kl)1gGyYuClB)bc)pO*V}7IGq+*irK3iY6;o_L^Qjwq@lyC;^@#BWB=aQ297q z#(+7gzRkJh`u=eB`_tWnL{WCqg*FK0zi=o>qA0+k)ZBC;yl+QO-xGA&coOA1U;%CA&P6@+d}u$uJ}RdPH8?J# zw0y((Q%Xu|blXm&)m`>kL;-#kA+SguM>+)r0-iR*IwUDADTnxBi&5MKf25E$S&Ev< zTO&DxHH?1YN^*hA_z3-C2N3xuuzKBmdS(9l`u z`ZSO?|7AR8f0{@WLeS90$zUO2aMtQwmi2@`+n-LIMB zE0C7vh#PKP>tQ}=$nCAer70BDl;L2Ilz&@VE*I93;-&vBuSIGy6M^#(Po#7@&D%&P zS`0#m+vI5usK6t;4Qj_gji%lE|1To1rM6tVje>_E@z?L+9L8y{zgzmcr$9ktcQO^ zYiM1?%k|t*vKe$Df4eY)uUSm~9~VH&>p>@GdHyncR8k?GvBXRjDqJK$Lc!d%pk$*i%a9<}_2vX%g~P;PR(UvGv~W*}Q4zvmXBn z8Ifx*3en8EANG{@^Mx1Y1qiPR!RJ&FTJl9ktLNudX1N`q_d&=|xQXusdK^jcb8P%C zaRF)f_dAvruVhqed2Zxq1I`BeXd&lwzV8u~eO+ve|^Vy$HA0`n-t|dK8vQWHrMf`f_6rh>iL`;i$(v|N;|BCw|i>KeyX;!WZ|b^ z=AbH<=UJddpG}AI*>00b7*i~)Kk^M7y~1b|B!mJtbgjtKThB~et9u77&pjCd7&KKcYz@XHZ^R z{s;uiw58?7UCOoQw%t9xZu@jDBwX>0h9w}Tz<=DI5p17@tQx$uytrYWneR(Zm@`Uf z_ZBj@m+u%h+rQR&SxvZfPg5-hH(%qpyYE>cMS829vkoU(uHL+F@jsbp-63?gRLeMZ zd$+Kj+2B;3vYjEPDcR5>wHuF_hhF17s+cViq30(^9!TlHh!jKG=uBXaDIm2!DeW(& zq58DLxwv=Qq;D99Lq4D#ek(s%X_;AsU3LW4Jjr9@%O0o*xvK5y=es=)?6Dq5h9F6t zl3CBvQs2krxqV;X7DEOW&&98!&E(==oYb+*gxR#O+OCnFX%C_4k>)3A#rH=wk`kj zN6)rS|Bz8I-Z4)T^JMY_7$KA-DYDQ5*&|5|xB#NDvmH|Iu4sgzaVz=}D|+xxV|(*| z#hcE@y6`)6e`_)HvgD5OdDR=o`nmR+`G0IsaF@F1F&SpFn%G) zsFu~-fFk4cU@xq)K?hL&_?v6LQa0>BTS-}isiLMnDh&lXA_31D~e>oMYFd zX#?`0pw*^beU@%H^5X+_?$lXLRTais7Sg>-F4V&z_*Y)YB4Q{BBT2_5;y`Txsx?Bx zsEmV$5wrD6sBqxcMLoF2`pJSJC5GlZ0y@Y5Mg?1AK!5*15EHFT6>%$?dzXPXtzj3Z zKD=wf{3Sk$aqIvP4snsW?B5a(owt0q{?Y)QC0N31be3AVJuN=BcR|<{q!xxZ;bJkkWw3{$Ta3)xGY$#(ml29@=IN!`9^SxPVs<{0#>1OO3xB z%}xpz#IVqu)F6r@L~DV?>#`XN#3e!)v8tz zwLtd9M;Ky&XJ=qAdoZ_}nauF*asJQee!bE8#M!-bOH&mA!7^uj)5SAu)ATpxkFRWE zC7Fg%_yhw8(Os(_LhQHhyy!_unf4T|QINtr5(opevjz{?yb2PzdQF_ht$dUz{hp@p zS~2Jw_*Mdhu!z6hI&{7&n>4G2Oyb86*-V3Rg@BWV*AMp!CV;Nk-#;)1ZcL9*6XG#_ zod3L@9vGEs71teyZTKX8$adtxHMMWBvo2*fk%f$Xd8;-Lx)I_N(y1?3`)0=+s3k1 ztV81aubGUYo{3Hddc;Ejoq5I&=RN2$_ITIBX0N8wnizuA?ZX;#|J#kC3@bqaL>S=# zGC$7pSA6?+kAIr= z`*YL!TpXg~le4exX*$-WY|B#gv-Ks5l=P}*85oNZ+J(udKmm+2sXz~#WMUddbjaw1 zPE-O>B*}QvstR#6h;a%GKrg)+1vJ`6$Xyzx`IkcIxp#3a$STt@+eBdSl3XSm;!ieB z8S(`D^3_bLgxfQBRvNk`xqkw&NnE1FxLbeumoAzGd2k;bTB{BQWQ*y$Z*q6h*C2s& z6Oyv6v$h?uZ$b|Xma?}I)WdyD0K zo%8!S$R@tdGX=tR_8i+s(TA`iNiT2SoH;GhCoV2AHIZKKS>A!_cdqF%Dzjo8sWSOE zhWKIW*Y4LylW(KM)Q)Tw4B0Yk~_pbysCI8!YBuL}`x9!-8;~6t^6&8&#!y5%-ECzq}g1wQP~c_BmnGW=Ip zT{9Y&b}JLV9-@!Z&o{XHzz-VLw)QK3brYkleUz?&Kz>5p#X8J;GA&2r2c*H`Q~%#M zBv8N>?}I}P?PlZz3=1=bv(_UKvk0+xQ`T`6Ne%;TYuPSW&@bE+X6x0S8k=;&`@H_( zSB_7O_6})Snq{4ie7bXnv&iut2c%-S+(P5|{rLS3xM_wQD*YBH13gr&hrUG2HO0D? zZw>MTQWPCSToiY;7O%aWe7%BB{c0lO_t($sV;GabQ^hVS$sGxOs~SEG7Qw{N-lyYc z+Lv0%R|M8f)BHhx03o%HN9}*SbRJt?h(7kQ z&?CK@2k|czy6__r%v}zYF`Dg^OW5b$PZ+jvcDJC#A^YXNt>ev8)t+0|CyD2vM|pAM zqx>|^(N9_KOj2W`vxl^-Zi%;$I{G+atJNjj;Bn#5+&b9XT0=|Mr;dy93u8Xl;--VV+<=NSs7J)`a6Y_?zh^LGUSr`BLn zJB!`4)^ZiBtxKAADT&(J68&Jj14C!K%WN1(L`PMKw4M8F_@k3<8&|n%A#wOC8Vo9d z?;&FM=Rm@CQ-}IWwScJ-lu zq$pUE(Y{@!yBX_S6h?zvQ8AsS15J_{-Eo`q@!(G4V)ONRHfXitc~T)rvKJ|chkEBn zn-}9}a#S>~>e>zr`cc1u%0FXsBRWeYtr^8z`v^bD>NGvzyEk&&UeW{#TX1_f2H+$2wb#b8VOZ z)#=^`<=0o^r!jQj29KGV43oaJDk31pez4!wQHoyq_TOX&R?^3NF+Bv|L?=K$#@Th& zP4QRo7q+pNFB-sNMKbZ5^o-b_lG5)aloaKJUwbh0hB{zUqo%qM@lIyocR z+3K`9MNr+Q1$KOTCikmX)kAJ;dZ{{q zE!SAmhX9STP&NV|Th%HXEUNo}h!?f;aF`DRTC9w$!^z$+oxHEY4g9_^l%G-jwh8G_2G2?@QCnVkAn#f@s**S5yMDI zfx+YxE6SqV8pwVhSF&8~%U!cKF1p=%^yN_j{uk>?oRzlpSEg5A>8h?X5t!xO&MZ7! zIU6W~2}>d$zcVFV`L8_tHf?`epxzXs>DB_4v~UKO!PX3v86a3y6TH&Lp`41>PIH*j zAe10&_J_7~QOd*|geM(B6*dT)*22O?1JQh=e?CK90R&(g(I(>Il2YP;QSEYiN@_{W zCm^9YZ75{pbqOV4*FpnF4FHJFo~7fg;KV_wR(G(odC1_Q%Jg$`8`O$Hg)X^7yh7Hi z{`;%ekP8Z^RI8X)V}#eDO~&wwG~Y23>MCqSJ=Tu{jOszKXG{jD>qDDr=-9SY&|x

    wA>4!3P1)Yr9i#UJ8_yy}5jt>!%4rxz%V*5^{r5y`?a1O-dc0HkobKQI- zm5vm^F9xE?zk0#8rn)-0z;VWfwiEy;m<{q8aGnjG$n24RzHV)GbG$S6-Kkf+wkumZ ze9EnB*Cyt8o@cRgqB_+y0uwfwkfrs5cr0nC8Dtvt=LYAl)pNh61-?gBWp^7#0$_oq zII9F}H9<(mZ=aaGXTs?o6OOhGjlx`SbpW8~N|S(MKs#;xDq>Ahya*sszXe=&Zzls6qfAh(pI^?^3=u9%5**E*b(H zHFO?2qOp+8?JhknG=>VWv+`gcyH`^EdYoWFqnpJoL4n18Jb?e5l_h9PzR}Z)0Rd;} zsrd#uL~e^G;EAzNYpqNKo1ckjB9t_094Q?sh*M8*EVRf4|Ni5>Qts0)sFHCS%wqo( zHZ7h*L{9Du^rHO)USc0w#E??9pKhPIwAkf%0CmDBLEzSl@$}g7 zQTH_=bFQJKf$ht9^B%lIE+jbWkLG?6e_=>@-sfD$h8h^C`dlSf#ku);k&S~|$Fa_*>|I!JnizO*n$fM$}=@P?ZCL4oq@QN{WimjWPYIjf5Zog98o* z8o-AiY@to4AO<)xj=3+bJzrgio{!F3`f{srK_yaY*%d?OoofCQO&7XRH8;Wm_SJz7dm(exMA&~#_7&)3jHa=omBgZ3!v7i^h zzV2@%GNn*IoadspEpRoISUmiTWU>PQKn6!Jle>|nMOak(ry3$dUXYc)$*AwDj~7)=5wrrm3JA#9YM#CkD>`#3v{$u$uSl} za3c3x4C^tW*Qhidl^i|)+~fhf<1L*I#|3qvdep4rfc>d~W$c6Oi4&GhU z^|}F7Y9{i+Y;N zV4zOp=HPO-^%iw8J-@3!^}#VH>h{B(6Tj}hj_F0t8ql{-haPvPYgKt#Psp~=OLKe3 z`Lw##Ly$+&l~2nnKM~;JDd7~gPpbo)BHWB;Ib4tM714fTrs~jY!jK3->&7RW`b~%* z=U{}iXFxQ=yODZcd8d%v+T%x}+jbi*kTTj4Zivz1;(iw^6b?~z)}Sr+=WlLXiZIgG z=f_t$8i#4*m%|ejUpqH4AWKn4m(9TAL|zv_d$kiaCqo@5b%lWRXpG9G#ap|SOO$R* zNnRA=xi4WY`9=5TZQL^olQmwzr;y1b!>n-5h^&eM^M4zw&O9n5PrmMAl53hR#E7p_y&m%2LA?kP(C}_kSL)y5M zkB?*?y<144dA>dCovfRP?2i6!+R={Wtfrv{?OhSZbTWDGbOqJ0gG~f3uAgObS~{$s zHin4XvM0gB;UamF-zt4`Z`;qtEU!y~W5R{-eVpk**M_&dO6+~$qdvT%E zjH6Y20V+Hy?2n5tH*qP*$Hd{hAG%81da!^nvxvW{C7Y=YXv~kEUB;*$QhmSrT}X(< zDc4EAJGM(m7?Dz24jlkMog~lMioZx}ft(&6aO!#Ly@6o`24h3*`nu^KlxuG~D_wl}J zsWS08I&k(b@xDHx;K+J+mHPVFO#jQ+oX@N#@6AMclVzkA{Eye&Wxa4iE8KKZEb)}) zdhMAtc&=4{4LTR-+(cxQ*;qy(W%>sj7QE226)Gvbe^^pbpPK+Z zT)`svhS3>Vh{1BT4okf?UHVD;$4a%zGuiT4qf{=ux9L(Tn^4H*FH}f?-=ryAyH!t0 z9HU4~w@g1Hh0Wc2Xa=wU!7a;aGb8UcmOWL}rC>j@CtoL_3INuYy@a%cI%$+ zmf9T4_!3{C;sUnUpuED>dh6^J2fQSp=dY+q(YHx!PLL2>qkAhNFL@&mOdwhllrAT} z#fbS0atCfB4UDk(#89G_?v|CblhXFAj;m|Kk02{h7!` ze|eM|4;9Zy1dVD;wWQCPQ1{tlAA4q6UJd zaOeI>t)dDmfx>BXjtX3b-j_iS^rbal%W5$yx&@$DfX>6=*7A_MT0}u}OpwsAf)Uoz zMA*CqDLl~I5uw7yRqLFb1=a~7WOuSQW$-M&$a5$PhQxL(bD?#^;}#bd`wVzgU$-A+ z&S8_EcAI&9kr%YOZ;ySOI(Z7Y+@)nf(PRbyKq{RnhS>e*;A2=1-=pPEDklXc_;(m+ zRLK7l8ATD{egGg>Hy0KhG$071kq`##9(){DFo3boP1GSl_nnfAhlgjbFX!o}0Ww+G zuY_2+*q?EXfApHt;Dxf}`+H~Y+k=Ao_8(XH$lLDC&C4MMzR<*`m*N=*Zz7jNC9$$) zylOWuN8_BnRIan)JcQLzLuSr*Gu(Y#u>K1%*Fhojgo6Kt7yy9v8@pSpQF+H%!K=MX z0t)~*NI01A`TQk`1q97h#B7b|zu-!&s?s4c2r61QuCUS*SgO!T;ayvj@Mgk(B4G2= z%C^x6kd-jj`AzRMuK|+`G5u}PRZ&T4dFdnax5I1h3F(kP z0|qub0^rXYl^cn)UXuz>9K^5(C^9>fWQr7yB6U$sgzw}2fpdw~%7T`%1n4_uEZcgR zy(YGJkB#`rL~VMvJ%I~@GmAo0Wk@$d3%~>l7{}bDC?JqU&;M!EBu~h*BHqK`wU1AUj-=Ua2 zN!At)DrB0B7-sBO``e?(_?4IV@E@eWm>}ZB&iqAsSOD@ZYREAw>d)DlnE^FtidaV0 z52+Z=K+VKtAH!iXOWyfy#ge2^cAfW+j`wG_xS2jucp&3En_&kA_YE!WqVR~&!qRbN zs>Cip{l9ks07Ty{Bf?4^+0~p)ZdvEc`>MpduofQ2%6dsw{QU2f8b17+h1u;}Eu1~L zC!U(Dy3cWr{KQD7Kcnb#=s!q`dNiM$XKX+&boblK6^9lb>`)wKjIH8GqU33TXBrs_ zUR8?~2mrGjah7r-PZ1BYl3lxjGs=H?ZYZd7urKGrb8p_u6^`dKIiDYqBbrmu3@`Bd zJw+-{z7m?3;s0>~CfbI+&mX?v zMa|*jeOKG=5H|YS z;kuJ2_}qQIPGfdmIzP5~)WVQ$_YK5(#&V_2a$f!}C&`w%6~EDzM8jx>iY298;>q0B+0K;JWB*o5AebFN@UqddwD@Vh#iT#vH%L98$Ph z`#eS%L23B)$yPO#g1RTom0W%STZ4rb?w83;d0 zH?Y?8#z~{ytpV8H$rM~8*>))x8?#6v=-AKr#pn)riHp7;)zPLvtyT2=%}9Synn`dOejl4ZVrMmszX*Cny63aG2#e_Y-JitNd=9hkZ`x(!j2N0uw;QP;X8Z`; zIZDtkzQ^_b>gZo2h7+ioX)aJb(-ov%;(L5pd+Yo5`0lyh%y2#%FzcaqMV9p!T=(0v zjTpyz5ppOTeh4IWEN2WY75R^bccP(yRbdbcWYfh-6n-2S@J&ItAZj$$7)y7AIT}{4 za?Q^btW?&SZ%AdV&(ZHQz{I|Vbr4XkvwQFG!Dw5g9LNQpXfb?Uba&UCD1D-T%h%X> zGXlv$3Ln^DSNiiFjxI3vN!W73?2WaPQ0iH=5{HjF8H<@3*tqg zDVL|!buKluh#>UM({pRzNcdm{g2VDwUi0ww$LAit|Ts=tFDg9r|A&)4lGyT^%l z?rU&fHrZs+?|HbtE{|id`{OLdiw!U9TX&SyBdu?&4;jUX;IEdir~R)5ljFm=ZWNJu z5YBN=8G0Yc2f`%M|F_xx4hR0wrn3~IjYO2E6?y(2HglWPDsGbeOyZeTIdb5 zr3C1#j5XE=Qc0(}mKN^f8(7*zUR~E#^ycV5fs@f#fDF0v$N~dWwI=)NB%;#k`6mN9 zX*5|U8Mz1+zr*Aa863Yd;AX07KDy@hysVu9(>DEfT^BrN2KWPilTj=_`wL|^fR5E1R>uxbroX4l@JDboFMBQb6 z(yV*yV-HYie1C@Y&x#xtk?UQfsGx?gWQxG^%GRcL>Gw^Pl4-MDr%mmogK z*GvE3uq@$p&bk-IamUF&h2&IHz~pY{ivP!ydKtQAur3H>kDYZa!0 zYk6&d_>EMFHWkJ-6<3kuwVNtlo};z=26#ti4$^|a27{pC>=ov0wmIgI5<$l#~-HobWA~b#l}q0qA=f-vHf}& z5}Kp?SzGe*9K)~m^waw@y)AE9#huPfjluiX{d-vVvpiO}`8Ml0xr<`!=HJd3!*3tw zV*6F;7=*tpIvX#uziVpnABX>L2qsZV9JbpRp*{X3<7{c}mpM*z>wcUG5&m73xgq&1 zaBqCAp-Vc`J#+r(b7z5cLwQcptuGHSFyHBfE@y?Hm_2m?&bg-*ZP}XX#UP}@{2!IZ z>f!%TX%a=(ci8u_Y#V~}7?(@gbnV=6T(We{8dkM!?Nn8hdY2TErM!bV#FSpSJ0OK( zXXg6k_@a#rL_NB4>+1tdESPtnmc zn$ZY-tY#7W9);9Q#*p9|wiN&mUUaQvziE|gFIGr%Tw1rsiXs$28XZ4*kf-9kJvV;u znPee2L+>nK_mZ`YhA~uf#T`>k1jDmu<`3Yi5^6TS;P1ss7Dw|iW zF$R_PT7yE*dyRqzb`>fX)u_iWUEd2WCU5|LitE6FLz%AU+>U#sX&hX_J`0}KF zj7s5jD(hYU#2ol zI~>Z~+o>nlr62$~w7pjTQk`dv7W@@W*0_P)nFTA}IErM;rhnE<3EK1xT&>lU7%XrF zuj^r(wn$kB>@n)2#a=_8cmPu9p9vCx03yfKivN$Zw+xE2Yr8}Vf#B}0!QI^@xVr^+ z4ekVYcN%vmIE@5%cXzko4yW@x@B7V}s+m9YlcKAut9$Qz?`7A*(beFzt;BoZF-MZ$ zO9UI5jM3PByPp2CsVR3b+`T4byzE(&iNL!+YwgXsy< zmc=8Fc1lVB0O+?%8ZGj_Jn0NyfeYUjfp9W7)_-Ae0?6|cf>DkZ=lN%-*2}3-OZaM; z-KI_x-zwVd@yr-fvKJ*%EI`YmN_NCUx-eeH2H9HIAqoCUn>G6pnMPCmKhLGBb?~NyH{1E*$>JJtur)aY-`v}ppFhDz7^Jm5 zapCeyZTC4N4vJ)E$)40i@!H3t_Kc^vLH}w?T7ArZ&X)paH; zFGs68=^Od4s4qDUz^XkmMNmJ23up4^c2mFf6gL$;zh0l5S!z965R9)7h`=+S zbZ7EW$6G<4^;(@JS=v%btt-4#Ts0@;71x(k#zAEQ-7l-A`hD9zaLSj8k#Vy5Bl1@y z_%`VCbgzutkK_DuSJXFLdUJG?$^t|*`f2#dkjAfWutLn6v9aQZ@3Ik9-HS&id9_Nq z6cu{wU))kd?nXA4+R(0I5&g{N=@826=#k1dbzlK;9{3#W4Is5m7lG4)p}W?J%DmL@ zN5gLhPku6+5Bv(BnNMkvvVq?eOB>R}g74u?C!9Ov(1tbM3OFEy?;h27tx_mtc9(b5 zWmk`ioXgm_Ex?JHU)xMcIQ{Fmrld3mf9 zB_RL>eX6`s6;X;W4T_m17Q<4*$5m~MlbjUKyjGjJ1LdY)Ji*V20pkY^nh@%cwA9Su zx~Psu)Q(xzLe|JZ-G?`Ug;#gppA?SV3Tx}TJ3&1}TU&KB(#}mY8i%qW-W5LxoAKn- z)0vT-CW-5}vOUTaD%)H(2BgpiV*p$p%7(Cs5V(xyl5E$W+7|^{2&3j>zQ#YWKbqqB z!!x+=J;Bu%g211<#9g4FeKm#HG?KFCinQ=i!NB5LM2S9@nj-|>Fc*?DftjCMJn zSwr$4avJOt_<2m|7e;T-&DGJ6^~LmbL@8iEtv5b^SI~aI^wDa`0F*|Ukf)JFJAe!Z zG5HJ5*4R6i3YbcvtDd#atn_=uwL*#{89_Nsug#gjdJpqqiYLaB8m0vsnzvMk*4l4q zCXze=8WYM1YF*vp);-Ae)MC3z=cBjJhj&=}AYNfAd>^fAQ@fI9e0Q|1>fH`sm>;__ zvQ+S4m~)NAggSP)gCW{*wpFaxhLKANxS~sNJkB35AxWFrHh3D`*X;K?|MvDOqfE|{ zDEXNr=$ncDc0`WrScm)VpJ_D7E|1qtMvC_5+0}aRh2w8>ZcFjswq4ggA)QE&Ilq#l z4hMz;W2YiLKBc2NW0VX!>;+2duPYF78$b7H#g1p*Euz^k;n&ZYvPdKEteh7*d*0Eja&#DZZTHMN74(0AYNGdmWKeup3ttL8ftKow%s99_=6 zBdenY`m-LH#u?*VqJ_XZ1V!rCjM2OpT2}?emX3Z4^Z8UmuU>>~iKuo3kCB+R;oQv5 za&u*zNGQgzz0%h3K;V!5=p=Ywr^^on%)`lHz=9f2n1^zh>{R)!d|{vYFN-Xq*p7#5 zxsR`v({fX70}=;jCQeaBjfE}SBG6zpZ!>-d`h!~c_n%1^u^i6ha~9O#^_(`xyh0}0 z&5vJ^>rMY= zU{9=ijzF8qs*s}&)|RUJ9Uc@Y<}|~qTpd))5n%VzTtd+KqshD~tfK?MEc^?(lgfMR z>uI9O=4rP00yn&f`fD3Ek;~ccC$U|Y9nWgxwPin!hyokj`j-lA!}8%M8Czf$22fvN zS0k)PAYPS6x0N%_mP0i`6;$f&Oj89TnuI;gC>RH_wH-41W0dE5%hB$7Vq@+gM`8U z@ZCL+I>bEK?Z*>Xu_1gMi&C{3I^g~FeT<*8(qp^I%}ITOJ*|4_b%SAWE7pYX+gJlS z0goERL}=2Bue*KDJaetYA%2F$r$sfRaI)lT4gV~rtIj%uj@J>dost8yJgfyDhcxb& z$g4gC2MB0q!NQwDgDkzp%SZ~S#tPYwjoH?yt-^>^4U? zQcA=5APj&#GEef=_%^3O*ZzyZel_UP2=rb&wI|fS+0?SX}c3!=C5!7U}==`Ww z01x&Gkl*<+xgQ&?*`E8|H7BjExhZf=?|DAs>&)mTf0y&| zs9&1BZll=~v3gp40r`8{;k$iZ`-8yP?XEZhAZ^S*NtbG`93v7TE&kTt?fPavH2KK+IJ zWy8;Dx#H9sitb}4&|`QT6Lt6s4a6zTJBA?C@V{=&ZtCIP?u9BuZ`FlXx?2jEKUhZ| zYyzDCW<8mgV{)`FXN}|R7XH!~7BY*Xr+mIWa1|Qxs&DTr1*4l!Q0Z3-8Vcx6-^5la zr#z@aYVKclLO`jkAnFXs6ymo@NCSeF2aOMNn-RqxI?x{VoFYC+AZgY?k)%A#ob8yj z3hp1;6IO`PWn*x`-sXU4V{K4ZOMmuO+^QT4*Yj=0MaWmlU`cZ$$$*^ZC zi{hjdP%~DGeX@MN)M&!89$gWyoUTr;m~3S>zm6gSs$1tCowTO;IZh|B~1_cH$Mc~$Qb_wFEKUQaEr#CIQIn7VjvdY%GiNo8ICLcQ1 z1U-WM=`W_)M~mx-ofdO+%bMpFbga-ksKv}T2!yg0C_n`hblPawM}YQ$25)EQFI^u2HZ6fumkCn+FYUZTPhszDn+29wklA zEMgQd(U7d$tiD^@Hn%%ko<77VuW?fmg~_H)Ta=cHPF5bfw`UR8#-9LDQj^=TUY+-! zR8q6gwreW}EM#G%VWM~`){_+i7_xfO>@ldz(s7#I{C~KVEx**JF)0B4Up6JfFE6?! z9%RCcD4SNV!_@i*kyH5$>^pq|Q*~+N@T9dDz4<4og8!Eru{>et!z!S6y;&V(Z0c_O z-Z*Xs@)*Dms=3g>tuJ4w6*F6dfq}Rx&p;@4^NwyI5@ApWYdT56`Jc>hC5u`$Zqino zw)LxfPTV1e^CMOr!uF1u;21({(3SITy46RGJ&u*yWzNiaq40j6dYwHb$o3%EHKXwy zp-i`(dFHJe!R?*Du~#l+oB-@X=gn**lO6DD)M@BtOV-U6m7Jx2u_)8RHGoV>7Hr%4 zbR(6g1Wdv3{d?HHd$qHdFQ@6Yv+vf8+G;e(_z8naYY{7s_=|YA=vkKj^dbH-KiJHo zSLc_;46!*f$DCT%cCT_RM#XP3hZc>2deu|g?6nBE0Bv|{Y$9g(Od1@Y7Arr`=f_-M z3De5R;X!SFyoS{?VJ%)IAk^SzfJ)Uu+PIlPgN5o(gxd8DVoXp{Xt0W-Lb`|rnhDk*_od3Je?Gv%VK!1x3~j4Xx<8ibvaf zv^&AJ|EoAmCAM9MVj3+Id^7La`B8^30ZNQ<-GY3nPX^S_e`t^uTs!N~UX(ed&tM68 zYvYZZ@4iQ?mcKyr8^cy@p_(uww;Z_G?lN-kmz=9hnRUjBwEs5}p`@e;s^9QGnVd)Vhuy+A?6NfU3<$8?8>%r)g1{g%6BG18c5>&r z6)7m8rAHJZ@+cGSuW}S{3|PG*0TYTn5v1`#UB2m%)N&2;wpAdbJqMdW{m)|uAy9E@ z{8Ha%xjAA#T5f&fuuk;3zb3zY2qg>D85rdRa_XU45f|{rYzoZq*`n_M5(x;t2nIu_ zeI`rawk)`e;;ES6s*4a#O&3nivuxBU{JQ*GB43 zhRv8`r-Jl&OQ(o9z*Jj8!hdrdw{TH~eTB}BnKVQ8L|scE7-*#-#s|6ZgpeVH!!wVF zODo_1OCEFwX55G;f|07g!op_Yu&f|ty_t?FZ*&-LuS^I}9$P)_F0SK>Xf5#%QvpHx?Ywrn>VG%Q7NtZ5a&}93EFaQGs zbZ-qpBGm-gGC6n`p5^yBrnAA?UhSE4-%`i|B^m)r!$(K@Z6Fq_zd=|pYu)xMQ2;nV zWru5AvchLNUr7o9XIZ_8Q2;_12M1>jwpx{D|BCfrR$yxI739Gb{s9w`EXpm4kZH^c z_@RMdTXU_ErG!O4l-YVRRigW8vIg%Bp8jtN21^XLkp!B>paO|E*t-AEHWEr&5F#vU zB0I=<30}=KRX&z@W>yT`ru@e;XjH#+u8AT{Fuh*gsJ;Dlq@OXR?9ef>p4iaX1J5%M%~Wp|^MA(RAR+xS0!`W6`$v)OnW z2nr2_uWM=E=J~JX^zE%Vy5?3JbZRav1rUD%L7sOG0jPw$-1%mGcqk)Yjm9nwOnLyL zUK}SHSX__X^VQ6Nq+937%Ul}@`e24sOTn$zCrYU?$}d0*X3(KOx=TcMhKyE<{MLNd z=mF{%AO@2X6INBp>0KzjxK+%o-~@a$x#0!3pvCUnj|SW%m7Rulu(09Y zVdj!UHEI$g=BdO7+FUJAZl>Mh8%Ahe9XDuXrDfKB4(R-nTt{lt^4Z0+oscDb&78hB z-*HE?CngbO{zsA`fCF>Ln=ouuF_P;v8?a`WBN*J0i1=*|8(O09?ENvCv)(wqdv{Mf z3Y>ENUg$`?K&@2|QthWA^=<>Yn*pUATQ7E!_(U4`}(CLn11}C8iF88J>OM`aayWw}u8GtX$8Y@P7aPyKXJEk40#Q!lo-exh| z9PvZZ#3Dz6L*x%G&7lOLXhR`J%bm@iL+4`u+<`U>#SHMOWf*?g_0w2mKUmKF%{Dr~0HYfOpjWz%gJRD-cd=uB2=2IY2Mn`+RL|*JfmM4fUY)164an-4m7gb;$!t%x2_ac$_@(tL%7-u6zoS6#Lr*X>|HfnNu{25223TYBUD@kNNz&+342bK2 z@PvPaXAsj201L1~!WDbj-U&e}ZWl@;MPJ++#h2j2$ahn?+tGy4negLO=jgDWFJ)Cb zV|cf9UZ`Xk_Gz#&Qye-w8}7oiCu?tZ+iE!3@k%s!HM;!a==iiQ#|bY98i+tIi${cA z%A(H4cWj%#W0cLt68vShKW4(z`{d76Szi@3N*&Y3!GpZ_x+SPr`?ZTO7K=0yLH-8F zDZokTh6_dv8dekeRugl^$t{|;3a~9=QPZ#Lx_zz^ytPXuMyvV>79aq&_+&o&?UR48 z9p`5-us=M~zMc-UG?8OTF)bgNq?xI;YGh5iuSCQsW)^4+-Cub}kiPg>Rd;28(oxMu z`$t81GOo9D00X{1+h>A=H1lp;8jKNSPs6?3QFfj;b4jnOs2qmK~&uu&&CS}SSv&I}J)`1I0 z<2(YOnrEw)oiD}P(NNTI$3HEo)>0NiQX}W}VtHByra3c*(^zyEv55O|L5(JzbUOm( z1^KA|+o6nC)7u=m#~5(DHFd>`IRzq28Q3Gx$9a^2P1Vyps%y0P(Z z{x|y~Q8tO8FF#md@`dIuYCc|giTr#QGV~4P{@G_fU}^8#^x0Q^uiW*CpRpGCR`iJM zL=NoUFOHYNQLzWOW-p6D!ztFU6{R3TbJ(YsfmG zF2gkM%WI@rL6>|6+k<5_An3}f{rm}>f7dcpWEAC=>})D{i+N|XQ3K?2?hySEZv54R z5|O-e-HMAjwci#2Uf97+>HHWdQT=!~R}$m{jckF%$9L5T!a(S=;8~yRKp8}#6JU)y zoY-=O*fPJWN9HIz7&dL@^l3%-UvF0~64-Q-1&*cVT5gr?|7EsAk*@jF%;_EI!uzoP z&tL~H%f9OwgAHH?Bzr4E3o*l}b+d+s#;B6VlB!iBuDH_XL&gej>|jA9qWbW^%JcGT z#!H@(b-TX31uBsQqk|Noc>rVQ|niQ5n!&cM3%lPed#;eOD#_PIYZJm~HfBSao3b_}|6fr9s7pdP9? zhR4FTe-Tm4_nd}M>j@4ZfpFXr!7s0#?@U{S?fn)3?1a|w>>wy`9x-FC`#<9qbTmj~ z2LoGHtMp&*|C8K5#iFS&1R)?%r8%p5*El4Le`I6az}Kk*n?@}V%D{B|s~V?nc#W#CUVzMIKizO8B%FIs~qqv)M@l&b-w4$Tk)$oF%4(-=CZY( zu9&ciOoUzF($J)`Y7ec%w5^!@AB16WTSzb_T)7sk)1ZmIa3C`N(|+Fp^A3p}L|KK} zC$K1qf-Zx%jw3Y8KGDkT+Jab&#$B#z&LetxnYbq0`hjz+nK2NG)2@p7#r-13Cg`&A z>Bw1ACCD<+H4{NG65Sm1vn~b5{tI%TK+b~ZWzioGcc{Lsvs#V3 z6|^v+y?L|8?F%wBi$ooF3c=+)9rO5zz(tS;$Kk$NC3D@5vRw~MHK#H9OToJ3L#UB; z18ye=T6Ca!8OGMyfRz2* zR*hw&I`w`1#bRU;Aft^f<%w>idR7ZR{O?a-;Ax|}p7lNXZ|<^dd#qpQ9(`Fi-fi}G z0Uj;l@dbwBlC|X<$N|hfMjpl1U%#Nf@hVedMyjZ&RJ>mj6VP~SsbCye{W-KNzgbgZ zQR7}d`6*3Ck9Wpr3=SkPgR32sx5!zeE;L%t=FuR0M zc~3KMq;wW09}UP6$fTooQ6Dm4FP<_x;ugd0w_d;w+1W)?F0e|%T#bYiC%qgcV;uED znC9o)_kl|-8~RC3lCV(Vy^%XvX@V*WN*XPml*M!R@r;6fmG|Q3H2Yba#Z=Ou2qn*R!Mhkd zitoXXGm*$5uXF-5A}r~2>~l9o>tpqFWvVohe<}%hM@KKlTKlSUABk$W@0S7UL3Zz@dNrE%vXFVz4iIiV&!YE z1|%6v*`%;YG0JMIn}%vZ^DulsU6~CFhND-ai@;6fn7=X9)+9ep1AZZ z(h)Opid2<6E31H}gqvn4^5~OU!#|8#c(GKjblCdFZD3VulKA9He8q_)H&j*fmU*<+ zX9urFzG7k!ZD+F=x&--4X-jp0(8WgU#SJL#upiZX5Aw`!YVz9WfwDzwLqo=FZr|&N z6O1HVQ`f>tWA3|TTA*h=Iy)4NbH#%s#bAwnmI2G2uJdLvKf@v;D^>s=uUsSIu(5+T z+BF^|^-Y^gnZ^}Y15wlAqU{Xcse$!ffR)n4GDnhO+R2GG*9prSbK84cj?mGoP$RfG za1@QRm;EdaH3I~m=F92pk{6(Z`c4C1#-)aT!;glcKSP>^&SWwZ3`U_LA?k-R3I$(> zOv3ZZ8g`vIUN)j9Mofc4gLl)#^yH8Yd2*~7e=UOm&Ze>TD{N<&Mgcnx){$E4Vk31B zat}ZO(W~OVpZuR)7OR(DRGVJ)I$ZJKGepl*>$IIkg7Gtxmy%nKBZ~U_o0kFp1Ud_I zMYlxL?^`;2T%e?T^kT&_AjZb!6B{O&ob`GR<5g+HAtF5vM)p>=Z^z?V8;%#hSq2cO z=sL7SfTZF$NedM$4CDBP*q>a|ITE?=QOXdt7ub2-+j5!RxzoJegS;S>;H;E=aaR+{ z>UvC!`k+%j7!U-|+&3=&{IUPeI0=bn?*kKUKu zZv|c+_Mh(ieM{6DApLQfvF=oE@fKM2iTIuFe|sXFsj{ zIFnzU-|-`bth*wih9<#GR#{w}gZ6z5n+tv2l5rmknuzsHcBwou&Mvmt7i5<(r)VNV zIRgI3Yr6eK+M)C4w<~Q`kkA`xoe9MWUUOrYSrJS*dJ9<~YEC-!g-`G`FtqvBX2lk> zl!Y3Tz3Koi7?&cXA1|J*!TdF>X$lj)Ocj=&-4A?K`wbtRSlAP@hFN>OMzL8uU7j;z zXhv00_n^Wn6*^pu#mp7I#=#u$h3_Y<(A19+zbtMu<3W2QhT{=2=+@h1ESc~$L_<}x zzEb(6c7Ciz)&Xc_7Bl>ov$GbuU`UkDTN^B@&4;h|8FAfi3hmKi@9y*BruX4z$_MxK zaA1njlUw&;N7%^XRldo-yK?8um$A1wRwaqiUhD2UiPT+v#<0AScU8}6#8S(yuH!NF z)`#%k9fo*OD2r_|re^lLHbz8PCKSlhO#nCBgDCzt7UW z7BXo1{uWQ_3uPdzVBeF~RgD52c3u0kaYO0));$_-efKm$3DNjU;8=tiy=$vJp?HB( zn0lw_>Olig5R8|+$=<+HBS=xWziM_l^h*QYXM)Ojp3twN$U;I}ERah1eH_YK?hEk# zTaw|KFRQJLA^TOC?6s4nhkjx|9^N~wh>=90(PDJFJ`w|PA|+i3@1Oe#D-dlmkinrK z!7e%!wv(U|Bs`)zKY22G)LIQ{(&vfB$=gxkK&h-cHcdTYhUAfMoHvVF}T4E~8W)tyn@YKxl) zy94$*+kRdETA%0TPZFubAE=+I5QTJFW)7TvtwuK$Y|DJg1}_^OUY3KANkEmOU?iXi z73JIQk&Y-2KVhGX$u!p_yuK&-kB75Qk-7bNW}q+e))KD<-d;ejN7>9ZEex5ZWxNQx zI~JrAlO%libG5lz-Q|4#LDIWSvZ=q?A^4){5|QYfgiG+51A&MClAO(`mM78cQFbCH z7Sa7s`>Co)#Tpxw76L=&(}mv87~gH)Sd>IHc&VYpXx3w(5Q718z*6zz`*Cvr2{?OV z582BUC83ykBL7LMid)V9@*7qeyT(}cU3GKJ%*N$~7Y%deY;NBb^4E_xJ`0)VCDqI< zwiEBG;7(tI2gmb}CM+-gCW4{oH)*Xe&g;df^i~g`jr^`c7`c8s(Mf5J3+C9$TdshMW@@a~nW%K)0>ruM-|!$2zaD|f2?n^ncau>&rA$Vq~R$`A&q zdf=W$&O2Ts4}>PUxw+Y`%&S3`Uj2I4Rz-b%2ARR?^l$3r^Bbv>`Y8{%`CfJ-73K4z zYWt9-x#~)Ku&q60wJ-KLSW0QmXd7z*cO&1Ks76QE3#!;OYGKTBrWlMO3JD5I4i`0c zYNYEY3S3U|nQ1}#m(QshaF--#kD{p8@PT==*tC9m57q;p*GhTLA#+w8;5%YAI?|Oa zJp88!izKK4w+dC`K#{Kk1hh~@zJC1*o@k!KhTCyA@ugtetiDm&-crP>apkzzB>sY4 zf0G7i;|h>!-WfVn_$t?N*n=4fNusK(ZIg?Kr(Ph5MvEW-jwwb%2Oi_x_y4OH7J-y#_sRvpu?#!Mv+QeF8) zJ)I~;PcNDI<3TiM!_GNDzP?`s`*W8O~=a(%dmY+107){qYZJ|nIt=p7*ssdR*bFnCshHquEcktPRb|170xPS!UniR+Sa;mf-WR{+?v2kgE#YFKJ7n@E+_i%?WJo_zm* z0aYv$Oh(6f>!>>BJm4`u(P-jZBs2346=zDl2@U$tyc)no!jdOgc<35uJ7_>i%cWsO zZ|=e!nj|Jg;VJXPxKl3DH3k!f5QRGGu5@VPY`syZzFI#BdB^-pR8-VjBfwxx5*jqa zOe^RxXG!a91;~1c3Y-zPzF4nqMd3ohz;K!DU+bOsr-^Msq+sevwd-ve5rUQ?E9g6m z3z;vq)G7R5XiBvxlGGh5DO3(OFpkoySRhwL8(9SOgwS8bI)X&cQr`XM6hMedI&htF znxsTP8mkf_L=BG?g!;k`rpm4ZWk1mR{zFkbe;ed+tWIuR6vhe&w@ zE+!@>9`5OG@zU}cvg0@7e3K>|NBq=b6T-~$j#Q66=~8sAZ+}-g1m<-14jFJXX3DZ* zh>BWTG&zMuYP?=SFH`^omCh*B;LXnSPm(a(61F|Mw{9-ZMgxY7iNv6%#gTl{y}r@i z&REq^aqvl^(!?{VMq##Sia^6a{49@p#P|R-)>w$4|J0tRCzCzoUeFZ{Zaon@7)`0* z6h{`*)x{SHbBaJEh3U52l~bnv2ZOrXdJ6J2a??6WV~P`_x8+=&%h)W0I=QNo(71+nr+jPNdFivv7Y_jI=K3>lJkT%Z09wv|QXUs_3Q7h!l_S zv(LCsnMw(QCn^cLRj2-U4T$IDyvy%BqJIx;u&J{kx?x~q(|r^kF1;PnbPF&1Ha(Nn zR{-o~-z`i|Hvui)SvKkSV0cTx>6ZudpYOd-la+||ehXiv)22v%HLGGXxuvZo8NhB! zYdI6%_Hu*`BES2(tX^fl5Tgg|7xQRXmY1H^9NxrakMgpVz6CYm;Qu`wAr^%ZdTr$g zx55G(S~j-&y9Gr--CsyGE({p!D{+Um6v_c zBt(oMQ*a0dav0hjL_I31W16q0M6@hI=fVhWZ-F$Kg)-Ir%D?9)pZJ_vT&dk0J8a0-0({P@3jPB#m`@;Ft;P8Q%j;=D@>8PyKN_ ze`@Dk#&+tRe|M3i)ug4)x$}LVlh5&oM3rUzm1X!RuA9bvv;l_B?;LFWOVMaTm*3NDVkiB-WP)w|6@#oPR!&<*Hh2lZ|?N&tkA$srw;URp0W90{1_8} z`-Psm&4hsu0#Yko`Z?2H!a9f|qw+n!W${^kWKp3+5y$uSHnTwY2S#*W#s{{Bkc0iS9DZkF$UReDz)Z>84+0y8LpkSjbE-&zf+NzUg&c79iZhfbI) zFHa>Q)ROsJp497<3-aj6N0;>+C`TJd`e*@u!;sLLoVB^b_hLrt^RgYk4`3Zvs1R9= zpMW6oiW9Km2n>JXl*>n7G6?x+Gid{%vX|>1^WYYeeh#yLWFEQaLW4$AP9-*%5;&Q| zZ=|8~qwonNm!yUYd2R#mccBUDuqeT!p8a#R6k(9F39LbQF-hyDZSssfMo`a*9mw(# z>O3jwriFe`Sonf0692Qs?O|Mx{k){y84I>BRIao0`Nyx8hK2W-vp#%uVW)tE(O2BQ^L8(yO2#j2t-vCBpu zP2VV26-;R4AOwAXM~U0>&7?;H*$5%yv^q>xYot9UBb9E?cVkSd zSw$(2_^ytJ+a{aCG;IB<`Nt2Vylaw1lWtbql68Aau6hk?d$CC9l&myE#iV8yHCzGIooDZq^}PxDz7V%%#@@?Z5jXH(- zK|*B(Yl`o0dySwn3gPp?t|se&r}lzf9{#mj`aSDNT6*oHYT@1C0Q>CmH@KZCFw|nI z1ia@>^M{Yk`FeiBCuZZWl}d5wWSy}^)#ZIx;2?`~Os;wJcomhO=UF;ul ztRN)gdJcE#&}qP~UAXpfP;Y(X$=_ijjCE3Fuu%Oh(~JMUSzJ~Z>-!dcURT1%*Xq^l zBO;P|Z&($zah%Y#JPD7xqzw(mtd`PBJ1|Q{daUH4(s$^Px%sSQm*{W$?Cf zGU25Tzqj!6QrnT=_Zu26PopYKseju># zBIE89jx$H_a~>GjV^?}tr)DX=i$Wb2J!qE@v-ld_`!8m%3lz_&686Bjwd%S;@fA7Q zt7ct0Kj09Cn@Y^fgr=@@tFQ*o9~X{>8(Qe5E+cai1v?e# zoNSNjrr58=SsEfBZhxRyuZZ8kfYTfrNtCJxpy>Ft%x`&O2zYd>c7k{aVOWR*yuxBS&etEq|du)l_Oq&jjRt)8=jcO22znd5ulwO|qNWLwMM+@ypyk-KxAgMIOPaf(NQgKHfz|QYy3PpqQwQikCQH4& z?R`-2zHr}cxh?tyq}&nF;Y$8)yAjA;HC?1bBnt$$54HyCU+^EQPWNLDQKe8~Fu9Mj zIZO#BK+EAkDEG1OT7F!tWPjZGSP21{W$q`Q>?SOsWr#C4oy(LpPzNIOf5T6^ApKrO z$AL7tU2|2Nq&8Z+fD^cgiIYhTE;U@|4TlLrCcWdS zjq&$ea0)hl$c4c}7uw-Z@Ar45xWLGjd6v=(8A9()y?gm_0Kl;}-(~FQDI{+bB36W{ zw5RKPH(&NcTg7Pgo(h8+WMC}A5vSzQfyOf#Tb;i@NB5aKyhsB{gB+r;<9SREpC*YZ zsz~J%+MXVDt`d@{ls<Y)lsrKQ%s`;KtzgpxNa@cuge zcv3+f_5HdO$31!)AHMgJrEQm;OxfqU;7KNC+AFrxekjeH2>a$V@)D;59jrn@0%5qq zGo8mO!l_@<4&%jKhxw&X*==V0z2*ua~|;%aepAPD8o)~TE;!kS)b zRW~e{Bde`5I7hZ4(ptV#kRot)&rV|E|N;h%$rXQ#=DK+wlQ> zEI{k=PLEkSPKd-}j>91bnwH6Y3xgT*t7q__DWOHU@h@?e(uo+D1_S68=caMXhmKzC z*GuNq_i!}%^%cpsZ%09L`PHAetSqFtc)S?~*~%%QM2A4x`mESvdt09sADk>atjX#z zx^4*meh3kU-NtBpdt*VMjkE_4KpAPQ4SgFOK-0 zR~~-*Idwc$(fIzES$t+0h^fDVOsD8UL3M z0c31)pSn>GZFKb{izN}38iT=GH++6LT;K!|>5tshoM)%uw17g^+K4LVvPwia0BWhh z6RL%g{s8g72d{F(pmeh%%1ToAwEhnh}nfxJ`4C&iUc;sM$bdc{zS8q#Vxj4 z>oLsB%=uD!K3xO8s4Z?x@shnCC(!J6wf43HKpBdkLK`zP;Xbcu#bD7N*wEzXrU#zyNorAVQ zVgFnAoIE&|mW1l*z?$8Eu!RdO9)#22Dj~RB~zxs&t$8?>dr`YO8m# zYP_1z;6$lx&YRmHANW1pGDO!xO{ksH_KMq>8jX^8aK-VMW8o{OrNS zQ7ff&R`-#Yk5Mjj=DB>`(=26d(y}F#QJ+3)Kn$&K+nYys3Lf3RZj+{U;yMe4gv2cF z=AQ-;3nS=!k*TzN3C^BX>l()fuL*8(HIr8nSd+d_=GF>ofpn0{OV73|IZzRGL($xzc9&yd7>( z+*+YC*V)Cs4KqikTy@;)o!I$;GY>#Zi?da=$=q_^$*XGkpz&YU^F#@$^ZJp-6^YcZ`}arpv$vt2 z8D&UR4X{CR`ZYC+_w2X`0ZMDDqc)tRK^d2SNglOe>6%5Lxh)$m0Fg4BHq8&|! zlo#Ydw06->2~;P5?GQFlR!$H$0XZ~6dwo-Dy-ZosL9qk0yz)p1kb8tiqgB@=H^&oq z%>@REEFv{6Cr5!Z3|YjizDzdHfeU}X+)1!2dGc*bH&0!Mpjc*+fr)CP3mgjrE8YaH z`B!X-m!Jt7E{aG|gt7=|LUoL`?LvM&DX@58DPFz8CBw04ScZ`$bKI$w57c3>W#*73 zAh&7v!&BdIWPG3bEDRTc(mL_81GM2-3v(n#wANlW1$N`pQYr3fr00ZItF3%hJcInm zT#-%*!`s76k}xz$kS1Ce9}fyi5HfI77oi_+0vBDv;pth<+^DLFa00T_xzMyd3$h5I7V8Alw}csOrF|-{ zJk)U6`_1r-f7sB*QR9Ym2YjsXK4}^8`jjTuS>hlrZGnCJaxZQj_;Y-EQa(~;%?kE< z?4afBtgOBguL)7fB4so*{>}Q)$sLf_V4(u`jLsEwE?OBy`#Uhy(GsxvtskZw1J&US<4~k=Bksd?5s(2h@mK#;xZ!r0BnuHmIoH|;iQ4HSU|5Igc z*QVhFMFe|6_b;_?n)tP70`{SDS<6PGW&mZHTni10x>14Ig#c(V__vk-v=8dX!**L_ zDU8kQMf*tRTNgJ*`Ha4*!r~W#wgGL^ngn!f>eLTTwsNXtF_$YEkW(WiF(ZR!>gwd= z8&6SiwaMH@_Z69Y1KV@9+w6-IanIxq%+-~yXEfYr*Dt)PsJOIu1-ndLKN)wzD@41; z_oa(w=H})=P7i6sI|jpth~&|Tu&^*86g8~qB$Xs&DQH(bq+1n*Kwx!m5D6A50|Ns) z1qMkU-zY2`QpsBWHxcsAtWoXwm+o{1&!chvdI{%NKi!jFeXNPz`l|*CJ5wTs2j4Kg zJ_+dmP?%lp&j`=+5nW%LtFB%dyV54jBfioxc7&ymMummN_0My)*n)tG^pfPz^WsKl z5*{Y46)li*O=^QGt;1on{Bkl~(A!X~kb5_^vCE3Lb1fK(@&9A%8-pwBqHRxXcI>2M zr(@f;)p63XjgIZ4W7{@6wr$(iJN@0dRj=y3{qxk>wXybEbB#I1n9~Ck&=8CQBV6d_ zY_1u^PkZtypDAei0ZeUxog`}qMjeA=(Y_G(0n+2%|| zqayDbmCm32G|K!;wL;2}e4UqtOsH-d-vEGjgs2?8S^L@DZ`lHrz_N{tDDhnvk==v@ z2?KSqDP3D04?r24DTIg$dRrbY*^%hF#IXUu&ktj;dh5dCqhJH$^F^=u8nJ8LcViKJ z2Y^@K|1RHU-b#}kY6!Q8pbg-K0bb)H5K5WDUqHBc+Gliqi=xbmLmPAkFk!YcjP6PA z9R_T0SweCU93_W^M@f7^df^z_cqg&QFz3Mwg%9sXjzVMebu!b;CFMC3x+=)>z7qI% zhuh}-H-{U9E3h6P@OaKY^5BWr)y!!;CPvHA6zy+bDH1FGp-j>yjOXVYZ*>b=yZq$p z4N7e6oSF+ExE-+e-)Vq%l!Vdk8BXMoCZJ7>*e#b4kh22JOEOlDRrt}cK@|Tekum4f zA3tX7y=sc`(2VX+YlJVU(E&?SZvvu0L1DLfQIa;szH>1tW#Z zy#6L~M>?n?w(u>BWd(k^9bm4@2!H}yV}AQ4ejj}fZx8T~h{i(>WGlIzBCM{`FKBhg z76JiH3B`vMG~X>xqmNm`!w70BagMNOJQBgR4SkMHtj#pN#mZ~K3*@``p45gqdjKEM zww&3t$G&4?Or+5J&*%vhYlD?mWzm0lGSJX!-AoCFC?bIXjeQAr-?f_oaGtzfkME03 z7>~xazj_I;64Mu^!OCY!;qfQ}a!7zp9A~;1yuCfh8cM&~6+A9N?!6bjs*+8FWRQi> zp#K|C0kCC+)ph-@?SVp(D6ivJvue%?Q`q#i{Z<{df|T^nH*!wKHVP2}z;=XOuB3I% zVm;SpH~(8ITx$*(I3axhYLIhVtS_F>&etY)tB-+9b=g<>M4GZFrm1rVxMA);!46sg zY6C6sL8dF_Pb>;1aMV zBVE3wk6Iej_wT(T#GI*r*#DD~xKB$IM6fK&{Gv{P4i}#lKyKG_$m*f0-8F`WD8V3; z!CJF7EiAPz?n&T@;1G+ zf#{`x2xvRdUa^4Z8sdq0-^}RNmEKwK7^=bw8U;QPW;b7*mBqw3Wx(4KfqtIooGh${ z!Nkw^N`CuBT8c~v@XJ2KS~+uiBz&vB@>6KYbI_#%&jm36mfc+itW7i~kkifaKCU2H zwR6c;W&!=b#4L_SzYR48dRu=V;$hdlhIFqU`7-%#Zm$?N`()7i`?~{H14<>II=!6+ z_V;U%!y$*?tZ697!_ZNl^ihND_Yxk0*oirS-86o>EW4WV*$jXxz?992PQE^fKK+GP zeLM~noaQ336OC9!FlUF{oeqySxul-MK>WpzvU0}pS{sO7Bn7Qv{qF@`*PDE`97w(I zj-_Xv$%vQeL%kNZ+|c+Y$fpT^7%kAR?Qdx7WfAo$Ac<+cw~_Tq=1DC-LX={tv7!e1`ugBKqb_hG#egnu{`TAmCri0cOBsIY z2$>wopQH3y)41??D9o32uog^IJ{uwjA*+3;QU_U6`i$`&Kq*K-A83%8VWUPFZYNCn zd+*4Um?Tt7@qRiyePB`|2JW$#Z)llTrOm#7NiUifgpfq_(h|)!PE>FY<)vrIij4>~ z>lRf$mq;0Gl3umz)T%iPNg%K-$;4mt060K#{v(xG_0spi&kp-{oV{RAk7$yZ?F70G z&(#dmcv43Phe*i$@I2}sxXalH7A=H1xk^}dlfuChhXxG`nt6293sqgADYiySe8i(G z%r6#hTGq*UIa~)ry|EMfL_6-81W__knSdEmZk%WZ=v@ynrP4*B#xET>i+>>LkN}8` z?B5=(Gv$_Ql!NC^D3z%l&0X0vlr(`g={_!CYX*QGdJ}if6DLy>liCuU`W2Bj9iW>S zva+=?dI5otL)~iXvX$v#qUzmn=KWEXN_0S!bmgyjjT%qp3vpvxpTV3OtGK&+Yp($% zC2fmzs9UJw2A-;!S(MtIGT2A}XXJlV&gfCcrQ%goUhW`ns%rx7&oqC$Wo!KZ$l*;b zhp-H5dFC2X!NOWx353ui21-j^^`Q3t)5Di1>i=rJVA=ceCz2?%rKN@YptQ#vyI~oN zfq#dbi#w~b#fR|=cj^YS{EwuOyKW(COdn<`Q*Eu1O{yo!2kdtqS{}A}X+umJgHpJ+ME9TD zIW_e-ZutMfA{UPTVUe#Dg!--*1>Xae+r8@g?|UdhI2OM&SL52zrVM`=+iV2lHObZM zKehSsgoL2D{gSEprv%4dK(Xqhj#%k?0V8mh=T!2V7g;TcQ~c=?$}f`C6SkNZu3Owd z6xgC;(+NsH4M)i_(CL{#e>6Nx)qSLIS=6P4pZBY-0B8!!V&#sALWV^`ib1dls0fQl zTrI}3L3AZ5Jd#`(Sk;G%&qFoJ*K&GdS&QU|9y9*o!p1Fvr-W};-6@fS5!$j_W(Si5 z3Uy*$`+Fo1*`}$GLy{6^Uqcxr5 zq>hvw4l_xsxX1zDTnup;7dPB<09!k5Tc!nY>kzE$cv5~;%v;~jcUMg2y=>e%63hfq z;j8}=%p4>$ZdnuO&&LawFc^)}sLy4eB>oEV(r8l52JZ3wQL5$YEyGco!sjH;=k;yq zS%&F{5Ia#F>GZg&qDc3?-#mgrg;4f91u5(XVxGEw8~Jxy4ml)Qkhr87UEP+7z8UEXa>m>Z zix?cxfZ#HqKW55zziIEaycW{LfQ>IH7&@5OI*v2fU5!XGkwlIeCHk8a>%EIEGrBpt zS}onGV^$RLN)sS!EPv^Lms0IcT9l-BiL%L=T2?wX*7C7eTt{_RCccF%R549SouGU_ z^c2Rdf3KmZZ^yh>ZlZ$7|JiHiG|3f{K&%T@W}8ueLaprph!)QkA;yo&5xRl^ff};o z3aWkGc3d%02SzmuWoINa=_c7%tXBX4fw}bTT(sRZ@{$o>Xy(W3=wQJC!KAuLA3(?G zU&OQa&v!fe&TINfn{hYs&=}+L{sf~e=Ej}NSv-C&ga=P-tek?S!88LDq};huJ01kh zo-uRFILV)XTOo_2t70i5=F5p&)O)kkyBZ3F_xIe5UG~ZBR|+VKE|}1AdM=*U>xZD? z-I#-cjM(Zb*S%^b)Ks~RWY{_)a|$J31uIr?HgWYN$R0fuVgT(CccON@oPHX6D5Q$( z+!InFA_A~vdo4@=g{OkbnT8_?OP>!oELPI6NB!?~Q4m1L0nT6Nwe{ZmhMJuHwbz)% z)4aoKI{W`sW`n>1h-r7Gm-!m!Xi(sL{K*+foAgQ9c|vJQTAc?9C?nIa!R6fapl6;* z2Kb@CM|`E*Hr(mQZKgNZKbevyBJiIm?-2lLI*F7_Qi)i{jmXRp_3bhT)63I2J^WD; z?sg-aoSs>oAi01X^nW-)a7xwP^3G6tNZlwr`tr>+&s=ETdcQ|wuAGLXZVaInyBGMl{DANsg{ zP$IiuJPvB3`>(fNEE~TVhr`FGe3o{No3U9^$x(qS6kj&-ZU|D2 z63i*O=xewy8!&$QP<2z+_7(USK>%3xWvrXAr5)P4I+`Y}zO%{OzY6sBTBAXVr>@g= zTv6KSzzXKSwRoSzj0aBCfj6h@{1nrC<&l%@? zSxl9!R*cN?-iH^Mb=+;dt~p!x zcncki2KQ$}`h{Hk*nvZ*kEp0Qh8=_-tKQB;r>t3 z+c=QVwWXZF4ZZ)X1sE|x78)2BP*hYzMwn*22`G8YbE8IFvb5fqxKayK^4u^137CW@ zWJCmo@asPLesQ~4%c`%xq55!BJ9#}!0kF;-tl&zxGvi;Ogoc2Ic1CJDj^^cdczhb_ z(!IY<_5VG{*XHJGI+;z#%WGl&^_fqo{Tb46`bn>p(&f3LoFI;Pyybh{qG{{pswz$r zIDG33bS0frwBB`Bw#smN@_TvA)_i`v#bn&}1t%Q1cJq&u^LN?w3g<&T625FRk3Aiv z>NxM4Fg!ojYU-n}RncLyfA(9M21;x^WJ({(;4F53#fHh#t!0){l-D(*;c2{#@9%(x z@IS9pgdldiel<+ltdvo2TfeAPdOFr)6dbgyBFP{8{3pl3Nt4;nqN(ZW0sx>L49{aH z_L60z@j-IU_s+?^_hTiIp#Y$A6q2)DA#Cbj<*)XL00jmd<)8^egy^=v;cHJRQJ*x9 zNGvqaXs(@SBMy#d>L&eLue<3c`M<>Zd~Qz&{~%SrUY4`Jsl9i7w#xvy3@`8gyqLMZ z;?9?o!>P?b*pAk7+*_MX?rZTEk4q7JTx4F?7gKCi?p^XPNLN){<>hxf@G11x59`f{ z_2hJ$&j%s(By(y!&ASxa(jU6ivt(6ZEJ91QEf^RsDM8f zgClQM+13o0HLNvaq5s&*RnM5rHDasZfZzrLROU4|jP=+SFUa}k&HYBs@I1vyzJ0u5 zwI$Tw%fcE&^dI(X+rN6_YIu@>t#{u_9gLEfTYE%lvt`y$AjvU!@!6Ml2?VZ} zNtmTmBP!T+81guBD%R-@S{HF-TKPPk&;xQws>6*zMPy6I9prh++G#({BD*vM4?PoUGzXmw9TtKJla`E{ou z(g@iSm}3OGG7Ci8i{>DxkYVjyff^=@waGj?cExMK|N{p+YJV#Fx>YDugbE`Fjw ziHDJxLpnaGD_)QJ*xD&R#z&wB8K=V*^2k~H`oVuhUGBsCbi8ze;n(M{>D_96EZjry z^^F5N;tZjAeumit5~1SCm1hmF2WQq`X7~9uL?EefjTLOpfZA@EJ~rh?3} z9Ih7VOw%o9Ve0Gq!wB*3DzgaxzP*7F!b;}x<+TS#8j{6=DIf-g|uD#q%%KX%gD+?28y0po#qZgEWG+QSLEx|dcNttv)%$O zXQIYLjvO)Yvo=(*Z^IOf@kgZx2 zX4>74^~zdOK;(PpJKe*!F|f^bCA!R=3H zEUp471nPmxp*H^a*Hd-A&xcQ0*$+>y{O>o}XG>o%_6rmi zkb#EK#K}(w+O?N-v-<&?F;O`LZ^v_BOOA72&%aixAA4gE(fSd9xn*D?j5_7jF_w>+Bcj(w8M zX<<;s4KTY7!)!uAm!mI{{d?=7;;k0}X|3zC8*VG<;0_==8nJK|`QdX+5{N8RtB>DK zNE|rbLN_#|dV%SE07(nuZg_|9fCKbBgedr)krezh|TATRK~NSw9Mg)|H!) zQY&7!qs2@X$udR_g2jjf&GCFLFJ&(X-66Ondn~@W@)+Hy>wTqw9jpbInqz}+A5~a zb?p)FnSghaqq&3CYk2*;D~+?AU3AO!2qWzoDofE!&#wqcByn+c*=z5PeNq3$VL?7& z!sBUySoT%!hvj&~!Ig^F^9qgfVyZ?i6DSZXKZh{4{r3Q{DT7&ATqgnV zmf9s1qLs89HFT!qv|H--Y3JbSIInhjuTNeN{(Y$fyk|+@gTGA=0Z;Nvj#JJ;`P-Yl^7(dqM_nF`W4&x{f0%6yBq=Nte z!SAk9rp|{DvX2jX(oIc`ZLZ;`!ugEI*cpw=#n~}9oKd(7lk%gE()NX2S(q@$ZaGxWj=Bn zdGWq{XkyPCrX- z1}sMr2ka6^srU&57;yllQ(KQP@II*)XCA>Wriy>8d7ujhV0b|q;D1lF>}G$Nl5~|Q z#$AJP0&@5&a8yr5yz}PuoD|K^m&MV{zA8FpDq2256RDu8dDty5l zNbWVEn0wcpWi$b6{pol1GJN(TZ_QPdP#NTww=VM2~{EO&Q z3`RPz-kGolsG^05UaMv`Yx&07yum^ld|~SWJ5INEf~uwnSyQ_2#mWRQ!7i7EU)+-43)+YkcH9Qq8U zp4?+6>gAepll93a=VchWm~aLM%r1r)r&`df~427X>Q=hTI6c{E?*9*~?Cg)vsF;ASZyzw8#wCuo(_n?V0BzrBu z?|~8qyW){H5yKjXC+hfu6+J~bU^)a={`0PmDb=Ls7n2k<# zl|uFO!|bA~j_G*3Np;T0MgIMG+!P8XCJ#J|;O4u>-x!Yguc@TEan<~-Qn~DpwLF53 zL>OH$Hol1FXkVVhrG3?w;Lw_CZ)Pv1^a$#Yn67NrAS_lbR9t zrJjRLUXF?ST>i+YuICYR+dfJ_B64=UL&yED&gpMMUO%@ksSNtYD=oUofVk^Ww09tV znSC>B6hnOWYbCJUP{Eo(sFGbioKT(b<}_;&ZxltKIXJi*6fmtABR?mMsK=}D;5qEyv(OdtNoPbOPycknTic|=%fXN-Q5V>faXl2Dv8~SZ*)o)J4 zI!}5F8fI4a5g6)9#*6EXl##7w)~_d9Tc4-6+u1NbE1X$VI*#X$0i>gHW(EOq2%?r1 zrzBwB%zVo5JoFiuVhr(XXxq(i2ft{)HO@+)gI;w#7$jYJ(1KPJpyBL#${G_v$UbdI zl?cJ_L#G64C;ZrME?6&;jN4j{9J?(nz3i#K{$np0nmc$bp8p1=cV`8Cv^m&Pz30K4 zE~>9Ko~7*Za{)_6-g~Ql{TQt$rp?g=j(=lwwb}=N!x5vdPcr?Hn4@vx-6WOEQf$$N z>1c`gJOmJS2{@jG@1AGR5+@i9@g~=wIXa7J88pf*Hq#_wMmrS1aJI7v>H{B0_3Igh z*(F&q3kk$RH%(iex+?2%j7uq1&$Nmnl`+%12e`Ov_a(l#>O&UK;68c*%;~1J01Xh4 zh+l6Pg9(ebok&Pq*ZO5Ii#umkRcD@XOh9AnL+WhA?K7cfQE^_2D+W-zH9S0A^2#4J z98edqSQ)HU>9Tni7jnyt&gr{lpp3BQv#OT#b55PZ8UshYz$h(mc|U4R+^TvYlL#(4 z`H@V-HSSF{Yen>XKg0r+Tzzj5wIQr|rm2ev^h5%0BJpG^?bJARr}xp{804wRcYe<; zY(jPC4&G*)D_Jd7ZTh-(8v1QHrp{CaAt4w^q9}oQ_Lt(=X<^D-Qqp-e52n8czj_Re zjP^FIC7VvARyUt6FE5_D2uaH;)YVeR>Zjxh&)b_PIhoB&Cpqt0(sZ_vQ655*Q;!-e zZKvv*WW^of-+Gk^rGg$41KjVzs@LS1xzN60V1?{>az@|1cJL&1W@u94Ns%c#J3C*9 zE0wC?{ro+fbK!4I3KOjtwyAd}x`Bv|t{H=c?5|tb(;>?~BF& zNI+qJUtn6DS!0(WMP&&he6cmvRmnt`GIjYAY@FY4n7k{{okgWbI7XNk=T1z0MeUR`r@%E{hb@{ z&mUBbC#MWcrGs+#iRnY%>>CAK2qMk-$ijY-#BFm_`L5b8I^`CD$*j|X`lUv~sEEYz zV*p6~>q+$o8che4GMss&3YCRO$Jtx8RQZtM7Z(;FK>bBXh{u#zQ0-Y;#Y#iH?DfyY zGrNo!q27NmlpY>5zmT#*4XEg-{(xa$nOob47n_;R8*LHE6(*-u%#PFB=j#*G2^eO| z*b?QEVpM_w)Rx&##3OEG7uLKjw}!4RtFcKSi^dE^k)7=P3Nm0!l_3u)!^|Wz)O6WL zwd35smIdTZ67o~&|G1iSLS)C**Ok#tthQm@3Y}HVlP7ecAe(N?G^y2~rlwwvY$pvw z`(%E@#tKKrz{vW{Fv9eXTA!2vDdNVSnBKcX2&#yMX4PpSB|d|Vow zY(?fPkM&9e*dJj{OlBnURY^;}snSZ5|6HEM<+Pr015i@YYeI~5{~edAKBUqtL#Em932VaNG_=(!HC5YS@xm5{i25-^%hK!G^~L;k#aJq8&lz7)t&SNFYh{X( zpppPO;0$YC{;2C}p!10)%CHrS=;74}g`}c4l%Kk}YWRgp97Z$)fyuoZBN2(UZ(P5U z*Kp<$Dulunoj2)qbRr)y$RR4EsH3AKjSSE^!rc7Y2RZNj(0Tc5UK(G9NL{hl0tpHb zDH``?39PWCZZvxCfV24r>srT2-o+)(n5Q}GeGapTG{)ls zA_sdF+h~ziLoibk(T?#lkxoyY)FPZivpA@KIotfblINpAk1TWEOVLFH5FO+?sxuCC z(d&DhAP9n}gPjV%%PDjE4jPeqSiNFoJ} z1(-f@Dn7xg;BP%by23rgNUF|40@Qwk`q17?AREcjFA2wNC0LHb;jdEDv~+Nhah*xT za&|oOHJiPDB4H!@yo>Q1onkIX$6ZmV?}I$;bBHZ;oBms(l1kzTT55{KZ5X`qXX_9% zh;jUcRv|4U$$7)HKFICj_(IeSxfNUok<-hF*xmwB5qp8cNJ+&+iTGE1g4n~Mlg#!` zG82W&%~hf<2q5uEV)6kJupN&_Ep2pYIrZWL zC28^@?JGF6c|XGMe$^B+v~_)#ktl}*zHbpNsaKXSq)9T0aX{F5dRUm9p`i2jRn$yS zv3|MrVXrKwW38yBqv0YR7jJRvT^GlKKhgb85LDQoAl)AAw%BQ=S)--Anv5yUvIw^9 z;qF*gn?^C@o;H58m5ak&-al2fHHH6?H+EWhgqW@PJk3{+#2KKx(*%XvM z#>TXsl;qO^GaU&tmlreFGzpwg?`-*+*Nvt$iMT8N=b9~PG`{1MBUraD7jcHGwBF48 zxUS08XBTmV#5dYuhZ`sBlja6oM8O-C2~qXtm0J01pdC*WkrOEA)AQ_Ou|JedL0Q0v zG9vlR&rHI2h&ufnW^X?-XIKA_WTl#`y_Z~t2#YQ< zQIgFHGVRr~CzdNS(VT4b;P*4YDpClFIfo}$PD>C&6oyjWAq~qXOFOm8fuMIW|8C)6 zDOLRR*2Jo61V;lsluA&iB^1D_uOzjs!fOQ{1}ZS8{O0ij8pNFJ^ptS5XO;+YplcKT zEkCc;Sn~dAH53>FZ1PhE(8}tpE?Hz#{c|*`n2%!nH6e*J_myFeX3JM&45+Q^3K#25 ztmQcRVNCQid(pc|?_FXGI)aM#&ed?NWwW~kp?EoS8^8PB960sx_PxMLIQpBO%Ur#h ziE>F6nY~@|&+n0mVs7?`{BC{3XGL+70|&0-2m}K|e~vNuHL!SQ^>o)c~l{S ztg@S$(X#4KrLr7Kn2U@yC#yq5+V?AmxHA*OsN%?#cib20OB`L3{M~G=zm;Htf4?c` z5ID-1Wnq{yqL*&I`tPq1k$}?C2dKYPNOGqto=z&-o8fllIN)?1vM@}AXy~Lh{+@bT7oq^vMg|4fI4^3hzXqNj|?nD`HPeyJ0X(N zO)BLwhtACX!cYS-68nm{b2H(oM_aD}qm=j>dWZ=pc+Z{<8hCD}mK{;5MB!X!xhxXC z7zfO6T96@!UIPUhG52Iam7*fnnB%H0Bs&N|0Xt9B@i~eZXpvEVLPaCAMcK-N-^m2F ze4<-im@Ac#*DvvPbHj*3st=H|yBiI74Mb}O9oZ{H#RX!ErZj|mOL0{kL#ANs?hM1+ zPWfCcN^}5)BvzW7j|Ju?OgIT3Z|Koa85ceXaJ7s(R+~X88HM2NO~RUx$|6J^Y<_}` zxkn}_mS}zT-9D05vX0atkGfCNPU^-sP^FE%lOkb@z-n8NKTQ27e@=H^@bs13UY^5 zrw{4qAcC%hl#MSipj_tSlkyAfGd#TS)VAVzeN}c}`pWYqg$M0-lw_OhGvbhn4TKe} zWiHa}BsN3_vjF6r+=q=q3I)~m0{R&Vs>KJ<5z;rI6dY&?qeOW(5wnSL(A=`7T8^$` z8SSVv<4t0ggy-N>Bn;YwY7%D=JdrKn+r|gI@o!z7?Pxa6OgR! zZ=9sGNJ#Z7F8NLQ}J=*LIAcDytwR;|6<&aT&-%D;K zKQh;vV2YJ0K{28Q0oI_ppeP?}ckM3CZV}j{?6q@w4_YGh{M0KKv%p$$ZVyor>GYp4 z{YKfUYmWvSPXuSl*G{OrGN-PbBKQ{PuJLl#Mn&?1-B%B}7ew7R8mjDoj-YzTM)`r~WeuDu6^bj1= zDrHY%E*+zJQqEbYq-Kcf_K~%taWzQe*R35rCZCelxH>M-AWgI!RR`NELv? ze-TP4oh!@QAc%sNl#QW4~m!na7i4C=-z{I3?k@XoY-Ku<|$ zmov*_=kPlTL-sgj7>Rg(AgbVvI1>Hw(eGftz}%ZZ*0NJxoz45}1_dF|I>r{UCNOB_ zK6-7pAwj~HHtGBao<<$c9%w_5CLLIur0{P`5KmvbcT9QOHkp1&CV&D}ia6z0Zj!MI zHQ$@wDkE-7(_mMbxC{4FA1zqmxHw6 zQ+nYtT4_qTTb|E^rJ~3`j)S9955a-{z27DPLa<+N2bY9hESyuOW=T?JU2`q3(Wew& zh>UcX-v%`&Cf^ZvoimDH0!Dz-gb{u^(T_wA_@{g&>IaX?@7d{&aSRGzSY@oIN^awZ zS3mnDNlY8l)+Di70)GVL^^cSbi->Xnw5+28iBtT>h(qvR!T&bg{)zc|-OK(oJZOk@ zpG1$T^u1I%TWJ7MO%|cYPOCXZ0{HDpp?yzPd6Z3bt=3EyhSZ;f3@P$ zIC_g1G8EcF03zaHh0I_HSD4TQKUWZx1%rreCr5~B){{yPTsbPqMn%Yg^+HgNCC&Kj zJS+dsc(kdht-YlrI+DDaYehulkVW6j$PysyMZ)I12+-daQ*>X#=3^? zlT9kJWue#0J?{OSsaU+wIcz-4;EXW2NW9u3c;QZqt((0JE;Y@x3KYP#O>lTb!cj_} z1qzM2d?y+$XikJYXFupjdnW(LITnBU+CTxvR7*5W5lx|~hIf(8AdyP>%@0Lr+}m_Z zWEWdA{?#L?$!^_$=nCiWl0gS6NjiaU)Nv&&f!iLlUVV>uhGX`2X@P}PAYB0IgVt|j z#bFei$TYtWv-Y|7)ygG)-31G;$0El{HUsa=A_&Lf9olABN~w|u z9TKP4xn%`MIW_A|J#|89Z5BR?gnD@8U*Fuqz7~#R8ommj8(;}9|H2a9F@HkdWsCuO z93~e?(lL}2>DMDsVZpKEshM%UX|Xk5FwY^_6HjQ#3&wC z4=R-%?W2v@Vk*>1P+uQF9)NH)fdy>(o?{vuj=j2 zkl$7WQG^_Ar~r{PoE>k58;j2I2mzdsw!X+JTr<$2Amb$?EDD`RdrZcR~yhHO5M89hx8 zvR~?323$u2+cD~0U*EaPs2=R|k_esJXfaa!!Rvw%@H{>_?Sq|K9UAqySCUgH4gMY3 zEyr%p=LCEDU_|}M_x!=x?aM@wT@QvB^{`XeJ?MXOi*PIa!SiU76`AO@!`zq>~T!2nPg2alL)ozpjAXjYk+?%U`TR* zx(h}Wjws~vdH{|tn`iF5xN<|S105A4QGw7_&3a>O%Y#)gE7^9t+f|+YyqJKuC8z6> zR+DKb|GwNb1Te{y1chtaMBjmNvNyXYNO43AZ|2ipD_HvWkw^G+yIB}gn58hvk)#N5oB*VB+eGdC8y=I<)MvmhS@osRPr zftls9Hawnw%n-!IH`=wB?>Jr2_PJI|00%V55g2(Louvm2AblLZigJOIQ5VueY9h#BDU0*=p*w|`h8@WWYhjqFgz9kuS537Q!BM4QO< z2XGaToQ?;K7HdppLH3ndpn!sqJET_HGXL-oEcgrB8%haHA=zcuV9cibL8e(tl5t3U z5kuieD(I|aG6$cRR9Lhpjs~$Or#h;^6AbC`P&-1JLwR^J8%fc3<3%(L75_(tD~hS2 zc;$k3(j2>2ePyu@CPqeS+KCYXu;T>i8y}2m@{df-Gwu-WyvoE+m7=ki**0jTg@rEy z<3xvt6by3qp@I1x)BQnP*tFfImsBiCidvEne$$$!ou(LMDZnK$MuH9kkTJ8VWS3h| z0QA)Aq^sUH+VLDAJ-g$Gppij;N=Fu$BN?7 zsGw`z;1RX~J-0nI@Bj(a6-(8&tPC{Klk2Ki<*VPjb(3t+x>cDFNe3KE1o?7^ssW+T zV4o`TXIU>2v1TVCEOT66gx(m`K3r5~fK`vOJ7%sjaEto>d&5$lnv5w|r`J z%`*H06p+{BUgq-6=&(tDmLFr@SOU!*;O8XuGnJqz>Eu4NFqXp^-G~OxaxcZ!aRfY6 zgL)eZkX%GeH=PQrsXILQf~alJC^a2ENm~Nw5$QV-srYmwTm0a`ONd=oNns~V%wc<= z5XF^p2O$~?;5vrka2SvBb$?LedtAu&dG(ex)0rzD87Byni-E$Gb!Yj!h`)hT zhnGM6$nBZFKAF-7X#`2${m?MKH&qa05g(_~ z0)F6D=5Cqo=y6){>>MyhM@M zyuLi2YQG4N<*)bm8i6?>E|5BzwEabNLfAM>XaQ;ovo7VsmDffqm1NNN4Pw~a0>=bG zzMjmyvDv3A7jsrF6$ymcxXNnU67V>c;|IBRpEeyEWsP$+ZUA(|{CL~c$f1U$y=^I@ za7dHyYdgc4cSKSfJq!mM-K37m$0C@Ke5di2c(3c1M)AB$Ad}Y*lA3t6^p=xml@avj zC#5l^2gSm8E7qn3Px*8HDhkFC78C4Wl26*H!o2}$3Z)hRG{MzU)B5zBhpC@9`TdvE zmhoO_Q}x`hB6%C?!Ubw%6Q&8w`q>pLNwBO~G#F(Uh}F>;=ab7zUv%1DwZNLo%5GPh z53p0AgvCWDqx3T`7K6QA;h`}FNb8Sneke-^>m4Yc_XG?>^!Lg4 z_xN2s?843ZoT}~%>y8nMd`iG@JZWhq%#rG0@n_Y z9Vb$%-TN$9*k`n4i5QQ9n56jV9b4Nzsjx1kXY)(7$ch>^e%0@5D|iL$3JVxGCrC?~ zu@)#o$~J3qK>f_wm^@{bN(T7cE%Y@N>Jdxp9%gS(g|oYJ^gJRot>$7hf-Fr0$)JUb z7dX$waQe0QMbP&1Bx&(Lg~I+10MtM$zhj=G_Gyc8ZQE$l4;EB^J^k`ekDwSL%W^o( z_mE_lfE49N_Rm-8GBYr(ET}pK&X1X9o;a&*Fj9FDHK{h6!Q5rDEftZ_5Ms4j+gVTn zfY)hi4u6GIf1P#2X~lyp_X=D`Hie_biHnRkKULnJZW$E;axAEWpMXP@mZDON6g|;p z9N@$5q|#wkwECI|Vad}lwLw}^O!~p!o^S78(S2*ydkqVxISQw|@D}&I?|yOZTmQJ! zPH+vc{_Ll37;pOJlfQJN4zGFYsh|Bx`^IAz+JuJJvmG}FKK|{Go@$u;gL{8EojCEq zQ;&TfQ!UT6*#Oa93xYbg1TKe>h-8G*X!-dwo_1XDL#|+!!_h;KTzlPaR@2d$QM^k8 zg(Hj*2Q1jOtEsL&89NUtA>0K}1b(T!Kl+L^Y_U#K(hAw9c{9b zL@^z=f9qZVz;NV(J9MYs=k0rAZAzV@xI~oH4|lC=9j^IV!7Uwbu0xx$TKlwMq+iBC4dpL9u&*By z2~c^46lye~1OYHAck^-K7RQR)Is_YN(T|;sphNC$vY{4T`66}6w6;J|Yf|Z)BVSzU znh=s_ESaUb;lB`spdp-Cd98y+Q$ zPR)eH@%=ysUg|b%TtMKGuIcBLSySqm~lI zozG0lxG+#puX}Vv6col3DpSQgBz|D9Zit}h{99-4`mBWxv}kP=qs}g2I5K~=#&ZaY zXXht9TSPk1o`CCLkYaEUH$64}SS9uN;|&i#H1RYi|LyNC&B386pGRm7rk)~^U?cO} zO*7j6d+dv7^Pv>vFzo2-C@AqPkC58-oE%E(6bPU0RJl{4j<0?22fzGKIRBp8DpY`A z7>Xc(VHk=qC*FI{Gv_z=-L_)Jq$!K8`}KXRHT&M*6Op<(&Urq6d%xzg2mg5UvY9i^ zz3w-Ey4sp+umPjz7R(`s5eUMV=l|?^G2$K;C1FyoK9I^6EWNUR>oOz-39qH6~4I zmSklwSLF-@%Dghbh(u+Mx9*z))}Oh5M>lbS!aioizA&DrN>xYVJG_0mN}ywWqzNQR zSu_Jx4{}-hu~+}lx8sw+iLY=ny!*oesc_*qHWu>j`1G6Hs};5q(qPpVjh9#^r{mh9 zaW!xJRb+>okv4w@69SDF@nR@)1s;uXv>b4Vl$Noxb$y`Kn zGLx;)AYZ%^V^R_T4FK$PjXE*K%Bk>bI4(YLRBzvsH?Zgk|eP-rZ#BH z=Ee_h)SR3=?-9oG+0m+f633#X;;a!k8}7CYzrtj1`s^+5SL^c6yL71R6sb}dp3U)+ z(V=QQ7G3-7u`7N$Ij7wK;92*yZOM1C^g~wuY;*RtKeE6NN+%SPd_S(jYzpJDg3$Z* z$z*SdM-`3<$>U!JHvAda+l@1?LhwPZudgsWB!Q=c9+hpNi+$|NZw+mI&34}PZThGp zI9Igp##aGkDKWQc_h%=MB5Qcmq zZR7dhRdF*< zvd-Ga60iRu@S_*b42J-MVIx0u#MFcojv3zC(709-(o>qg`s`$r4Gv>CpW;hu4is70 z+8cI!D=O$3FluAlPnus}gys(w`?GSgnNk_D)`%5(P7(z)Tf|hiJ8}C<6LWTNB%sd$ zEnz9tvF->bgBk$PXh(L?zlqwGjl(Wr+rk?prX29L_380xOGs2(9CXJ`?E#rcf*FrHr?x4=$|M z2!PbC`NxDCetwpSj!g$PzGyw~#;m_;3dWiz&EeC@Tp#Ny>YUk}7rSoW$XErUjSun0-@P@@n3Aa_Ue1+y~jJb-LmSweS3G>W?2DsGp$kvf%*L zlJ-BYlzCBzc|u=qGOxG`03=2TKXG7G^g3fct_tBiEA>SJW|UCK{oTQxYB(sS6Rq6> z#fiY)&Blo{`?MQ3l@yyi@{i8h`(aBVg5d~GP_An4*Bj0Nv21^_Yga6s)(GM-?Qg33 zDmvwSbIHu?x@;Ibx$@-)m&{%|8qKC5k6iOW-R=JvguQ+OC()Gf%H7QXKzptt4q3T` zVN8wW#r=e7r%LH?_Ax7)k(sh$bSSak20Jx<;wQrk-zEa`WKq0zS4W zYPJyuGo5UvO!IDNQ&>|Wu1QJ)Ke&>2dwIQ*n>)q_aJtm3$fo$9OJs*?pZJlDk(>WE zVZ-N*;gp)=JKTozZq%%KD3)`mPB$#-mBIPv$f9WHpHo?ib(5yI4ql-L_ngc(5U~TF zI4<~M|L(tcjn67Xt^arNr<PrQ+k_; zPR9i*#OtiN)t$qNKBrw2#q-LL+AYnNc(wUm;=X!mV>mknofw{h`+zG5+H>2n!TT%)tiUV4^I49Y3$TUolcC zq!SW9wD}`8)FN{1P(P*LNH*8(x$jo*_VxXX(iJNF8>p4exmPjqK;PUH3{hvBA|g!# zr5-L{q%hb~!JjO+yZ?0|RZ75m?MbP}F0h>UJwl(?Yy=x57^-PT1!xOZ%H6 zm3s!g;V_d5w`-$EUQt&}!K_Nms>Cd#$!|dWsBH*>0@DA9$;h4$4Kpsp6lwq{T)vuW ztm3I?pY9;i6e_0mCT;us7$?pOAKt-40wYJNOK(42N`(tG0RV!L!}dN(9Y!cn0N`W3 zOw}tGqb(Z0_rdRvMzgu%PG|Fx!I|+&M+J_SXROlBSf!mXZtwK-u{oC;^2SJx2}Tsd z*s!nWt*1Mm-#WS-Z;&-ADDQPifN863yc3^#shPIp6E83sout2ZFq2a#h9EKzD2^&R zxfRB<^b*+_#cMfu}}# z22fHf1&s<+Dczj&P93~#=U$&Uwx!=U-})a9u0V0SwuK|+t+boQ8W4*Hv1kCp&dA4V z3d)S*r}h5)qN&=Ti`lit4>=4k`WyhD#(Cdaj^Fdxvm5TazQn!e!L|J2N6xjNa`MpH zJ#5*Hj=;Wc!IP&NCd{3d*XarBN#D96@3zN(`2_sr((!Qk^@rbdE&Y1|+1+tmvFy58 zFFgGF`<=hMVLEaA{ii>QLM}f$@Np}93+4oqLl%U?;mXRkAW#Cy<#!GLJNW&qAP96i zEeS%RdOH|`5zTl_vR!F43fT>sl6 z;YnE%*??b6B=}%+^ReB6pmKZM38C9@lu9Ns856voR3a{>k{Qh9{f5ZPm7k^ofRA{E zRDv+ta9vwniM(>sHtuwGa3EaZ@mLH2NG6kP-7!g${3ogIqY)4V)Z^_)S@O1G??*54 zjBn3l3*)opv)CuWCOMOieu?%958Wg&8&hPTS_bg$u*Sz&i zu({E&@RIhQg27;vj7MyI9+7>|_z5Q5J ziO8}Xj)YOUZ|(&}QCK<+01U%KBH>QQQJyCPCv}HPU@VJ-Bb_KFFf&l;j+*+i91I3Q z)$zi~B%ml3rUY4*dudVEHqkf4B6d3;_{ z#CkMDfdIoWjK}M#KOT`KnR1@T@8@Ijm?R@mHZ#8N#roY}YAX7+$xal7CYOs*B{Mp3 z2n)H;evVl*ody8hoa(H|9Z#%_imte}$~T+V3j5`j#xPb5q8pcJN7akl5Ij@YQW~VG zI)R(0)S*v;yKx91U??ZNM-4F2u*orMfSL#l0jBFC;+fo8gW|~QR>CL z7*`jTIb9QbMcoEpWy>y1t>Z#&z;KM0@w}>Y#$SkJ@%H1(Q(-16QKkp z5$m378&X}zQYKM|9{do^SGC!qRy7wyj^n)EE_FF$u^2%m8MpiJ<4R1OWe2)*DrqSp zPI2uhQV?SCSX~Mvo?#egn(eGA4|R5wv6o^oQ4|5dz~|T5KD1_hqyGx}<~tv<0O%~EYkf!IJZ zs!-A?6hQLs{mAc`WHqV7k+vF_8c~RCJ|^ok9i*`yR?CacjZF$erby{ps*v->wnZg| zNjJB-Lt;Y&QR(SU{k(Iq6OY9_-ptv5nqdOL0F_PMc%BaigF@HY-+~!vG#Yg^^}#Dn ze%dJ53{{OOaLsnr>fZg}15sNX9ywW!$6~cV`)tYm71W5=f&cGyz5QyWrTcuX(X58%l;8EvQKkdQaMbSBqX;Fl8$G*bI z1WvRKav~xN3TeJ&lHHbuQBNS$Z(qRPvcU4M>>E|pz z-5R}taDhjRu>^ott4;5IOIteKc9M7=r3DO6IP*I#cF1?K2R7x|9fm>rv#bXC`V@SSG)u-E4XAWazf)>0qVDTdWM>?8oFl?jtcXSJj`MVIfYd+3WiK8}AuJMLW2 z%P&%of7dw{k@&f>wGtD}c?SGMhUR#8FBl2*GY|6HBw6izzsZS<~Hpdt#{O&vycVlvl{d{SK|F@fcD6*uG-P&ln6{vs$dc z^CkB!&A16cr*J&3(P}{wjm6`llNWRKJDqdR7qBFaVRpM+;!<8&Fqur4%Fvqb3?PWZ zo=Hnh5;3dQtTGeU3%+&qx!+nyq~BIzocD7QC2=}adfGHidId!466UGRS}CE{>-Bbf z=J^qo3&U}T!%EJb9_HA*;yQ!$x#PH>*15tHR+tA;eA$1@)^p7L_`$xGvD}XG- z{eWTu&R{BD;ABdW?Xo`OeC6Dt(1A$#e48`B%bZgJ<1sXGYGa8>uJKQ6JYdyXZ~zDg z1Of-8IxR9OYf4O%95fJU!LqBmWO&0NFF=Mm3)iT^5A8kwyX8%;acO@#1u0A+Q#Dlz zOJNDg>FtgaoQ5Ptqd|!vqRD7fDUm4YMgYmn%TsDl0Einb%#M#LzJG68b2tsKX)`_N z(n5#beC=x8-~R6U)%AtlDNbh?!UnV6(mEz?zUwd1_k125D^JX$O@u&@VP9LMVlUAt zDN>>%j-|iu642ryk|guY00~sY&{!AWtb33ujaI8A1?%>|DwX48 z$%rPKAPng76LK`l(P>dDD3r>~aL`%M?8q-OpJL}d&uyE=hic_%F;`1ch_qHM7;1&; zD;PZC7Lh=Wy!19RdK~LKc(n#I8nwn~wKbz?y9{xfCI}{@ueRFigBwCS{uL~lt_1)- z5lYq{SK3P_%rBBRvtRF!Ifg4Oa$3x7qmumXdE(ASpfa~dSX109@{_^qyOI_B25w%?ld4$kZTyXm_2 z=b7i+_UNm(o%$ZxkE7a2tN-}Q>OY?R|2603QX+$ZY%Q2W4kH~3TW`%e3g{)4Wx;(1 zRGKMDeOoPmmX`nf33vH-vIJle&;fMJFCi$V=*31{TRl$Wi(v;O_(MUE}I=8o+j$_;3*tHcV&$qC6(w-Ne0Oe=4GeD2lvoUWkne~%uUId;l)`z!!h zckC@=QyRfAfuZDx2Rbdl5~5)kS7-S4^2zN%p$|}T{hUdKXWJE>51hTK@^U$T-VLQ;Vm!VhWy^|D8!U%r6sT5 z9}=%WCbMZu+gNbR%}Cd+R@)vAmRw~nnxC0y+xdW3KX|%B7;MnT%~k2N`1s=uD#3T% zL3n*T?o14&8+Lt;g3s+}O{j(^O{0 z){LG1L%k%)Yu_RW#tgFakC&Ay?wt22-Eu_yV77hE z@uigU_~$SFFe!Q%Wwqx%%IabQ_~uc@_w{Z5PQtQa#&G(meHvpe zMP%u8mFM^%G(rAE)kSv~0zk6P=U($MX)U(Qz0R{4FndK=>WC9b0L*TCYKa*y$-;=b z^R@8UV4z!Hix^+A>bRIp=_XGc$>v>_Wl@l4Uax8hf0QQ{PukU_ySO|Sv@{>;L&Z;7 zlq8)xRWYsD^duj37v1`=*5kq5q43{#oc*boTvVS!4msqIWvpeAt=?_73Al!8bH!ZS zti=GJFPk#%x}Qk`FEaFK|8PW>p?AA}{CmC_^qR-1hJ&`{w;@E^B6j2q)zRoFtm0Dc z`m$^8Z1czq$2rcqK;)RyCBifoj8MhucAXvCp(>e4Res)&3pKVeDwC5>M!LpYY2)lZ zPYG!X%8V1HCmO2;@*(P21XfNWtY@s8ndRp*&;k|eP`3Vwf!Zx=AZgQZ)=>2_D#5_ufAd1eF*{NQOs?UeL5uS(RjXl(ZD$ z+K%YUHfP!KS05kv{gt)?QfD3&qL4*FW)qq3Z^=h+qH*otvY3?oajJrH+H+#S#+eCh zc}?lalEQX-#SgYS>ixP@!@)Bp7J&+h>3CnSBGsr$k3WC=>54)gI!0f-!T<+*tXp% z=o(BH2-lbdEzLZMxu4RVE1{j0o2o=JvDt| zD$CwE%>{|dEo`_20E(w;KYFaO^2;cE1E4CJEDM}}4Ga-rqI>>P5)A&1P^e$0&6{`GQC*(C(Sk&{&4YDW=9 z4t^Mqt90$o%`nA~-Wrjn5UhtP9;KJw;@|Pe${`J%9f!~=h6%odMQrh8)Ec-D} zEeizzfsn*;S(4ytBf9OKcseeT`kt4HNJ&2Q!ok$;ZfE9Y90jD$94a&e;ixnS3ZAb~ zpXY4z`qMf(W8=F2G;Mf8NX7Y>XW%a>ivm^kg?9X6(o#AM{kG|o zH>+NLh-$0?1R;#h@R!f^<|-;8v{a9X`H%LPmQ229m7XFVcxVT~ptfeQ~sL&UK9YXvV=^!E=Umy70*1 zsC4rE$#e2s#|uQ!f2RV)aN1LyN-*{H1M=bwQ|EXWg?boOKoArml)>I%U(X!$ag7F8Lr(=UKH z3y@2|<&Z-TqXwACS@YDl+y7jz{pK{002ex4$w;1xoaR@BtwiPQc?I~+n!KiFen>xA zSFyO=xfvb^shDNKYQkVY{b>qIUe=_^;NrqN^#1VphPAJZsD9Y6_N7rldV6#iq$muZ zSXD5CE1R!t{X4iJG?07JP5TG;B~!I~x}%Y;uHlw0RFY(-u;;JVL|O!%>DE6>?dZrl zr4RZOpEdICLDrx;7sB&H=pdpP?l|ZC{`sJVE5AjDyqH3X5=t=@86an$$WS6fDQsg^ zrE~gvt;RX#$lre}GJROX;@a?9!-p>yU3f$5e1^g{wwFRU@JYg$k2@zS&Y;ExLmU8z zsTiTR0f4;}A3KZm*7jNcAAaPY@7QmilBMcO;e|KEP8=Ljc7~4!_?S-?g+%q&$=W^M zPo4vS2Dd!BG|-uS)>A7cqY_4{QtlJc$}BsKNK0XC{|2i1>mhp2cRy^)+R!akLNW>f zvoE*utdxul6VW!NHYo-*h0UcUk(cMMHf-3Gm^DRh&|@HSs{F~S;u!#77&ldAbuz&| z?+P@{x~Ta2`vCw~8*r_q)}!i91CMUXqS}X5&T)K~&D-TmjHUuI)XenhHDY?597|(- zmb8|T45>@zW{Ov(r(8#=hAjK*F@;)8#pFKD7k<4rbsFlH|%`i3LTKc>dHuOg*6}e*6SY)6&$bx&hn;YvmPbN9C3Xn-XD^03j6S`B$f1 zZQaIHpqO&Lx%p_XGrTq9rIm%-4~QzFqjwEpU7?(=-9Gw4GxgwST|+Jg&T8Zma5?0V z!>EB~@$E&|-Pii(xZnuv{k60R!%p#26l#P}bejT2mgPm?&0OUtbD}I{S+IsP{de+6a)dTIrchy| z0sv4_!AIS6(}DhcoiUH`)m8ub%7$0&MgR;m`%}BHwM1s`(})tsdbfQ%>9*fF=B=E5 z&tF>?mdb3BY3^0Z$x=w_tX-Cbi0QcJ-CuG3{%mYS06_?4;ER`gf62=)HGu^5KT`Dv8vp%^x)**NJ@5$tXvWO2oO2ZbsMKUZd4*O3 z?|dQx!1QHUQO2 zrWz0NF&_Y+IHERSFv?L_p#*@U0#d0!Ga`Q?I{`6r*YBD?IXVP$lTe)l02p>K=YDiL%x)jhV5Tgb z(MXJDT)WM?C8Yx;Y^8<SM`3KTX#|G}ZL)V(SIMKZEok9O-!sK-S@3BEF=X84Xqe9I$u_r1sYfDoN z)rp46Z^E||#)3h3WncOs3%vKMb!T5sWBm2H@e}dQ%Ic591Pm6gY~Nt~})_xOUQt&Q?W~kj9t;2vu1zHq<-kPz;`Wl^;<( z&@}U+_F{s06Vz%wKGJj~RkK~}z84`V#wS9F%B=v<{N}yXi5?c=$~+r8_!-DDX)eG> zHHIqy04rD^d3KA(-W+hZLwl?MGosKso1#AFx@6tnK~p^xN@kqpD-UqUA%`4t=mz0% zxU#Y>ryzmk^1HI+Ya(>`nLg}F!yOv)oiloWzTeLZfZ^Mq)0d-rjjyEbCHgHraX&)tPTV^JW;Q=V{3-Maq-&yn0*}EegdMAf%5x(tTxlv zNo{FPZEi9(#;%Pk)Ndk3H=ca7P|hmWyDo7JQ0&QcN=iT=7!d`*AMnd%=ZZBuy*>{` ziD;B@HJxs4ehs=ho-VZP#7i8$1>X{^*>THefspxGEvmd?BNpzmXl3=qs3Sap08h09%fD)vapWrC!Ri)PrcjuYH>A}Fa)PP zao`!jklv<`Zr}U=6jy4fj?kFi7yo_gROIOJi_L~Q6}DhpQrV|_=3u7Fh~Hsyq$t~~ zZRzKBN#5dGUkCC~uxjStPUa#**A2snMkgXx664#d;g%_zXqx4aq-Zo66o&vn(=?26 zG#XJfkWwRk&;I>+0Wyq@_H;Ib_>!A?InD5ogqc^;X%hy3jd3Dz8u}_RO#uKb$q@bj za281!Kq&FrGezIKGpVXxQISo85_XY5P9#ub9sO~094C3U&vEFlOZK1bD%6K$z*bD ztYlX6Ii---qS2C=Wh0ve0G-u5@!{~vUdrd0%_@XPa^VrTKa*UOqsU2NK&$Iyf3)x4W(x346<5R z@!nsjayhj>sx2-zlIau{4OJyFxirC`U;pdyx(7+DDu`xJJpJ#E8}0{y%)za>zR+nUbX^M$4M}{-rK(&m7boLG#5}c?C<-c~B0qVbT6q*%Txx~S zP{eG12eb0tNHhWfx~``)_xb0r1ImTH&=| zq3-?BN0{%Pc=*q0Ns_WTaa#VD20M@ZVjwc&0RYnKCaq2YNCwsNrL;jA@k9a%>ZCoT z$uIP0Q}CG#l)a0+I+AJ6$nm3y?E(N-&h-3W46zo2@z+j%MdJ;8%grS~p&@FipPkg> zA^>Qq2-Y>)Q!%qulS!w?g{K}o1Q-&@q%O;9Hk&un$uZ^qXM(Lam3Z2x_aw|1Uu@NY zj>Y%RIS(dYU9%sn62Y2_xVCPX0QMh>R+WR#BP53M7!d_7P)H^L;Q4P4Z+K{P@`ZmJ z>N>?$7DIhcz7?kd1Ar!G@&&$-yr63Jcp`y}sV9!AssKPblQ}OvNZnu)$)u(zaxSME z^muQvssKfw{MF-{W->4sG?{pEw{~aK_^(FCMhk^PGNBIzl^Jywe*W|u`v;ByKt(n! zSCISwJA};%PVyEq7pBD+$r@=oU&xEeWGPQFl%^$<$-*S9luoCyq8J8a7)B%-K?S80 zF`uSaQr0-TFV-q*s(~>gGdd5M3;+lMrKnUQnF9bLo7GiyTqqgobhZFCN1@T)pU$K; z&0L%lMb~zmHFacuaqmFZ;{yP!DQYx`0T_R6S*vO77m61@RV5Sd{}nba88v3;cj`ta z>*!9$e#Dbih9L|_o&x|zuhERck%Rvd(#?C7%fQM}qUvxYbNr2ORSs<{p9oD^l%f()N)&!zXrNG&Noa4ReS?13&(ies>B0w0foU)VF~k z0RSO892=ZPgJDizSGBk4w|A-U1WJ(w&y=c zEfC4Puj$Lv^meq&!=F1X9ea{WcM|}aOh#9y7CF&06#zn^5T`O1t+EaXG)+q+XJwEe z(%AP>l3h8aNH~a+6pcl5(@9QMRb#Q3#+QIU8Y%93G3j0`yJiQeXA=nk&^0}i$uYnn z)*{$z3-M81*9AeGjm1(FCc$VSql$v4Sv5p)p|^i<{tpyRdQIE+0Q}wp0JlI3v_QX& zrgMqnnm2Ce>o2d|oO`+K>>(&54IR_+ai*-9c2@(ynH~A2>wP90na-pg_RC!3ioe-q zli2d<-uS{Vj=F2@HS6q0PwSef)>YHloDu8Q3R0ch#r@&Z??$r2&B>s8-k3hUFTvWC zvIY)kVpV$1RawmI)NQ8`rrH~)VqxcALn#T@ZYiBuNm2&*uXGY2;K4z+K!{E&w?Y2+Yt9m*9LpU$w6W z04QodLdfUydz@`Dhn#fF`KQ{G`(qB9opP1LtM0N|i3ZEc7R0gu0A(t?u~TnYMq5n) z0FfT8bq%j2(n>nwT)uR?8p9DGLz()0ZRMb|#;D};`QO=*14tkOF$}|}5En)OK>5sc znHd1mmP}=Ho?kTQ+c$Xu9|)-%0tg1cS4Y{)Ct8Rl@mMc3wNE%6b{}%P%(HA+vKR$2 zYqeO*1J0b9S6Fs7Y_HpFST_QbgkdJ4_wEaQ;yRuuk;w2amz%N_3s$HWHH*;2&C+6l zqB3TS1q|Kcn-GHc-W?bqtkw&Cc;?w-i*I(A%!S0Bqo&&Bd`0o-n+g)PDhuuMW`g@H zg#lAt5rSvAn9Jn~1WIPL$C}hc#pf&8+WExE6oC0UB%Cx{tYwzM`EX~s(<8QCSIKDE z6ir{yGUfNf*s%dpvei_Vu>HX&4w}Tmnes|&m2S}w4!X9C+00c-2`wy`52SRnxjf*? zsrk|1*~uNaSXQS-<^FwP$0l2&s#TH+m&1v8Cr#(wUdv67;M3cs@<0FpOm}=Cdh~L!Z^-_RL#bWqH#{ zb4}y90dh&+yX*nl<{fbcNY-55x&{FH0q)vFIf?+QJ}xmRN_9A`J|6|XyZT4ZIXKD| z2mpY@aRA_QIGjHBOK)V&zN(XFHFoF#$E~tir`M+)&;Q`~{!ebWKdUEt7!P4*#pAfp?X_~v#Lst2g`0+d*LTt zdooI#pYrO=k&)QZSlL%r0>Hq_Z?f)yfAi*Nx;7zmym7)7Kd9hNkb45hQ@$8JhuhK6xruHbAI zt1b>xHou%i8G>+{Ri{}Z?GB2<<8vbSC&ZpHrpkuO3E1ERfXta)swIx)LMoF_bg#X@ zGks^Cqe;zIJ$zJna^(p`z z^c^<4vjPCpXZvim72}Zf&A*=i} zI0{|J6td?kmTdTV`_~Hq+yX7o0$nL_)egtz+SxoQ_NP+s4SW>(Tz=CRA_rbCk%kr* z?U)+zbg7G0c$XisblxKERmI`@vO6F4F!WRa z?!BMIZoSu2c^;v7DNy;g_829x^p-*mXE$URPuHQF0{W@cU_A1g+(lsgxdI~T1yZ-ERAihDjdyr@2Z@7)a=GZH@fxXY7pt+@vPlBNhnk}GSb z&PKsKrq-r!zNf@gjrNPN{lhD3R?U7MGO>{(?BH3c=J;vLA2c5@tt*16iGQDeir>`X z=dMb2658&!lue8!O}#&jaY-Pp*==es+GH$1{|TL3#&UK5fXGV<-)^zFyo0YiO4mz{ z#kBw+9E+IYD9Z<+#hHBP#NZ=ut`-d%L6~>(Hw1(oqr!=(9v8VbkLRmP2@?$f6pb3% z=bVcpnE`3PQdWiZ0q|5KqGf{ZjZx5(FpoB;vClY@ln9URe6{?>`{pyrEVBDZB>{|g zQS_~Em4Ej8SKg&~3Trx0=dXED6=tepDz;eoAVf?Gqp{Vs9{Ssdqs<$A{)*ME7J@SY zKu2V13b4A>Ryuvuu|o0O&_HrUa3S<2!SIOjH!+?%EjORyA*lmT^Hs}%0F9a(%xZUw zHV-LXB=#)s>CI+r;YTX{tC}@xEGdvx5=nCHq|%##19PkV$Z!N(Mh67hoGQ67ZFQqb zs;b0I;sez-_o@2jl5^FBoP6v1!*GpBAsJACG&vYo1pt)a^}h^lB9f@FW~Qshw0Rm& zuWwN6I4$3Oi}_?%A$9C6CGG>LyHpE^B$s@msc`xnZ7;a5FAjX<*Wc#Ln=b|nC?DUS zaIe}x436S-4zYF4+`!Lml@CxAJ}RU)Z#1Wl^wv+xkC_a~81Q%Y@97AKpL^ib6PT%; zE9$R(5vsyA{_j~@HLSl004%MyNTE|&KEY`H01ea33_rc-AXuv6v-PoeUzI|q0-t&C z0-hd^#g6Py&kLW!8U4%0&l)RQOB&H-5o0_a363!+I(}&5z}p>V(Jb#TyW9>+8|&I< zb&H&_Iu7<%`T?5jS=P%N@w|Fuh+aK6gm-gI5oA9_oe;8!#==I|TBuk)`q~fD&i7nv z?j>ke&4k%{A6s5E-7N7V+n#KxX(Q+>D#l{-Gcbm{ZXnR~kQQw#BN$VmXScom)04kf z`*#Cuoam2%vN*B$%$+i_e{)5c^8c+1yJ`>yxCA7%;huDX4k z;|_Jlm?7s8P$s4c03_5!Aj<7oMkj+!W@e4kH1oLHjvP`DNsP6vTOS5&WGt2M9(Al) zWwBW;eMjkA)>K4^beg2>&It=p%j~u+t0t|}#18ADQl1#GS{4HUl&1>24|sm(pHh2Y z@m4m2vyWiMEH0bly1M{C$qNJ!-f|awxTFxmC=n(}DSjKC$ z)Bpg11zpS8U8_9f%m5h>^$Y+ItQj!Q4Rryu%>L&Dg*@fj*#0bS_fxi_Fw&)*5IiR9?*shp$JlXR z$N{%N3$#GLjYRU7&o6)|=G>Fh*J@DmIU6h7{i!y$^TS9KO545H`W3>^NnOayIRP9Z zNwXu7C2!eey%>F)D@mbKmgdb|S<}@T6zgxcwB1^mZvv?qXe^?;9K@|RPFVRQ%LU(g zMvRr*9_sDMlZ0`+&$j!3NI5Ia4eeN)Lm{?gQvHX2?7fnzY8YQ-Ezeh$RAH0iBfr62 zj7W-cOmpvssez z5=9dhhD_(Vt|0*6eU!eil3fr_Tz0;Uw4miUo^c}s_9~V`kH+E>*{M_DN|EWJ{ z`O4q04tJsRurzq?GGf_eo3`c*$<4n=A9H1OCk;i?va`beQt7K!_)iTUrUFY5&3|MS zXg;Zm`I7OHX2%ugRy*6RtXst{Z(|Ta+FCYP%;{bKm6}s6Jgh0!=If`4)|s*+7T;!D zaeuyh=M2vrv<7z!4(5~+ajmQPCx00H;nxP=Kk$2Ry>{mhzc%>8uMPg{??+Q(vn_N> zFubX5Emz)(RZ%OIYx zhzW$6%d;NKZ1Y0;U31XvB!%oYif;6x6&32Xxb&HJ`@QW#J$>+sG^5y=tn5<`{4B+J`I~fzST7 zXZ*!?#(~Xn_{!qfXeEoBD~xwUII7%HK;e+zi>f5khb+Q>E%cb`sD)V~h(!r?1hwob8{; zcJ4_H_Jwk;!>NsP^TZ^&YxJsROGP9N7iAk^9iHf+?a9u=^JtKER;tP2$ZLORsFHCt z%t0|v0zfWjkR+n13D1Q0S0O|g%2apJ#AquQrJi^;-*HOm3A>ZI%kWHgQWC2pM>A*6 zE^tDvsyS64Si#0e(wnBU^98#H83}Ekr%F2o&dK%NWX2S&L&oP2%+c}kgRDo zztZyAnK#v}!6uR9t(K;B(+wG_%8=DEj~z{M?lM!LmbSYuBk{!C70zBFsb#V{0I12J zni;X}(=e~p=(M8@FE~fziRlfYfHN%e} zB|}ondh@geHgfq(?xuJ2@L%PLX*09umRu%h!)01d4PS?XRZC zbn!Ad7Gp^~7B4bDI)_tPMB7V(pbbrxV)PMyk~I+aT%uQEJZ*{D?3#dGmFH_HXurc}&h zy!Bcx_VIRX7XY{gTA&5`NRt#BRrASdkJb&TP#8LSv7z$ApM_uE=gJs?cna&r3^tao z&DZ5qF9C@#Hdp6={lirm_~`5Z%6f~}H%Q*Gm zF9R5f?54Wz#&Y4>@qlZ|btZq6R+!IXF|KOYhOd~LZ_=fLA?B~JC)>WeT`bVFmHX8z z`GAj@YHwcR+DnyNyze&d?e_}L?p^faE_WhvNdqO+S=j%4*jY}dkH3>X{?3(jH9ja6 zvZTQpH?9U*zZymyHsB&C+>i>RAN*?GO&;!$fBa-No6~n4Qs3B7^2ube%Rc{@%6InY znhw(%ucViE{pkv3v5RZ>nl@Mb@EP@o&(6(c{Mn0Yxz`}pl@pClWWLHMyur+^!DQ#* z)VZUqW5)6VhAsv>X5~|{X38|xiPv;Edd6_sv57-(d~i*|29;zebK>1gg;0@ZSN`mF zH96BA|Gs|dz|q3F9!X6GBX9k4YTuLDj+Y&4KhISzRWo70^9fHQIg6!k7~dks;h(bv zKmgxC?X#DCwWbxJ z9G+Hp95f0u7FU;Ltm*SSY(O3h^?MuSt!M69!~ggv^UThHS&FW)j(_;2O`dxnMkZx` zP0i_2DtV@tl4UZzkFd=u?SotP}) zXdFLxe8aoP{BkMp7=BL9X(LY$r(O!?UJDt$l`vQ<@1dN7m&4sc#puQ7UctGmWR*Ym zXup`%R2F|Quw>i>+I?7Su0=~5ug)c#ojc;+`upxwO5L_C{?bd~|NP<5)6a~i(<%UH z*?6&<*cB7YT^;j)2?-s^} zuFAl*eA2t=i<8Xm591J|4Zl{H_; z@OmYbiAeR!@|MO5Ma6e+nM0R0UdMj@VaxAtIT4Jo!QdsG9!by~WlW9Zjm*jS!&@I$ zqJ#79sv_$I34Mq$aR>kghm|WFd#>|6LoOhq#Jg}U-#lJ~etwkA8!h#;%TC;PGx6p7 zr+%))^y~{7>y{(G8v#H(1-p*uyN>AEd3T<$(AInFtoJPffXv)c*>n!ye!UDfhWzpp zRAv3mN=U7cGgmgamRzz%FhQ~AmYZ{@-=6m?Fb2OHRxc(#eJgSFjD9Uffi;CRJ1>^= zMbx8aHDY)}o#{5BC>C7^H9bmMJcgXtg%oY`5saDf)P88oflSjQp#;NP>)P<;0`V}$ z73JL0zzF=0AdH=>@0q$gGSmBVk;=S+btgpufVrY^3Lqj%Q061ot}Nawv^1_&b4jdT zMrx5}2kS59bxL<^rR{#kRX)AOrx8M6=Ao`)s#HNFX{u^NFtIcT;$VB#-Y(- z=|;Jr7t&V^eWolPHJiwEADR564g_<(L~a^z{6UlVuQy-#5cWy2kR)Smy;&MQ2>^xx z$Iob!c<=I8fHe*C90Gv5mh=pa81jr0{OVSLB!FY*i@{Yoz-?}BHRG!kF--C{(&G6L z_?ndEY-_dGbB1WV?`pLv$A+@!_Kh2Ih-N9%tip*0`jw#}1u&*7HY!i{89{*}FaUmi z2Lk|Pt7CrK7CFm^Kx3{z-CMhr%MMe?L1gHlPi60y-_%Y<>jy$iUy$ev60g3`?mSd- z^B0*(x^emvLzLB7=sGM8cL9LiPo92H=sP5S$nq(b(m4()1L&cZI}UZb4xTLOJi!sp z->>{H%VObki>!ZGYyb1b=C8Q8doQohQ+r>vwyg(%fgnOxDG5y?t{@Dzepdd{vUe_+ zKzH46)liyDT5YZ583R7^nX<2cz4}l8r0&$ITsWdgV?F&p{Ua@(nqRY1;?@nRAgAM3 zrNib$>ptG@Hx&wnV1Yi4w6L-ZG#is(>}<;W=|_M3NZt4uF701y{vXnHM3AhnRvhb{ z&eVpluG&s6{^E<I0k=w`%ZvX07#~B_4y!xD@^@6 zGZ|7&N=5)GuSMge*&z@112}HI9v*uZ%d>-P3s4Lf~ z<;S1M)KpG$cf(fKa{T*$y5R?}UFwHcPAPNA2M(r}E}hB96GwY&ts523s%%IG)erEMI{;KgxMyiQ%3iIeykHK3ch9i|D@Vew8b7dxXry}oxNe{7061&dTNwd@0It* zKHt0MvOX2IOG%P$800KrafAqpo!zciQ~RDYx7@^4t|*+_j%d?|uuO_i3JLyxAVx@gICyJy+iJk!9wA*3>QFanjVzboy{MJ+qrj z&>ZWYv!-L*l^5!t=JLkQeIiYv>u0%$i4keEdwc>5n(B#Js%!0cH`M>S>+IU{3IIrq zNVOyIa^&%A^_lNmAPDD5+L4qJa@q>zip1P(h{byHPUx&;d09e&Z z{`dv!T{ljD)M}_w{_H`@>PqhZHNlzq+9g77haqGEpm`H}W}BE#8o#NPs<0(LwQlH@ z4{B{@V@2zP)!}v+4l6!z+`2H;-0D_M^Nqth| zCWqC>0D#?2dA-b@J*o2R$7Vj+nu&R`apBn8_T{&nhs;)C*eymw09PS+4bdEYtcjXZ z1bu-%9yBsCvaqsKCS+j}rW2ouo)NwFPd;_u-Q^$4XxKY9q96R?id4R#NU@Gx-j$m! z^}zae>RXQ$o`2kat;b?svKCk3@eS_NJ4&|O( zIXHTidlio3N`Eobg(TqSbp!yk)WM6}45wXhUpxVx9z3b1vzTQ*q7uxrVq@auJ~{PK zx!^-lV7I`dPpN0gnOU!&nE*FyIDW&pu9Wsz<>Zt}+B~8c!%;#9}(Vu1jpnp(v zHmS)BzN~u9_Z=+^of-$=MhZm~sm|dT$>k?8A##=%IiFxm(YL63Jq0InGLkVF-dSWvcbK3>X`Rp3mn1fby=R!+Z0CM=)ikGQCP* z0|026mP{rYYJ$b4WtDsa>$+Zu4Fm6xlM^TTx~^*~e*N*`Y7{46{v%?bF>+W+*s^Xt-oDiLs$;dODMl zz=RDW8ZF)xee=hiOYSt!l8ZD&Hw+^hjjEC)=kr2xa-4-Z8jXmuES^0q+bhY70y-o~ zibkVBacGsLYqu4ZV=h8<)!-Vi=}WI)!x%0J^~^l5rVh zNtVJV_X;K-!C4BqJgbQNPN>`WYg^VT=}d`}Q09WwZOqHtXsqYzDvJzojhr=fAhBC~cCp9m50Y*Acs-CBkP95bi~}eBQwLVlfgQQ*~VzC2@+_8S7SP1*@u-&WHf8xt-j9in$;h zra_Q3Eucoyae_MUl=kNG0h(Ny4)r~COi1Q(Z8ZSEmlLmk^BtG32>`M&F}C|z@A_5Y zNKg1OR$o z#yZpc`l|py^MtihO`6Y?pd>*xkdsf#RZ419bmom@0svH%*tIXcz6mP|F)o3-3&Oda zf>pw_cTcinaVj&Q`tX=7uqGOIlSK%|!1(Q-oNND#6##Tydu;d5Eu}S8&y01KlkOp6 zo~lX|^M09QRG}s%v0m90D!Tu)sZ)JQPU(TaY0SoR-0T=6?%1eMJ#^q zd<)bhC+5L`lvi69hKvx-mGIKYS!urWZ6#(+h&i6a9ldoslt(bJ7D{ z0FYHtq6P|qHg+V;D0?#TFhYb$JOu#pK|{_105s=pyg?<8VS$pdK?;IM*$MfJ9spe! z5mb%LWHLeyV*|GSEEO(SQ4Y!J5)`TB&Z+4YbWOw9Fmz2-)$pjGX&~hD29^+DKxVP; zgw)so1eL(W8tg5P}sj665*&G{-xn6spzO6kQ0MK>4 zkjq+l04)LC6~<2>_VW zb7CHhV(69`AdyU7q#soP2!%qN%9Kh}5fJ1Hf=$!%nam`cmMI93n66-grfKC`Y;1Ux zwG#jkl%&q{8R>Gax}zN(1ZS?d*_!Qk07xYBnyzJY*~s|{AruV&kV>ZjKsWT`9}K_w zRDU)zA)_o7V$JQA%H>mI&$EdH0GLzJREXy^ZL)W!TYY$}KiJdBi4z?bkA(BPkL$>s z2wO9>oF~+PVkR)sA_XuEaT>#N0f1Ye1zMnwHDPhID@p6r(Db8*OfIBzhqn-TC*V6!S z*Y6vl?g{%+l{l;+wZPTiW`@CGI@&n?mI?VtCcM@4%E)L<^#lM{$ZLZN0RS{b*<9m* zh~V@20YIP!9d?Hjp(j5-53-n4i;aR1Oz96zkMTS|bK@jIUWk5u_lwV>6B#XF0AC{N zZ9?A1D=foU?GDrVR|q1esf^1(Qg%DX0YbXlRXl#z#DYYGOeRx#pyc|YLI^>ONRo8e z?EnDwc0J!`m4}V~16UQUi|+vdii){hE}!p$sWCJyt7J%xz*)$K> zJC?Bs-4$;U(+MkJpin%fWIEJ4zSK}&JrzYtvJ63h$K#P=Cai0gNlgKYTL?7}2oMC} z_xopFf}M$TbqU#sDU!7$F2`LC$FqUvdM4oK$1~{wKrXM5q-f$zGt)7(A0V?mo+;*;+Hs&IIO<%p ze1K)M3_H#96wM=KQ1nbm^doWWYulTaw~n0aEmKt3vZ0G4CPELq4biyj%jSw(Mw2K) zRkT7Zn+Rlvn2OdR+FoK1hhA}ePHWKyIYUo2=(Wa;!z5@1p%?Un{^~!=$NOnv#NN72 zwWwKyF6Jd#ERf_V&+*<;vGdsXM;`=O)=Z6W^GpE~1h!dS^d*k`wMRJZ?+)3`%@-td zh>o+lT-lwU@B8dgJ2RZAY6Jj7Rk8yyE7OVya@nm;KBHBv{X}T=Sp$!9YybdW-P+PF z$I@Bq=%I>$FHso~GzthMMA^IqVNis`&=bHVP0}o@wbVx63wqBw*Q}>Y+&qoeh)<3= zf8~s)tg4hTWYJJ%Ml;0*y(H2CO;dB$Ac6w|GmH!X;P(iH5Gh9(I|BeDp^=0V4ZgZ; z&BOHh!fl7S0U05hrU`!=@ALowO&eAV=WVhHc~U$i7-;|iL&AWOo>Vo(NM*6hK{2H# z0q6Flnzbb5m1vXI>aQiNZUFegCynp^XYS_ZHinVQ0|7zP{;hNOjTB`VI*^btoMWSU z`BHR%Dqn50U6}Jvebc)2Uq?Rq7fS%(@AyC2Gjg;5CC>TU>gDY<%WGWoQB}`8H9}}0 z=98zJoc3JF5WLak{kfpNyo|SY$@)o(r~v?q7Ivs!E>GB4i6SKe>4*eOVwx5aNzsph z{5KS5XtXQ=05%BDP?4aZvy9b{DN^1Q%N$XtRnu4Hh1_)6~*H@thWiD9aL6f#Rh_mNm>K(qXryNRl_3q%`Sr zIys*od|w!T?wd9}>vGqZ3yIG@+}QbAHeFCKPiR>N0C_m!~VlJA^BQCdHw{lM%+`p-+sPe)Y)eHt>qSB1Hhm$wc#qUi?%+TO=P3WmK z%gV?J03=DU9Lu`g2BoYF44e!xsg!A1y*Wfttl7j_tp8{9>}r|HOeUJ9D7)Qm ztyGg1&atZ&mX_;lTpRA0D%nQesga6+I}-QVEcpmE**mjX z@+xn!fm&8sHJ)}^8qFU0=8Jk+Z#caMOyXc|(v{=%0)sI^G()x5@Z1PJ)v#Ou;1+0s z7U*M5YV=%K5oP=R-+S$!t;279=TE+RW5Ut(+ylO=9V%|ET{`b zmwgnLST<>mze`(MN)ogSdxzA{=#<+^v3SWB7(*n1$8*?Uic)D!QL$-qq6SMDPW6K} z&x{hZ&BuBdP5H8R1!G6-3#Y|fmmRZhL$F~{co3W8#XxBhp<>VKUTHccR~ zQENZn$hA)8ic|lYUcH>YfWFN@{e6X(UKo3DQXxM?((K}U9^teX-ek9|NUf<2dvClI z022H6tJ!>^_pp7XE7gC3tZlJ9P=DsQaZJ{{6soSE_<```vUTV|IFoP3>sI%1er{2k#_)@CzdxH@uVTEqa|Zy_BK+PSg3j%m&v~ zt#4{4a=CU6O_Roo0g@t26VFd8rVwej(08qOFRjNv`*r=A#eurY2__M#Xzen3*=MVZ z;8E5Ree0*#B897dH`Z&^~$V^adnVX40oC@I~2O06_7u`Vl7pynaAly!@P->aBU` z+lDOk{jhj+Z#B0#>}vpk=fKRTHmx${hR9>5u3K6!%66C7alrk+!!)CMeAN|I6CKEj z8cB#8TXRp{+piqIef4U=@16jLB>gBYb{OpXTfV+5bNnr=YXG>=^ZHQxUpsu&i`q6$ z>FQbXprK1%_wu5f-KTKWB5UO)05Dl{o-$q|N~-r1^4g`0Y15|ircD<#&H*uW%6a{l zvW395A0^j*fuC{`aA@b5W+ByHp9H;?uonMhvk+XVw4n*&URu+noVTsDR zR<^#M?y1QNpKEI!f5q$1C-`@OTsbhT7Q7_qwYptv@3)y1*q)|sp56iVspoT_zT4E& zFlPtHwq*0zYu^t)_)h>}a+0ebahftlS19UVPq#U|;wcmvEw*WHnA>eiu6otQDUH_L zO$tHgd4MmitJ97g1|fs7PVtKYK#L84rcw&r#onff(Mce?Wyu{s|Gu=+ zd5kPuh1{(dzqDlo+p>WL05JIYzU&*v`^|-B2#Un{yzv3g1Yc2=7F%UWUWNE!olB|i z;hBomG^}MC))qbX6K9leA>tH7Ccb9$T#x@UNfm#-)lnHRrX;>A z^G}no@>Y%~G9lh|GXM++^}ciXo=>{R{`;TZtN$P(#j2Nl4gf|A+U{QA_AfC%=o41n z%U799r+=(F)4`QD`S1Wa?Z8OdSpOiL4&%B;ZN;P#v1i`CWMTzG5JaF609LdKB#F;| zIZayIx+*9mr%rM~AkOxat&57?K{PVmH(~B-;R)M}tYZ}GTlTyEN_?M!L%)Fw^6S>| zZ~#Cl;M5QfEE{L{FW;Yk^p`d;xzJh?05}HUb%U_yC_4_ojhnwpHVH7b+=ot5HGkSr zIQ2e;?p*bjF3!5<6NL|96x;UcEp=!#ik)`URzIOzNjEn|)!tQ6=4KfH$i)MhZBN*~ z_>GB!x{+yY<}ZED`p)Z#KU!Xt;e|5t!Ee-#|GV>S{?PN6%za_dG_rEM6Ww`k19!JRP`Ar+?3-1qysYIy^@`-qy}z9f_(B!b1zMorUSb;m zY=1n{_3Z!q=`EXXf8<-gdg5O7#ee$si2gr!L{B_1LQ{yI1bJ9YkVxYuKj1d5MCV>}Z<$6qUSe8hom)|NhE}|No zSCq-|n%2iC@f>!)F=5s_D4v#3XFttN9P=JM0O97r@Be2`ARcMKmRuCw(9^;<<~Z^k)HVVOv#2R zo7>TTEA1|4Je8E$k{|5Wgn1hMYBH#0BXj!6b1&y~jy~p$^yND}m_)RhaHZGfA?AjBete)Xt z{r$`jg@W>#jsnXPWM&8%m(L3($Y|Z_%SUx#jHQmPQ&cc%wh`P+wbVP$Dre6sCR(~$ zQ$Tym@VpYXPH0b;(Nd?JJU6S9XO!JX@YdhF3{pj+`Z{zzF`w0hAHFAl{W5uyNieZr zNRu6rw357JDNiLnWLfnIU_PW00XMBBZd^^=w}n{RM9kh?F)v;7V{zx$6LMGF#owX=QKVZCS2=ovJ29oDz)(=dj!-D*#t zx@HB#Q_^pI$DHj}PIk^zTy@d?Uke}HHowL#jU|DKnhw`3B>=$CWUlYc2eECQ@VVGfGcG=As~N5+Lv{;zoDh!DP~=?ZfMHOV zn@>yc-UQX!U2^%W@{tX4`Kth6DVo1<`I*{0lv8-2fHT3Bp@!SFVB>_@-Mf?~STI2o z^sh>WZ^Nx~R{iv7w?aZ=XAZS8`7<&uNHrfrib-vC(V!0jedNk3{&L0TNlm-sjn{nf z_;^2xlBV4_d-2)-nW5zzV{vcT?)l)OV1_+Wn1>G>QJPu?9uMkLLJ4<%5(W%0EMpmS zxm6<(pnAW9+*rbh$BpK&v$2h>Yt-r*wWl}4tY(Cnv2E?KxRFwQ0d~y0xFi#frT)0} zl6mO*%h2_gp*yd|l{KWxkVtdm;(~ud!UU}~PDwwW)YR*1mW$IT54lEo*Qf&h44rmF zh3&`Px7~g;5YT!o1i`&%J;U{j5=G{8)|!qO9smWJ&Jlt{f`ciz%rt3f)=j^<5R6Kn zzm^e36-=6_3-I-dxzTv(p_ch>$!LrijS;F{RIgjFo0@uND9d!q2StT&lny&Z;oyle zPr{BMZfaxV$zCgN_hC*|d;b-AxN&u|)z9oL@7MU#FVXwHH?TjfAQs;A`Qkgj7C5-0 z=GlkzL<|6Al>q=nMXCmWe~$j_53uI|P|465`r)C76FEng9_s))DsG3P82}K=J67E3 zx$b@}4CiuU7K~^DZP}gL2a3pX06?5zoxThJVjYGYC*v@y3=L1Z9DAnxvvvo@>}c}U z1a@ucy4p5Bc)q zQ;Purl0;&0@}v9hg}KZXD@C)ZcYQhSc->;As~~d9SD^h*0cD1xfTSG*WYDny03ZNK zL_t)~jZMG;$bobBk|WK>KurY%OKCV zR0@w}(ma)QS@ME6|2qGyGaAc+$BF*(YzmjCP=wmdP?$4X_(?bD_B~Qcz>aD%)n#x* z3Onl)AD$WmTa0rtQ8C9eqNVnw-}dXC4gfG3U)cr+`bJbn&z}LZ~c{K>#Wm z3Pndd-(nKC`9nU_Gj4jzyKl7zIurmfYTrlJ)Q2=POHg|!IaWO$yOk!fo=Mo6P7(oX zO79GkkLqT|)p6S|Yi}e}fFBYtMQ?+=wVe=!e)mO8I4ggM3xNqV7 zLWkp#!v|I4Jhm~SciD3N?NlDQMw{{`Cb3s$y|rkOR}^rT7b`0CwOd4QuAwTe+c*Ai z74a+|JNbbc=#A3}NA$PekDRHIUO6)7nRjfDKO1}e*;r$S4ykL6xIn$BKUc+PmF>d{3NuKK! zGFj=yv?6uiF}^*}y8usCp`qMQOqqFwH{X?Pt_vL6-O;WcKFI?>Lo-xV(aLI+ok4aV zfQ|sgR+gXbz>eNWn+oSg%ilEQ*l3`YZ961^hm@gwlX776Zi6XFKJ+#KAk1(MZ?f@( zkq8gC;Y=-foYZU@yGcMvx>K7sRSl=IrW9XovhD<9?p}A3Cbt~f9B()}a-f;qS`$0I zD|URB9v@i6&(KqMQc+O~goA=)%qq40PI$L&u6R!x&4fM(iT^DHG(5`h^rQA zA)1W)WM_LO;LGF$Jv;_94fAFSPG9o=);%{^239XdTm`Gy2zmUM5UVyL7Pxl|u z?Y0z5E6LZJ+nBza8URG&_@U>`e|=uumvEUc_lEv{=AWxd005W2*yFXOd+;lp5>=hq zU->jLiDQ%JL*_osNa;Io9}mcCbyhW0`-_TO#pBc zG(i*eFGOtN-EZ#u+huhRetP+mE55b??tWv(@0QyyK+FMW_vBuEZ*1CqVbNjJx;HJJ zVQ!r^vG_+SWOoZr<)U z#0!~MZOSARPr{D9@>ylt2#hv;()pjNNm*tPE{J)_aj;-^uc{~e>*Ce>gtUABFrq4U z3f3zNBJp0%w{=k9c#c;RqmDlA4rdrAqMDqn-FyD+Xr=CwAFXb>-*x;O+-t>iB6rMX z?wx~Xrd|AXQJRl__kgtp)JenLA5WqU?}*Fh$S(UB@Zfk7SxsbSNw0)42uLzY za~cDH>#LSaktlQoyFb>)> zcx<~nUtG~!A45nlVXQ-P`;FJy{6YNoiBbS)Y}U7YBwezYw_C?SOAIN!96*!+fZlov zI8!}Ej#;azt{PGbOweSaeSLX4LwFdEjP8jV^?@7#ghU~=;4%^sA)~-t+?~GD>51md z7}#cn+^(BydN!CIEZgJsv>U(rI*K}}6tqXrtNdI119tK*_ApHNT( zpvYYUqsvW6gyB}r@d|wa)=UUAwqgK4EEf#-D=iY}&?l9OR$c*sf&zrC#WmIDbpV8CQC8(!BA9qx z&dwf{3TH9V8O7R#=mrgc;xc{`DXg4&9~hp?U*cN*wYf9+bM}?)*zufY(!gBUffh~f zQD2XD_-~j}YG?43+tsUg#_lr-cW3l-64zrL;sr=9JORK>&;(7;C!COV_Fa$ds1E2< z?WlO`7uOYBFg)d{m!zNnIYB4|prH?6@A@#uW?Pn3H_pmfxKC_JtUT+6HCya+?wL zQ8|i+HlH%nz-fnVd;gih;^LVIW2kG?u3u*c0KW7xY5!BB`JvizMrtZ&EdwilQliJ3 zs?NDTfuON#@3@_LVwMjF6{{IN`J&8NoCEZjCtlU_vk(9*o`pXyC!Gl^W(7kYtyT2% zA6KRIKaY1<;baXtRz(0njf^d%T8sG&C2pOvYP{aPt&{jV$1^@I9HsAl5FY*gS6+f; zg&wLPZpWy)@#5QJx{I8D_O}oA2)fZ50Nc;1adca2!cw<I#pS*6Uo)ohdn~T)2!7MLcG8TBH(MKiZ<~;>kxHZKqSnP(s zKK$rcw8j$vkYlpG@2lP0F=V1Hn;WwnJ-j>boLS11eU2?D8~y!f8pxpv;vOU7idbGf zy%kYH{pBOx{M?&bws*`g!Z>q{sE>rYK?l-ym^AM}ysc~IBh0d^@^Mmk)SWv3?e9dt zdzks@AJ|8q`9v77B;lvpK&{}ClE&>jf^B{U0IZjl$j#xUJ7f4%y7We^^hT}pN{#vL z%Ap!e2o};5&#&y+>x~hBA*xbkP`u}brrbPV#Pa zzz+bn`PW+KT#iL6Q5E>#Pev@Db&j?g7&B?DUjEFuvK_tro?hF0UZS4LQ2;8;Sl@RC zNvMJmtfORX2e(v{UGuN*`^n5T;)!j^_67q0UavY8QDh6p!30eJa1%5^6Lj&ZQ%U>t z6Up+Ey44H_AuO2HKKZlP78V|E^PLU`E@&Tyg4KeGr(GIDYWI^TqC2+_G}z|P9Gjn% zuR8hscTPWcPvwqhe1p?5l^+SaT-=({N{*$EKQ~aXVg#{v(-T@Od~SR=Q7Ne^001&n zE`JH*92Vm6sYq}`V zsV<)1Z(2^BgtF2g6r})g4yrI_1;#X-!N;0y&F_y^*YpSAhSk{X9-qm>nr+A4+=MYI z@I=QlA?p%aiMlVg*p188_c07#&*_kx7#eR zW;v-JHfM3#;#D2{KJ5CtzKO~`W?*3+om@nssUl4s^KPU=TXbC@$;NjQt?kCm*A5pI zvSJ~U6{?H^kklzKfNspshMv?@TH^_>aQ2+6;_fcZSXKDh|K^;-)n9Yl4X=Ah+d$Lk zKVImU?LQD~FUOZ2@=QCETWwhdP$!vV&q|qK^2fJNV#3KqN*qLEn z|1Bmh7l;7>(;S{@4v*0TII4jLJ=RYLU|8UU^D>E%#SJ~NrK2H9p`(;t{rYK1L!6mX zLi`m>;o9MEjwayhOUG;>GTy03umd%W)0_2uf6z}n&y-%z%(xx*6x={=QJ>;0*xGT< zysB$Nx!L6IJ5+{&NbH|nBZ9GcX|S5A14V6`HJO#wttYS<*7WbJdf^VMalOvoJB7V> z3j3L&-l4|~VmUq7%Mb%IXu1Ocl1cLVD*>t|k$K4AR%~a51&;WeEf@L4=S@|=_1VaO z-Wff6efO@2$kW0cOda2+ z&!2{}(hwQru{a@g*N)&YH&mR`SkcVYzXSUD9x*yI*z^LnOUZ2UHLl3ZD{weqf+hgC z37Vh@8h5I!h#fzfc>4K{r=Rb5_x*vu&?AqxKXmGAl#;t{<1@W@Fpl6hiIbep@#;*Q zZ*2R=){l*O(@^%pe#+>39}ekfbo5B{%4;&r=HU=G%n8xD%FgoLDZg#4O8)-Fqx)WN zKe09N>Q8H4{C*0g24i4J?DngTGqnRvWoOB3HQZ@v=eSL7dM|R-YCAg~ftBrkT^~Q6 zeFPMUHlkH@7Rz zUbjjr$UfhZQdFpK8B*z5K2LGly3Kf1hsj0M>lxv7Y?G#(E$$glpnE@~pFL@~tSmJm zfba;5KcVtlK9s)pd8Bx7ngc%CDxOEJW7O#X`VY2hz`q6mZ;B}{-TJLnWu5+ae2qR8F469V` zqG}QV7}PsC<0sy=6fa}U?s4bI{v+t2CoW)4Jqm!js+V9ESu=aMfqWhGwfDy_WdTAo zc!t!xhqITmQ*OeZDP0wWdISg7=bdlP0hU3c04tUliaP$Fv#)s!Y7N7i5I?XW40hyb z%|}Qw)Q|a;GE-Kj;15hSbo0sE=?jG zZ~;JRzV@@Pg@5+7aE`a%280BY7WDn!aDu~T=Z6PuS1lj6s24>cCytNy?2s8dY$GL& zH(vMMerI=}y%DS*tc%W@YfZO|n;#`BTAY*T#u}<84CLXsbTJ<$mH+bQZzq59+Rqo7 z&$}%WI`ZlpEemeDwn&^9#W9l+Jn-zF9(!Z=i4LLk^1HwPts6?s2tcj+%dMY(!QU%o z7XJ3#KQFgVu(}4g$k988tmT%{8_rB@QToeAT4b<*$hgcLCrlGL`0K(yDJZ9*+4gkR(uf1p# z>pJiEw`-c_+h1qBCKtIfT2CEIxVkqyxm{`h&>wO*qX}BSEvYHJs>$A$+vncSn4P_9 zcOV$oblorvYD8m#u<1m-y9b}F;gA_I>9&kuRTVb3rc%a`myok8QdF5xqU(An6voI8 z0Gh7ZEHu%X6k~>ZT8&5g#jl~J7}E8459=g}lVv6ljMAt~2uUWBt-jue{^s}EjOmno zOjK13NBZkkhGM2P5DWr3X1(*~8&QxugDh(w}T)w(*+yJ=8!G#X`9H5d#MGJ?Z`rfZ6- z0i_)s9bjSrprs8l70wc&sB9cW04X%nP zHFtHxAf`aDQ|V)PXqu*Lx~i%J*8UJ!7y__pnjVfwJzeF(ai*a;W;Ig)*tC_`G?X(ZD@}J10I1#A(zYW607+zILGomI4FK?zP=aN)4B9|#Yc5DAt)1OE zIy1=%0+W=I3}#x|!Y8ZjH>`{5)tZ`27%?3H48xR2Bq9+704Sl+w&r%9fEX44WEoKq zq-2s1B1w`SivobEvXZ1lB0a%wybB-{@zqmJ2?TpQmMO#6r(n;UwFt4QzsZy^M>;e!r2|+*h$R#bQG*WRn-OlR>@-cAd}oKmoeN6Eu_ z(G>M)#|baP0{{+M)ueoQ`x(z9uD|vt1@?uchyegZkqjxuIs1hgc6J(uMyRGDBaxmV zV)H=HEMKecJQ>3pQ8aCswMJP9u&Sxikiq~*qmihg+Y*U*BrF5KY6kw8Fq$GBWJuEr z)xCsGCX+_bx=2WZrt>POM8*I>4P8~_R-ez;Y<#dezV>hKv802EcWoK%Jwi-X6+7TiCA+2puWv?^0lMBxB1W^Wo(MzG3a6>LkU+iFEya6Wt{BLj z&-xJBALtcIl8hi%D3427YSIO8Mv5fky$DewW`b5G)FQWbw#&^D0Jtz~H8@I9XE4B- zy@c&FGRkzE1^^%#v3NWw3#V$rl+vc-jS1jHzg+%QC3J*9Q~0S_j%6u)WDG(mC0(59 zzc-mR7004+VgRLZ@^m=EL;7fXVmK^|Tv&<(n@?9TR(CiYPRbx8_0}ELB63`}Un%3&`0l;H_j9lkMq}d=+RFaa!CWI5+lP8&ERgDbLo1<|Ok1$RT03>73(MU0iY&P1Zei0>!7*3ew z=8K05I54wJ!4P>6TU<jdliA!=Q#?$cm!Hh69)++3F9Nnp)!agiHw0bgi$t8wrPF zwJ}OTk|i(*A%?>e8N82E(TGWLAl+TUa$F!rwEx2II=F6WX%V$t>NJaFB5ml2CnMF|AY_Y7a4D5!MdlLCb;#l zO&*WOa9f```Yr%yidr3rgtCoQ&saDW5lt4o=A>?RD~QF|)W!*fDgkZZt6HX)C7Pmk zc0`ERo-};P#ep3H00if?xg)F`6HH3d>^tq6azocsuEO}w2M*mbZ$XB|GPn;>R|w%q zhBpHM7|f)l^*cTdHdr~U&6AZ405+R%`ZUU%!W3g?f466zCp!*-ZqNt6+iyD;Mcrhg zPKTM8RV?bF+ru*S{T;CkZyErwOd!LXk&M_e%M|1n0D#lkEXy*>vKoNf-JNMj)(FdL zUawPJPnv&!bn;z`X=a|_bmixF2Y9}7&nxEBwL8p&_8xP2Te zF$`nN5@dJVmI#;C<8T+vC9Hr2y;ux|a4bGpVXccq9=&8mZf-a5SH+74l3A8@x}1X7 z3jiG3X1Ckyc~;{@B`ybK8dU2o*CF<>&Savhfjn-DE+u5#Vm6zzv(oyo)VFYkpl?U% zd4ki?PY0s9tRv)cy8(b@JKS!!*W0aXOrQG7kxh93Kr~rMb77tZfEJ>|GVF=X4Cr zLY)v`Qa+dFb^`#_q&Pwu-VBr2!Z0)|D=RyH;DiYZJjUQ~*p*JsfV(9PG01^zw)_43@CROQp7LB$800Iz;oDR2} z!K%$_bvgh5Se95V7KULMrh6WaytOlav&F0d^j!#oKrChkFHWC&sBQRm7|fiVMW-mW z+AKt3J?_CyABxx8%}JRtWR}ay>K8~T21Kvdo9``ncYAwvhsT~D_hxtlrvyorGt$jn zNIhACX`I+x4%9P?p3&P_$10E(TfRNGE4F;L{7XyGv)U(p*l37S4vPyhrJbOS32$0Q|BFmR0#;^?Ia5w#_vGl|TKPTV0Z4719604@RmnK}q< zD7mS>1)`)|Q9=`#ylj@s=*3D%^Mq%1)924 zt)33Z8ZcQ8^zT`bb1+vgTLQa}GQw%}z!y0H_`@?QMliJqefl~sFf*Tg)EUPJ00^cF zT(aJw0e~)=owG!|trGwkN|)*i6#xLV+DP^Ga7Kh2>bY1=-91 zz%Y!(VsSV`0AMVjHSTd+W^r~G03?$XLDXb6>lm9%CS+6u2rAT@%^0=oA6h|GF>GRu;otPZ2Skie+)8idlIOqzC z66KtNK7d6S8!o47aH1}0`kIa-?^IuVQ)`4xi^?Y>;Wt)Y4FDY9XtkLsP@d-xyJZud%nJY>rF)foth#x~mel3cd(*+9v@TZD1vTBaM zp7Gf~mGo`u$hV(wJsQOV3jkD;DK#)3?2F85DnSSVAp{`Gmy>u6001BWNkl3eu z*qm0;CZ1hH9Lw^U2hjuoj9|iwoSl_r$g!b?NB4;5o{n(Yv$HdcxYO0ySy=#pP%JCgLYsYzAqiF&W_p!LbCB{Jc--Au z$H5aM6w_>WNffgQ3a9m1v$7yC4C8b;S(f#By#N3mV0Su1&IABJRT;Mm0-xb^6GN@3 zKf>!`hSv)KhOIM@+5*xfLob?Fg-0b%d?CIpOp_d7iMW>xj9nRUXWV5lZ>HZF$n;2 z0;~imqO|LTk`kcQ5hc}rsm?y+(oPzN$S}WMzHsnYs9A3C^!i~DIH#c>5^R@cemz~0PtpHSiHi+ zm6dLR&rDCVcja6oBshKH>{Gj+EGp~dFv2K1E8AjG%{C6A;F{iBFvMon01{@i8EGgz zE&XgQ=0G6MvYOpyA0XE1YLg$ldw8;nTnIsrkm+lk@mx49&E2d!Y9+KNBO`-x)D!G$ z&Y!j_&ey+bEnR|~o^-*A5CVXB(5@r}hkN`TiYEZL3HszwqWq~BTNZ!s<$Kq-Fw8Ib zPSf`{SNhdig-o=m!^pgK)uorDO++S7&{>+FRrvSoz5oDUn+sxrK_eQ3kHDgZTj4H z7eX;}(e%ELn46K|6fdkRt4TodBpHwiuF*&LA19YD8v(<&<`tiM>er6qS?-xjhQHH( zNf!M6=-|&uAPxY5Z00|HGWm(0)IoJ`wjiEMTE#Yi6d?wbqb7ojj^uu3UN>rUc&Bdj zayii#bFR9Mbq>@^BwhcH_qPtyy%UvEO}(<_QX8_1Xe|PRGWj!yx!-Mvt$U3bC3uD{ z^$kP^pfGL47!EufmB$S9w`ch1KplJ10}-0o?E z^K9_Be)BL)b;ku(P8f`?D1*L1jxIYCWR3GvLQ&imsN!}INkO{~b5_%Eu9xX<^2Dp+ zo$KXr1m(y&zGYHvjSUc9zq%VN8yu_uU%#{7|Jf+Ny=@F?AFEh0`ObW1#ij1qv%A;! z4}W<2>bVxvOVqTkk)3+;Xxf=Q85~x$bsqr0*1bk&h<@&R{MrYkq`+A6wZc%)4vb3_ zki=_Gb=E!L7??CirsTy)%YzwH0ig5e2S$8crpI{lHPiR+O^7_jf(i3uE|-HC*1hUh zZ0?;}db@00<>AHmSa+_Y?3Ugc_{2w4O1+$|P<7pX^J-dNn!V)jft`EbJCuIrApXpf z7#xp{b~DPI-B$atz~Zul*;lU1{PMnC=c2pYoMPb=05DPYiOCB-cdNE>BfInpWsg1Z zqqJfGX#0;H*_rh+3@lP4HHHtf(?0uH2>|$B|DKTu=HB_&v)^ubBNUAq>(@D&Uj4yF z|L5;t^^tq!Jp)=exZ~v3Z5?+MyvoeHb2MFzFoyS_8|I|&aS%LN2p|?@XI&N*okQAS zti?uTRcWmU$OM3RyH-$i#_sh}fOO1nk=hG5O9^i~&((S=5j_?UFSgpxQLYk0PsUlZ zbgl-|km&*Faa(Cu=004G#!qxF3FV4H+^N#G9uJ`I1OviREs6FJq z_otx{eeNqXbGdP<9e@2k0|4}mzh!2uY=26ensnVTuVknfIUSB~s!P4`g~Oz%m|e73 zj8w(|VCLdl7%54k-AYRGhKoK91pb*7$<9(<7W&;2+M_=iIDGArd}DsL{cObpjLp** zx<7tzif4q9L@o#7`7!A5@t7vd#)t{b!UoAF>e8HSy)8j0*qxD*%e~r8*Mv>oE7enf ztT+DSj^&?v^o$AsYSmwEEGxU^u^JTsl*$LMUw-Ec%{r)UJ0JeWZC71dw&LnLzWK;* zzXkw6LtE}yd-uj88-Miat8Rbfv zm6fgi)Yl*0+0k{}qxwF2@V;BuEiYSp(>=f3RFk9tK%y0|{`{^R)?QLpw&Lnff9IL< zC;_eE>3{$9nwZLgdT=+qAVX zLFWU;MQ|hr!zYfpYPl4N;qp_g%W+{VT1KP#u3e$F_HoO|v+JPo+4Z=z0F@MASpfiO zYQsPMA9eeIamy{W?h;8>5G6wc)4B?3KfmDGCMF?`Y+6`mt<~B`mA=qiIv;>4ZM~8E zl})xUOnDB+o5%Dn8%xS2$3Agq{;h4=-h+vf!d}x(i?+wBJ{oYTo%O)-+{&L}&yCF677t(J_D(ma02*v8|911~kQCFRez`qGH!#EM$$0C| z&7W{7M9pc4v*?-g$Gzn}2b8<+u#ELrC~GPjHIg0(!m&!c{)%3af)K53Hcr-%Pavyj zO_H8}uC*&iLiFex2s40(JXJDck>rU(y~ET|j}U{o{|gzGs_U7qT0N$z%PBFaK5)}{ z|6LG8Y9D}mvpg8O+SaVKHUASjjgS@#MNg+zHEaB?c=Dgd&vj+z74=6#1M50X%4fg* z6?@r20Ve3;(hy(^o3Ut#r(@rV5CIZ8dDIWka-cFwzzCi=6wD}Fn64ao@Jru*U0?Oh zKR@>1e=L_?{;zu;I^I1Y)!OHN`8rv7*F9@8QQPM4-22CaX*d7qp+ElD721a1eGt+C z0Mye@KJeRJuA9F9&|m-XwIv-d{N#6gqm)Ju{P*48d*6Kh5B~c1Uwu0Jz^}jh>wQrw z9e?+kwmH(p7V&W5Pc^8A0^wC0ja*WLV;-)%mVnBdn04ZTA#T+87DgG}zG;sjjBt$9I*qBJzAe%Fha42QPN|N)>8_yA@IZGfgU4C%J@6?#WdSE zwpQCr_8%Y5?XNUne|n?HGGH0j1CC?6IdeAAGRM->4NOb>m>m#Bo+1Q~ym;}_NH}Z= z063=m6#@~Ql*hAujs=8l*0nKA;{yM0M6nWt2oVMkLZ5sOx8=u+*Wk>w9&SV#Im zAvt9&01VE>J`^=-vW}S<;qEY49^*~Tb?3vG$wdSJSQd^~o}X<-sKtz@Pw6)M5i9cU ze5I{wz(q6}aTd6kgb*++0E9y+b_xz%z*A$b@dN-|kyRK{_y@u3QH) ziHJ#`9NYA*u{K_HM%{l{_IIAkiLV~-j2?gcU)G2o3Xd1^FP(3GbLZ*!^_IH5ZMka~ z$M^54k*=9qy}Q}Ipe$E8`onjl3m^FDmoE2UfN4KfPu%k7mrmX@YXJc0lKr+{|M*i` z4CqbIZa8dN|L~7LJI?|zZ>ppD=soWM0BFIwkTB))>n@w`#`8*Za?AXg91ZXO#};k% zufKQ4Qagf~-~2(#k2bc1wb|Br|Nj53S$FMhCk7}+yI%OVFQxesQG(0Zc1(`Tp(R*Y^i6U$XDn?f9BMeD}5m7J!-GK@|u7t8xN1A$mg5;)nnI z=G9-n|G&RtoP6bvzyJC#TpND5+7@aEXo_ZCao=yhY;OPfAAkDgS0lC;f4b5&;So1M zsmQbJrr-a#wRj3M$V@`lu64Y+Ik-AqDD;F29lpW8wJ%!J`s6Q-NRV@RN8SoCvheDh z>73bw9(g8y^ZJogOrmqsZ%ZN6c(`ggdTYC0l8^7WmTeEvod@+bOMB+GTh!mFjsM_7 z7T~KAm4ApDt*9(W^xzeckwOPz-36oPm7)@BueBfnfWXHEzS~&?v%P3hQ=_zLqi=z& z`!Gu2fKqz=gt2CIudH^IT^Zc+qHX@USKOlsKk*$x-Z|&o!t0x&0Mul`=ZsQdv0DXv z@530PU@?tW`=@e5Z9I%(nM$C2Kw;w(9ni%Q#2RUO%XoRrN$ zLmU6>Xly(HNW_hmtJ1jB{_|hviTHSiU35gbidv}0XS4^pt&!YJv}v1Bm~&ow z!#mpm&dIt#oCyitg2iPg4(q7o6E@jQrHkc;W2T}7-RNOV)RP<&2KOimbhDmfUbiX1 z)=_ry$w>23UnM|yR>3cnOnIxhX?^abb7oiBLIVIUUtv7%08~_=KxjBgmFA{1rxG}J-VZiZU;*=@7VKzx z@PYIjzGxXb%SS~OYWAVed}`o|wvDc)UsC0Q5pX*$(#&BqXV`A|6FS}sDo|n=(g-0A zxNqb#HE6U0VDiz^7R2xXshx;(GmMSf%!N5d;aECWUyqjpeEH4}#(pvQ;B_13{)=5| zZGgF%KnNH*wmDD#?dfADZkQqw;q+dJFVNrq2LN=31+YA8O^@&2U|Dbp>$upVi0sbW?q(Hag3ivlc1{04$jCuLU;@X(5=3S51fC?P-Xn9jFx- z-+yiE-oGAbj+#C`!_K_5m<_(#sO7Am?&x9`cFtIkA%A!#q%SZ7a2fMwc^Ciywe4g( zKjq$%?xQl(q_WB4y8r;D>7Twa?;Af{f8)!mR+g2mT)k>uu1Tw`Xdroamsk-10J6>h z`X9QSq_4QnTyglF4JS`k9NWL=aG0(dXvoBBDm&w+esKM2rfbU}ilSw=L=wJ=CMdYM zxJx=CYw6-VW&(f%gjfua_qks@a7z&nFt=2!`qcf;ZtqxiW8T00?VVdpb_<69V9sPq z{mtKf`lF6ZZ_JoL;3lY-N=%lY)gJ#<>0H+0l8xZH)fPB3(2=aPSn#^(mewog<)?-K z5X*A*F$dt3PD&xKg!BUcwkO5RV#}ntBTao+G7D#?!&7e<1=(o-abh;1g)@2+QQ>>% zc2%rvsiTc;aNE^f;)jQzf5cNTa4tdYno0n`W@7-rM`mO*z5Z*+lwmI}t$yzJv+w&2 z0B9Pq+u7_)5!$*i7s$yTs0s^trF74`W5zxL`o{<2i|3gEfJW+(DUHcqe9qG#7B`kG zG^M%e`C~kiOy3Ld< zS*ke<)8>ulY3_Sxl7tQbZ@#Y0C;?Yj+A?ue>B%YrlW!?_4q(BEZ^UHHb4&D89Y``= zID@%xi0Kemt24un=Z*vX!dd&~lrBcp_V!M4s+LSr(gZ-? z_mE#`)xBcG@X}X=2pilvIDvI&F2l zdPn%+%Uz#5{>t1rd7Z|oox}VpAwf$!{mN~l9o& zU_oRZ?qm|Ky~T6n>G3a_*P=+fL;lQwbKC2((}Dl%ejWhW?5XF2!BYx#wFyJVZyhV* zTMYoHYo>Xbpy{yvfN{sQDaP96p!qq*&b@f{bP52|CX>OdySko!cKtZZa91B>>clC$ za;f2VUML42E!fn>RihEfr0!Kw9)9^fJueFt z<@V!=Ijgv7%ai7;NdRt_Rg1~|nMSgB)<0qIi&t(1LmSHZE45PK$!#qXR%|=9@+U(E z3r{Q3_8v1-4gMDq7RaOdxZepy$kiq>Wv8T@Omc0mz6K zgu=;z=O4|u_y3*Gb7`y{S<=Si4d}=13$fV*FmB%6!JziclTDYcNxN`tS7hVHo3qEU zLAQrDd{RYPcC((8gu(X+dZ+tw+QlB(hBPO`z-oJK=kWY^o4lHO<`?8?rh^mP{oke`g%Pq%zid+C=%K1(N8<&Hr_8)0Cv>R1>d-N{ zSxz_21gbVN*V7he|X`QZhGy_H#a|aX8}iaO8ex! zL<4&BuL{5kE{~bdnc7{H5SenOq@ze%R0qancUXFc4%>=mXLNl&K@-%6u5)KdWP}4) zjzyO*Rt$=%5(Tw8-Mb9Lunak!IE*({pBe>%0Pk(E`|d&> z)JPgOO7y})liSUowE>naak?D{0AMW9g#wQz0ih0idA?9{UG5_@v3DcZWssU z;YzF+8U+AqXll5XF}vN%{vA=$b;V`=%9-^*2~fi1E*qLv)M_5XE1L=KwW<5idZ%fJ zb+m7a5P}|8acXFwCdPN|H_C~8!Qy}4zI+*u#^~m^_04bVo8JZ5NWDuM1a=-Yj#du0 zqWykl-<}Wva9Hc&>ZBYfEkNs6p?$}U{t^<}JGwQTf*e{^gSPI34IA-Lv3CwD8c|c~ zOKojr#QV!nBDqHmF`FoJ0ZxLt1~5_ku}+czfGqAVSPC($b?zmOvMWO$Zj@_J4*ucM z6DS;kjC3-%S2If>D-8|SNv-+QELp|WFaUsOsa=%MY8^^jRruilXYae?q&TkpUsZLQ z?#Z*WvpK;oEG$dTNq_`G0a;eCWJ{J$wjA!9?tFLl`FxIN`+kn+Y+1JDAj_vLBOyxy zfe=9?5k*>H*_?NB>QwdnV`tgGY+?t5(7sRq!4BQk)m5)vz4xsbp6a=%eu=7=NujxY z1nwn-#WX&Cey>ze-5-Es2#`TNUF{b?5^PG%)b1PYA~lB>gZyRNPg`3ri@A(>u~$#i zIByzvmW2G9&-doLV`;A}y->+`RyPkb3=B>c-t$zVhLJ`u&-d{m9}?r@BZ^sb`{2?gR}~p?cK_N`b+UyHRP5erM&5$})rt zmLS0j0I!-|d_b`D^8pF%-OP-Xg84x8jLqk3M|}H+8({nH-OTW*-J9oOc^w&b#tM_1fo4}sv& zjVlNDP%FX&nrjAx64>Y5967pqlr1csjaSS?L-8cdlJ)W&dt_W@rzk)w?uoIE1KUPz z`U0bD*Fj_W!*USqA_Ue z%mC*+{-U^e=Ga0@cn%ykh7rU4lATEcSQXf|cQ|Qe zJ=PqhEkb^KUXvir1kFR4x3S3nK)U~cV||quErr6 zwOi*djU78M`b=;Q_4<}qBMTPxos=Vs6KtbGxHKK5E%VdWmEy{#kf*Nsoev93UTV-P zyQp4`d8kd@SVPZ`Z6hWG)duoqtbT<+57fgebTh6(b&dnkKwpkxsXWjq(ul;t*p%aH* z{q^A6mg){?>P%*mdmWf#?7Ce_n}CUpdL{&bEY)ZQond+IDWwU%vhA z*CO%m#;ybf1c1zNLt{r+vrM~tL(Z`ueBp=B?>T;a&lBJN%3teqXXOjnyW-x(>Wkm_ z?i0I@9ozlH_kX-UIwl|BYj(=BR^GoP`q!`g&r`dO9D4na-~8?ymRlZL=H{H!=h=_{ z?#thMX3yIv4sCh#8~?pcx$Wc2Co;fIP#OsiDy7f75KS1Odibd{cbK|y;)&_LmML?m zjvnbT4#k^83mVn_E}0N{MesQd`B+(YU{=g!4QO=F|?$mk4J0%E=eZRm#}Nu7lq-CmCRzte zOQHPqiIUlDFDp&97!bM=~qTL z!x9`rhEe2ZOEytkQW5sp5xLY|h3BC#2mR*B%N2Da3ShOs*#>&}6rGxhEOm3IjZIBf zEcAg5(KQPPJ7wH?Lv+uP!QTI!e%#vCJCp?B`wtU5+(n4!d_?1Xa0>qio7jHmx7N>jEhiHn~#B3YofbNm&C8#!=R{ z(u<0a%L!$rXvspnWFekeg?@MH!_iGYWyWPg{Q&^rOfAasf-j2!Kv6!u>lS@d30=Fw zXlOVn&C9lJI=pDssw7QD7|(IHOhHbeN1rr%N7p`o zbzs|bi8DtAb5SO`q-iNdMwz^{2XPj&Cl9*kv4`6kgHdLnv|FJxf4EU~IuIa)NScyR zrhR}Akcub8kL*pHJPZJ0|45O9nYu>O6Zp7E;uPZMg$u#BR&-TWX87Q?Rf+!2;8?V@ zZGJ+J1Oeb@zqW4PY9YOKdtCOxZ7j@yh65YPXZ>nPb)`%ZrVTuZdlO zH1hnBU#r#CA_8_}#X-sdqj%Dmzq$eq+F8;Lq*@4n34^UaS; z;sF3T7XRCS|HGtHzx}t5fArJed8Tvmzy0XbGi2;q{-3}8uciJczxIib{>vk0sz3kz zf1YnwW_{+%cUWKj+DGoV@8AF2w)~sld0$b@Prmj{qt0c|Uo}1SyHEV%@6M>=l!t!x z(+Bg9KJvwf9{SHGLaV;|(@$5+2(V}MSAX%ZbDIA2AD{ZdPY+K1)Pt1*e<3>dy^O%Q zmVfC7pPAA8e_#95r@ymFTlcMBeQvfD0m;4Xd%ykS+}0<*{>hJh?nke|hX41gPtQ~) zhHHWbDQbkcIsWI3r|X-H`))C9i4y?8w77Bf&!DH=a~L&Pyo;N0|3&Z_-L?z&Xa$_j z3Eaq~Do>t_jIv{GZE8iivT~*S%8!ZDjtGW7KHV=i7GvI$f)r1kjEG{Nq@6$C>9h^l zyfGoFzrADs*0hZm6+mJBxYnVVQ`xoc;lrZC}>6ec3qmbXukGVwyGyJ@$gp zF|f**Zm`oenK4K@{H&q`EuD?Yl-v1vpU#+~FN7=0GA6uY3Dz=hmnb~V6aV&(*ne&7 zxzxvx0f+SiEs)SNtIF-$8`!@uNQOtEN2h`9+Z))k+Yi8$l6ecOGccaAKtFz)ZE@8l zBCQ0&qH4r0zv}C-!tkyQmyf(IGuJy<2hDa<0H~=O5P!98uP|$h=JN~(w&rJ>9_N4? zrj{X1L)l6Y0Pejf_w3o&p1nb*gPT6>(#!3Y=8qf`H;7u>E@gOUlaDi^yfmM2aN`v1I`#Ad4LU ztXV7(W(S{39aIQ8MV6Avo`1)aB%$-6FzWWhu6>9bIvR)P;MLu9C>iYa~8zim@lR+A=0Xb(b4ZgjH2vQJy4Ncq^Kj&Y!dW5`{*lL$5W?X@; zPASo|JwpOO)20akP?6R8<&T9X7rm1Kb=S?ujE*b;`0RYSVpbxeMWU(RvSftIsm8Vp zgu5byD&6&8xj&VTbNQU{ig}L{&8tGK9q*djvg4;=0}SQaUwwJ^S5qs%edDio^{Aj!>_L$uy88x{Q9GJUikZtRAoB%qKAI?#6yE8)Kaqk-+#US-v_Kb z@5;~rdgbRY{JsA1>r($;{mEZ%`s4*&Y2Jh1d+fo1&1tQ>HlHc|^lvwPl>mTt>SxcR ztRm+`NF@Na&-lRCe)EBg7L+`B!8 zYBX&Apprs%j*Lyb{^BR+7tF%i!MMTgGT#Ma&9gGU&#xWo0b!^R$%3(V9_gKpbp<^lNZv9CQaz@|TN1oCq)2w)mG@DfP(j{)w{le^-9 zRnq0HKIrqZnpQY`(oE|_7YQ4T!J%VuRa<%_IO%<>Vdj*}2sF1a`8p81MrPD*Gjk~g zvRv?|zZq+*1(WFj;Mdjd{!ZID-{d(LjRpglwoW_t;vFj{Z%o7&_CwGV3%q@XzIEK3 z=^WeJdkV6@dB|vNhdb7C-~~~Di^~fLJj|MI9zIT5+f(&P0U(h?%V(o8;oXMG%uzeg zcXXl3a zP-M1h$&gb1&KvL#{$Ls=;V<&ANT~4a#^fBUmMz8{WYl@?wPc@H-nfZ(7Xbi7+DAQ_ zNt{;!0MF!?&AN9~MODmh?fi8PZN00SPpvijgGOk9tB`oBlgT#3@du7tS1uic0?7=W zndW80hOb2;7NxmGDk+uSZvJZP8^#1+KvKOzAKLgjee1Y6clvu`mapXiL!>us{Qg(I z_uP@zKv(UqKm70Wp{ko^IwoAYCTN1jO&X&nmr~Jaw1dSuku6`}{r|?SQin$_XEwR4 zPZldO0Ki|L9S|^O$)D1?@rPh$EHEwC3$;zOe*mMSeCCPYe_M-;IbG4&GqKk;cV7Iv zTr_>a8<*XpM*@+$Q$POxIY}bAznNrB!Zu&}Jyo)Bc2E1`KN(X`WC_ZPGsXbfUQ$?a zb)9CMJrmopz56mXUf=_VZ4CFnu`{-Mg;JP1><(4w;r?khDj9=8PDF{56f^k_;GTs! z@YZp9@C3awuiF6b>%n>9@&FedI&Ik_kHy*YIftD0&~Db-djNVaqkKC3WZ^Jb*s z?D11{TH#bBopxbY;8K1S($5^wbkc5v!SE(e~~Eqr_2^Ygl~QXy9NXpAY@@^~uDQ zV~=$8k4$`avr&+Pi*g~_4krs)c!25Hc_|j>T)4notvlz*CVM)rzpnFcd7q7?>0$<;2I- z2Nn+k!iF~kURR8=n{=NXqkH=Bf)X^?yF~2x=U}sN=?lneg z^0Mn21qBwxnlW`fsb;3b8s!it;PTrp{8CRy_1=k;W)n0)k5{$@r^9qaVJ zW25A8lM$=qy%%LLs&`~F47eN+E?f!>cOrV`y!qhd-)H2#ci)Ik zoVywUIJc8ZBK!GcmjC{>QrChH9OD%W8wP3_j_Y^@dSM{DohvMqT&_$Wk4qxtWrN2t z!Z6O7PSXmpc${3nd=yiFkm$`H{+81GAr1I!4?_s7GB^~PpKWj)dtzrP?=9)UM-*Z3 zaZYJrUzeofXP2 z=;1H`EK#1hZH0c@DpZ*BZaR^!J!gEV>@S&O(a{jc!J-9-G2lvPCHFkb`T=lt&FsUs zBB`H*w`3P2x?0r0*e3DKLnc5P_qQGrcqqze$4{fSP6PnSBqY@_FL_F#YT9rioINzH zf7h4pB+ZLFu^*`*I5-z2zv7G zmjbM7`ZeM$f)Nl5Ti)#3VtAXYwTr8qqIt7MJ2Jd4-!gqp?9d*XeCOa9sx!ubw`8`} zU;V7s>AE`H(}=(TCHDPc_`81n^E%m*UQpDWAA|1n^vi}#0VEIbOD*uBja$w776t$# zmbiJ>FO6&Z>xA4@pZ)1GTXyf>z2*7eef{nkjtTf}f+lF(|}8;qgU?Va}p;L`t7!fBjm#|A2WVYkFQ8YVBm(_hlyA{l&C;A!_Sl zZC!L?!A{WVDcgOOnmsm|efZYs=kE`DU1lJJ!!cO3ATcpt86m~ORxe159?Y^%U5I$G ze;R=i$zZh0)bCZi>A6m_N?ZK@gsLjVAK4`P7__vH!XSupErjO14}v~3s3NZ;E! z0Jv;LW}7{CHujI7Exj6M3JHo>zJ?`7$aGnIF1F&jG{W%aWy!Z|#xSQ%ZS0=)gAZ00 z%m9}ibe#b}W9v1py-#FBo1h7rpdqQdyZh9sQyqy20GOuP?N3FFnnAtY7p;ww9L(_m zhG`_iVVSX>8-%vq0%O5YpB#@MdRmOkPV$Dma4dzp}KtoZey|e%Ii-pp_ag55huBDG(yGnXG#%7Hwu9f z6R5^2F(FVY8{au|FzzP+U`$#bIJC{ICSQ7W$DO?8sbrwLt}HmE_>DxOOU5N+-Lbf<~Htw(*4z=U{JPqeu3*VZZB|AAc#dlEX0J0=L`mFhpNhaUn zZ~lEn^Y6C-*?|C0Z+oe>t8GQKl7*13gr9mzIte@soU0~5jMC(_bP?OXo z(4Y~D$2m>sy8RK^B|i3zP*3EMy$c`VKXBqRhvLz!^4Sf8K6$Lyvmc(MBX~yI6aL0gfIUEWKzy^)p8BX0zMy`>%*tAv{1Kf%d@i+sp$zr_@ zU0q>YKY1LAkeXVD%bwtP)ZR_sY7H7;zVkodbbGN$`S*99wmeK@90LH{Z6xC{Gm=^v zu^30G914XbJKFN7#B)VV_u*Fp-gdnCyHO+L$)D61jtrV9XQ=m!7-NB;zr9gtXm&(n z(O^jDRMiAikA!(FG_{Gl$rKtA7-MFngA#i@G$`kQx7**9Vn~hOxhD#MyZi|Nz|jb3 znidKfz_tmI7Ni3JWuR>!UvD!301z<7LcvsnI35wjNXtR`_$ONdfEqMZrK+a(+F>>q zpvyn_Op8$x)52ZN=tuX}8v@ntpnrTQ5b@_ZXc7P-;l58soDs4oUp^g}ZWolT^(A5c z#c4kWN&$1S0c$V_RMiklyGFM0kp`SX^r!^|R^u<6V>X5Mmq__FE=sMVSz=*ctL5xA zO*M2=Rh^r*@XP0S17w9mAzq^ZkcbjsXrsmD&=RVq_n$b^6#OB%Dao?R5>^jAxX~MR z>Fz`333B_{e|Req_Y_ZW?B53)!hh~Qb8pziKVCD~s?;d6kVu*iTMwMIx%i;tOzg#N zCp9{ttdTwJW``dzNA3QAkF-Z&5W(Ia-Ud0AL;fnDmjM&0@&UUGc&zMXzjCV+onR zHDr<<0kx}-Hv}3`(_qI2zu@&xz6|%(Ce=ln`FcEb764>gX41Dzns)kli{+lJJ5gD2 zy-8CVnrZiOkVn}ICXEdomPJuPjs2kwBszr&MXF>+$3mh804N}+MSBc&K#7vLQ58ve zYTLF0KnLV%P!iHP03flrVHnY9w7sLRONkuXc<^Bcb1|oG-5DcTEGc^A;xu*?Y!p`q z)E5ANv5(bt{8kjSX!54oh7(asJpj<`Xk+a!82ell*BCNyb#0n8!~M+PZ+&|6!8tRI z0RZp`-L)Y0(!oERY3*KT{N|=@V%T%wcx!$f@`-NJM<}vTwjiMXtmlnUy*Q~?)wXS- z3~%1MLN{Fj^*LM~!kw=OT6O5ZpBdam9hp>!tEX$*HZxcg4t1#@^lB_BPicEcM>4dJ zG1k@HO$8~jWn&R+=#JYhp)fEZ*BcH;hC?28LyX7caZUvZn}(4{B*MLgLW)TOEEYpx0)Rx#WFld(^t%AzpiV?T zGt(Ty7c#tBBLE=sqG58OP)M|K&;F3_!k~R9-nP%Sl7S}rL1AluOeBv85_Sn<(X4csyoBwEkT7AHN(z{9yo~78hw_M^EYt0m8$!!z443hyqPX0IaE!7L6Qx zR?EMx-z?|=(A?ZC=>hUxz(+I1BG=+ThzM04Da^dxtWrIT9P4t;wUHX+}{@u}jV{RKxOoP{K}olop* zL`DLrq`vqR0Jt@%qPE>nYMyCY{)_>eduMAP+^$hKi$tT;NPVnb)UL&4l}na;-k-+|MEa0p1A7r z)=dC#6Er~+bcM*{@l2gM)fV;xfU2oy&ZcUNlD&pCPc1A_>``k1fFqp4ZZ{C}*k#}( z{n!CK<#wl|FTu=d8uXcBwZ*wl6xk=Hay?pwkuZ+GitIk1MzUzeHmCj(Zt$`E(=Yhv z$ggEhpC{xv03aHPG__kJF|>a90K1eLiTeG`XIm}nZk?v@|EWD^Cdld2+nlCBtHCcs zBP8BBbROHRm{Z$t^%htrTQKxh{{R3W07*naQ~)3m&i%gE%TStn1&Y>`OU3|~%7+s&RSbJh7-SuAuiJ!e@SOt209${br2LONn8~>3zP3@@;nPFdcb~4%( z6{XzVRHBfh-+t4yE8Apm_o|i$j8dD;4glQ&{izp&#YO&wZujoP>gVpm{bywKLpyb++lGLl#`QL#A0dQyU4zBY8~d>8dCi2&mJM#y|08qjd^X zt>>4l`-234gT#NAq`#m0+X;@W}&PAUDz9x?rHQ<#kg%&@J#3b5$nPq&)1(nP{5R*K`x1?Z%w*E&t`8M_V9Sp9%qn@oY^9-DJ9sp{ zc&@@ph{f9ZuAL6Y`~h1&M?F4;SIl^{Z}fVo;i>12;mRc+%>saMptE7$n`dG(mMklX zoqHmyvz2Xkda5W+Pp_NUo&V0F!9Eb`@LSy-wfw?m*#OYeuC;a;4^?ky);j+3 zYT){nisIj45uBoP5&&3v*=|W+u;sAPu$H;pwgDx$1z-{uoWvpfnZSd`t=#gD5bkwv zeB?=EL6zo890b{`xCM)2T&b--aW|+Mb2?HSt5I2_ye!x+vWt<;mL{K<5+NKRhfcY! zo39q-S$OVJB339LVE~G3MK=DT^@vZ_T0Zl^u-!J=?12#z+w67>sbOJ3FbNT5*^)gV z%cS1f9S1aMJsvljs+ct&hKt$T2N5O+&VeN|5P-wsxP1*fri0DlaEN{LdH4jLT*9iS z2`r%PGw|V4R{NqUXOAZS^*%lje|^Ub)7C6B-Mta4idO^y_nO=oW8!c)mfZ<~Hcjtt z(K~8w^KSk5@AYL%1J<@}mKn=r+u#9r^nQ^aHM^Yd*AJ0-Gg*0=A_`!LQ4>sy&2G0r zL%R|6D@RUv?_Sf5_nX9(FFOXqss;WXnb~r*{oYA}A z4_N@9>+tL@?vDFu-++)5Q!%1R1r1~=wkkFNIK8)?S)v8m=A1&e_;|wM9_&bkaJX!m zYx37YOVA{kdlP8cgW{I0#Lnxm1#I>{PM8YN!2IX!RJ&)EFY-2R@SB>c2bv;1heMDV05HJfP6*D=aWEB--5bw-qb zz4-dIlG8Dmd7KVtX`#B#vm8L@FweOh&hmpt3-4UpV&&GXm<2WGc#AMglDkSL1M2dX z6<5o7MLo~{;OB7vwDWKPS-2ZeIyrFte5jsO|1bbS!@1S0T!;fxdosy}x7#?Gn{9=@ zNr}bdikxO|YP$(5FGmm6uD4ZmdF^GQPyzrLhcWmFN@MyF9`4=H5=(Fvk-N|za}gxV zD0t?Ofpumv&S{uw6cl#=K&?Se*V~qGw6HhTn&iv^ZU81IVy7fX+0L~9P#f<=hT(8H z0e~u|%9&QH)tB8TVZymgiIqeRW><8^EC}JWBy!E`=D5__DLlG#tn+y*wX8}nrcg-{h6X^A8=?8QDuw8 zZX3=mm*+THmYwCI`LFgyBbI03?Y%`9;C|(euh=#`IN*rpwQUlV)6=_!S*|$OFMR<6 zq*xVnqWoLVOPJaj{lEuQ5^Z~gQ0B)e>MNXMbU^XIZ7ws4JM2Ew1 z{YT8>Z=>UWH3!hra?>)YuX~Dz%N1CCt7=lk?mz%QMZIt)xU2h4uvik*9naOb%re5c zzU=I9Fe(JqyxhU-ediku|NMn4Up5Z_q3%d0FplF!xm^;1Wm%Dw?)Uwp*tLC^5$fU` zSx1Bq+6t5Nri1v$1V3y$QA!l6m7oja(P1VQTFkhuSLp=+j_U@1{QP`tt_%PKxBIev z9*+P3K>)_sFw*r6v!j&C-iW;WAi3lH6aWqgj&=TXJw}I55-l77fR1hTc(HH$vo_mY z2()#yuF_YWJmHq|rK5gFcCKyU{$f~FwcL%D7XSe5KEq-y+>~_y5ZK#hKh=?yZ*eN# z;v!eSCmPjG-eTu+3eL;cN~ENE+ZBjl6puu?tVM!5%RH5vyt^&n-k?6cDW6ZM>#k$l zc5#HaI-CpuUVB4WwSbirST8Q>)uZlu+-}+OPL`UT0N^HQf+pxHB@q#x2ja*>l7k_H z2#@Rq`o5p=OTGv&IItf8WWX{I0I-ybsSGShL-U`*`7)R)svuo}e3P;VE0H@i-qUWI zlgjl&LuOpFMmWS550YIry)hQ_YmBP}YY>LbH!K6rff2HEDKTQ_5T?i{N)D?G010ft zf$)sUZU9J1(7(1s2Ce66yjdvC04->oE%xcRi6TnJjZUmav2Y<@@VN^AYU7 zDVIo25!HotAC*e1d|%f(g9Wm^LB*zwlw1cuO3`i;c$ROPqA*k{EKTHVh@$53&KyUU z*9!o8i+27s!)Hyon`J7tuB+R4;;G=IvMaTBC_GXIV-P_bW&Gfg;CbZr3Zp6fg?VxG z@2_iSdydfMbB4UZ0h~=2gdj&iR4Csw`?QTYB~&2tAX=JfH`YbVNV}~9WGJ@0kg+Ih zwhaIv+$nOKuX{`<6M|Pb4DUylBC~p1-F|60O_njjAX)%1jYy% z^21$u1c1a%j82261QRFX>}jlG2{0kRWS&?#2Op(cBHY;ca#%5QIo~q3t~C+ox}jdt zf1(2vC!?f)|3yFm;B}3>I`){sgG}&2R0ae9qL9W{^wb#+>kJ_>gwRVpqPRd-)kGIx z7obah$Vs|rb{=B1|1e!R_(4_y<&8FV7f-;G=WP{Q4TEtJ!R1WzC97q2tjF10wkM8Q7jsjd5#Ezf(f6(Fpw1>eXk{zV>xwB z08ibJ^DB7)01<*X9soiXr8{6!a|{ML@rj8@a-dKqL9mkI0W%VI%1*lx-YWx601O=% zi;2^!j=tPzRyz3~1=iRL6I*2#2&32b2x}H;uO1cniiF$Bd{z^{ylB5*13SXp#eMfM z9VL#mw0D;Qj8LS)mowfnnw}!WvO_9s;ayCy5SxX03n4M~Iys1wn3>72Sv!y{Q{Vzz zk7VY$39+IAt9%hl$RT3oi>bl1Pyr`I+U&j}^IVAIv(v869x&S~w6-EH! z5Y-z9k3=y!=tos&onh1A#HCXS1aX-sgzbVRAcBn?FBAJ!Y!63SSw7-OdUaQp;Nd6q zL3MJjJ(@_0HbE0KL6=U*;u6Z1V@EL{LW-^%o7pNVq+%s@<_-1#n$8l*G$ObwkS4i) z9OnWN&Boo>RmSjDakP+3g08VUKiqG_3pUb{B?y!H9UfO~=qd?Okp}~8H z?phcBz>TqM=5tKSxf3Lr1M`!kf z_ok7z8;$lNxRf#}?sLW&o^-PeXE-wB0W%R&`J9IO;4R7zwO2pSlEEu5tuN4y`7Abr zbq#aR-(HkwSC964Lm_GDBAlO{`8;JFClCT3xMMhR-QI45LL6lfOwyOOi;9Jc!qvXA zda#U!=8?-UjFXyLIGap!y&0PtO354hQl6OWuM0hJ*T`9hm(5O&zAROH7Y>Aboob_g zpsAjljKq@}9;~UQdZEu~>qLy9<29tNaX1}$%ee(T|Jb=t76fXQNBE+G9P`mtcE@ya_sN%rw=}epxUWRfBDN#_R?kiLkOsq`ZP$ zrg0;(iy5l!ZvX(L;yR|qa&v^uuZNEuPrR}%%*Kvp7zXR;&`>}AGLrG#9SadJBky$N zoem_0HLO~Qo_^)(iVN7dRkz6?3h$L_8#b9%P1diCZ*e-9MFd#{M=yYU`kc4JFOR+j z&YibwS0or>tZw)4+Ik?2O%n_QI)?^@rf1hpw;d8tdiziu)~pCIV?Pb~L0<~?OBf0v z%Dh^f`Ojt&g7H;wRxgpAd-Z#WsJNDXoL}(I{E{z&kfw(r$k@8>X=)0nJ{SQT&Gn7p zrLqi!WPUG;BHci2n3&p`GjMBbyt!#Is>=MTamT>8WCYLf>GyY?=swEuMKb*_O+1P4lJU zR&(rfi}xjN|KL3Fr>7+g4p&8zqtKwrMvY zUXF2v`{MvgG?U*HtOgC!|a3Cbw+n3I$K%tyb+zkEW&8?YQlrA@(p_d01TCO*E7M!0I^sX z;L-Y4wRenQpJTydpLwfm!M(>Dlm~9(fRj=eO0$apVzg8+OGYL-gH2rliZk@=XS4W~Js2qQ;gH_Ryq?&ee@d2w(&V-Zf!*u>@K?Q@hIDx2Kf$ zSu|qaWY>y2u^?T^4v@Tsqmo()bn48=%9uY0)LsYx*i#9-g4~m^Ei>>XHPsOfJ#oew zqpcR$0BM;_LQO3KxW>V?QLW#JA()^EnxHWY4gxuXXt`KtMTB;|jq?|Rls-f;%MFh| zJ6P->jMJI~@A5Gk@KKB-2pi~e0UaKo5u=2rH$=zUwp2Dv@c0rnNBaLL2#6!7sU@}B zZFwX7zD4aba$7g=HfK$}^x^eo!Q-2*_GGB4MsL0)rAgDy6Fpyh^9sE0sHy9hqM`Df zdA!L{+ei_DtImc&NG@IInEk#db{93Zvw6Ow0Fa+6{Pnp2AQ1TB=6d}ybg@TdB5ux| z=foHQ*cZRq{9nJozdZXHKu8qEz&42_+gIa&33^uuC(#bnFuMl$TWPhs217hdt%Kxv zUUa|=0#AEn{=S1a&qwE0o7XQlXH-%EkO2pov4aK(28R{>=1JY8nCB=)L#;}N5&O#v z11%y>;M?|fT5L_sn8&zzjGM~HNv3UO2>{5+GJpA33ji1f0)Rw<738vtvh)BSvN&NB zMKd1#D@>Vm`a=Gim!i!rMj{`?waC2|xusF))kt>Xf@&P?1OOK8rjagY^)bnX^X3CT zDtm9&n`SJ=a(tsm$9BXFN0>O05Yri5f<|1wXf$ztqigD_pk|iTfbnRADOzkzj$ho=i3!xhy)y?v^4J9UPhgE zT3a{dn#GC@0OE1va_BsFP5E@Mn44TRKk0Riuy8ysbH`q#x$V7@p8nOBJN*7Jy99bY zth&-X{Pu9dzivNe6i%H*1w|r=c2c-(^zGB8J=)g)FzdF`i^6dp|8rAnC(si(XHI`Z z&+DR#XM)3q0f6TL0G{83j>Z>7W4O25@a!h?_9+yN!JJA4?|GU4;3jB-CMa_P#aV_# z8?xphl0IB#)e<8dNwaNWLPqpExRS(iaQrl4%Z^-xqZ#0i~9Gi!jb5^i3wS# zj+=2m0J!XM$?0z#7F`yM=e8K))#^bIOR$zsHfM^lEPJ=S)cjn2qg{_ z%iz^__Dt-x&7Hh}5kgi4f8-v|19NuVyS)C>_l_-u)*k|Ygj-*EazYHgD~LD;rwlh* z4x@opdXKQkY^D@}c*$>hi-lLkJz;-}m$JRAxPTTHP?v)NfE)NKP~yzsd04*;ooi%o zpUxSk3V~u_*Dp#&Vgq=XC@x&O3|Fj`5H1E%3}iZRvltg+6aa2p6~B-p31fKWb(9f2 zeb)5DJ+h_MNbE{rZgPflS`FSG)aQlwZ4Xcn< z$$-Jr9RSS$K-Z4#^frwsKs+OO9U8J!ziL@yL7qM#H!@DvEaWOgx_Y_VFUTDQw8t1T zG-R=YU;&zWD9~x695FDaZs)LThj0M!;XAu;ye>q@wKmbK7bZtc6gaIs;j|mN@m_I{ z+Pb;(BOfhtxyGDPas);KXgUT{-cKDiC@mZ_l54i?C56k0qX;faXtMd0j(`2)iAW>kEFgw;#v}$g5k2mX*xrQJxlOj<7k8{EUZUTUtpb46w%n`B| zaEtyOd#4X|!%%<-nz9+>|5)00^UKjD7U;^&kCweXq^Q+z#Nzq0I5`!Wb4N z*vH>Sjx*5}U?0EpDpiDE*=goxL2i!1O}`fs-8fR`b`qCkbiGAb3;SE%cmV+F>*7sh@I0_BdtLHTEGS1QbDq33?Z+goF7<0?dssTEvdXFkw;`L}9iE1P-by znrBtE4bz%%Q9VtB5C{tJ12V_)=t7h$1AtCHZtcPxfiHeafB*VRbYJ2Ki^XBf8v{D< zbfkWP=(45@U=#&-|4n)@1j7fNCH6gA<1Ye$Qji`uTSf_PojyI>S7bTS_kXW{=|@^H z%<7t{RYnA7z~D(F(el~IVZG{Xryfh@v=$!IQ&ga5Sezgme1|#8hy|FML^e0O3XEb!rm|(U7)*Cb8M>;z?y;&J`QKRqCcw4xlJIg>gSu^_TF)^^KC&zis*XkvJw zi!FAD#ILGPwh!FB{`RbT$DLz#QmCHOs~a)O1R7&6#YKbeeU@zlac%*5U2n7g$94OYah6thCKsUM;{ z0?86+IdMBfl7+|Hn%IHC%)u(%Y(^>{7$(^WI~}$#=3&vy0Y`C?1bdF~>w9kF__pmY zmF1UUUO0P3)wFB2GK?g$<26zsNG6x+Hmh2iKW0$7u>}qs##Z(lIr$=UsMr!z8iI^!$TKm8!pkcUXtMWX*8uAWh%yU^+NUc z&t~VL5wY1V?!~rm%?2)R% zz4z$=6Q>!#<>%*5jKZ}?&CSgdGdpNNCI(?V^xD??KmGam&Fi@wAKG`oeBZ4E0M<#_ z&nH@Mfe}iMb}{W7i{ApCcl#g|6f>*`-u2KYivhsUXypbQGgLss=|;D%L~os-qX*!gd_J&xsdbzm zxBzS?SS-NwK>*zL{f1430^ijoC>mQyd69l0)%T@L=o&A}5s`f5l+@(G$N7SJW^Lx7 zOi5xluQ384v~aeW<+@yugf>#gvs@I$ov_?0!{#?&)^zH0_yJ%?ygi>-TzDKFK&B1= zf`GEzW9yqFf@g@+fqBSwjKyNe=rMBKjx>SigJ_G{RSp(gCXAx7Hr0(e+eiRV!m-u~~wvdJL_&-KV=Q>!DF9H)m+_8V z-C!+kdT|ba&5Ro(g5ey}u_zQ|q2Q3#DZ)tCSu74hKgEr=yX%_}AxNPl`Fc$|oZuCB$N>oq;jaez%0g66rdoM_TwEY`&ptTI@~YofQBSZixV~5g;L6R~Qe>eK3 zCSm@hp>`E3BQFR^&&op3gNi5~X3HICC$UF8X9QM*i&=Sn?%>p!e8R*j49H}(4TjcU zoVLDYP6QG4%d$~5LxdgIX0CBUpt>^t`kU6v(YVD{GS){Bf_M@iJTWSDQQ(=}$7tDPl$wt&l~y;H!kB2$d%K#POr<(mC{ zrm!%aH8FFRMBb~|Mtz;RW4n6u|7Y(!hJWU^?Ymm_E?JgrOP0HFuV7P5F`;)t z5+IO3^5h{;-V{PYhzSV^Erk+72qC5yn-;);ZETEtmz!kuw$k?9+h^VnlC3VSq*YmI z{y#q8)!v!AXJ*dKoH^$V0syFM9LSoO$zF#2)xh#FXh-J4$%qY=V#|!SPYU#-Cuj)j z8BtRaP$-m<*F;7e63mETsbaU%a1sdq_wjz&}!`V8w9crY2@qpgG5M9B(BLmLhVDcZym5f;4 zGHJQBXtjI$i(1G709hG)eBGi_vKC$-yu3~p`2 z@FY~PS8S$wGA7U(N{A9tD|e|i`?&NOjIo20SS^tc;(#GGSBbfVDMi{rXx*`;C+mG) zz}Kp%(~8Nl4}1B-g4sADl>oqLHN0Q)7*QX_f`SS7@G<7t$?n9cNnP*T>2t%XX@D`g z`G!8)Md0D4iY8zjCWr_Ml;p<;bbJ+Sl6D2rsHzS1+;J|wkmf-b|q^|q;V-yPY+d0T6 z!1B2fWm&DRWoJy4&zU6+&(BE#MkM%4EDqq=>4o7JUvtO`6Bd!jivVD9fx2fOTXJk5 zj%@>TxFQ6ann6{2JUbbk$T9{1MkE|Z>^599X@qx1IIWmHwTbG2+Wl(fE@}5me9l5XYi_46_=%bV{7kUM;;)MXrT`CQMqturM7mwaY942v zBlO{b93Mw=v%1*@F$p8{+hqUYCi2sDu>S~rvKg%A*pPWBL^ix5Tdhcy zVB!*d%Z(%NYev#$fM5kmw~Gxb0t$r^=^oop$}91;SGDR53u23T`za+5-CAtL7kGER z(#sdjMo8?q>rpgVdCm}K7C3cM3kLgl_^&Et*&>~1TX!Rf5H(|d+q6WWxrg9XE@wIt z%!KXxSUDgH^U@ES%)^&bAsmiU#H9}Muq<>llZShZNaB!J@!~sbepYvlj#!j-4v9Ey z@P&8H6ALUX*DeHA4YQLjAJ|*1e@~r|Io#6j8k9|mbn$0$u+Cg**T1;bsfNOfV)R$| zgQ%)LHttiC75L~g=H)k;lVu11kTp@9c^$vv?ql^SXUFVpunhl;9mWA4(fYhLp}Gz^ z>?9>23Pl0Hh7U|>DH=OMO?Mne9+ub^_J#-%fGV7rp{x>JePPJP_pw05VIP*lKp-Va z$x0uNb&1=9aP&Nr3_NaVMsEL@+RiHqGMn&%bCkQk9k}_r(16P!ngRf*pnqZhK0n&N z3oK>`g#i$`WYthzaTW|kvt&sk;Sj#^s;C7T6GB=9*s2O;eGK31JOXPlDgp|H5($$g zId|^T+lQdCfF`ifKN~WnOdf5&ed9aQ;w5}9@u2`v-vD*>5!pUC4kqM~_de-i^I@1{Xf(t}|l2xD+F^UBNNq@aL*t|4tw_}I44p-%}Krg>yqA^z|)MvY$Y z$Vt}=vQ<$UU-O_`ev+s|wfYrhxb`wVC2?rA)22^cbv2^NJW3CckYEQw6mLVj#chqI z62U+XN6~@GkwO{QsMYV?f5p5Bc7)6s^KN^3chAYx($lQpyosGXli}E*o#_$7vPN6@ z^o~&_sGMBH_w1B5YdWY3IaL}_Q7i++lLvc_R9}ndp2KI`YKKz@H8t9;H#i1Uf{hQX zdq&xP2rrw1(h|qufU2*L<_Tn1zTBhrl9Ysr&qwBXT)(&Px!+)x-3_X{ro+&SDNzSQ z#BN0kW>5p%KsBw=&SlJu^cPMey#RwC zBb6j2kas>DLcx}mhE|=QJZW;&A~ZkyB$^gU!t$tNAWmh4c;z+eDJce^M5UWi5l|@O z7v=clYu7$>XzG3Iw|%+(2N@sTb^h(|H1zCDker0pT)=QcaKpE%kx+=d_>Kwyh~?Qy zYY$W<$B;pQ;~+Vy$Elj=D(KP7lj8G|ZC}a&z{ID^8~zYOJ?_fvH)l*nVTnXNbMrW1 z`+jomg^Zym6CYy1k`_rh#`t80<2&Vm=ZdcC^Fgx-c?0C#P4p@&%4lIPQLZB0@1N}! zQjnvWjU%~1oi{^YbFLZ|!D4~D9Q^q<#^oOP)V6D{%#SDK>Gp!8FsUhh7ASzMHmqJi(sfmUO{gaDS!x4_OZA!uFIocV$f zwG~4}xo$C|Wy?7pT`6shB?Fv^6X#Du^QTd-rGaC&Ky<=ox|BlrzPo6W6Xq2|FihKl zGK#SEU{5g_$t{hi-2SMh%&Y#E{!ol1Xn=R5@|8m zQ$e7kX54c5%J6TVag3RtQ<$etl$6?fqlW3qYOq{!UYLEC0D!;SOu z$x42CQlNyb)?}ruwfvkQuK7xmJ5F`-X`#`l76KgjbqU5K_b4N}R9CPVpZLHe>k<|M zfwq017!p*~sOwli)mZyV005kaI774Mt0e%y2ZLc73ivy`q$C=w7^|n;2Yy}B$!0LGXQ?DPA9l>xvXL|8X9yWIeQiMiR`Boh({1aw_< zxm>*7jfcXClbWXKx~40N;_X#*5+^C31|)wt%u5-i=!m=xY$r}D{KuOiif_)Tg!917^2Jy|r98kWNlMvBUP zv_;Cx)T$d;UE{ny5dhdwNCoP@y)2>tGJ?aI9Bpt&|7~U{TWi0Duu3 zVp}5$odjj;2mr!ild7@4GcPGgHd#R!<4`!%NxHfboznGORXOYZW&sFpse*R5j)BR~6A45B;A+A|)jDbHp1?S6&pu_zys#*W z0Y3o5pM6jYDK-Tk{O|tG8*Co!f4@u+001UvBy)FIg4Vg%3edG)-Jm<?(0)QlesxW?^A0W~1^YIE+ zRUsS>`F&wc(*QtLtOgs}Ewogu@?tkFVwTjRuv+$M zX*SUfB-qlG4Q@#iStS_FOzMh&2*dch%W8bRV|*q}vp9Ukws&K5V`s0}wm0@>W81cE z+t$XmZQK6mKIgfg-+8~h^=W3hySk>jtERiB;F#Rc^|b)2#nvj_z7#bCSf`DBv(^WI zRTt6B!!&>tsjOcS?0rT9yiq06G*|al#iT&$j7d7gMD&S%>rad{Zu~Zt*;x<%>4^TF z+D3q<(g;DPEk5tew#z#J_v~~IR`^IU3JPGY6_|xA6B`Zk`9_#Y-kVsO6!6c$ky~Ar@ac&SW zJ%{h}_0=B|BF?~4;};CZAdEXhEWMiA*Ju^~fAX5$^p27JUBnHBDeo0uw$w_`b7 z#F&uJ8Tl{V6M3eFa#BhuzW!L#X!PEL^U%uMZGh(^Rnxi*IF{R)V%ISEHLScLL{fz zHSZdd@IlMfe;1#7MN8XDKnE(mt%fkdWi&B&Pq*clfJ6%53??4{@17g$;QR-qy*#h& zcgVtKR?AVpj+_(=rOddTU;I8VnvE?puX`sD&5i*P#yMQ8R{_Y$!xUVtid2_nMF-yV z#!X<}`NV;GxHfn{7~f5qzeC61#$8Cq$Oco=WPHucAL?Ah-x4oLC$4UIxho)OoK_@}|^ z7yH$*!xAP~C&a;)7kpSJ=Z_4{fqTcPd&#kx4JgS}HyL@bx!`1)d}peKl>^f$VQCHS zr6J>v&iMu))Me5u>NUQ!?g^OgOHWPB)cRGB1&u-wXHHs-iwu}n@R@ziuaTq{~6a@pmOW-@~1dl5pVQpwrzMYjjk1h;Bd@({)ia zg0N}Erpg{wzHkPzE}f0-`ERIv8W9#&FZ?cSYRC5V#z}BAljX?& zu|Zv0WR|EU6|%N@zh&z+Llu$4yLXmNe?lIi+bgD|a->Vz!jN z{irB+Iu2Wtb_J6U?1McJ7+3R1J`;6MPIC}T>!^t({ntm|cwCn)X>lhG!+N*i;IYY6 z(wNx>5#%eqK+lq(P*ZfYAMw!@)0WfRY+UvwB`G_0d9=ds)3vW0PvZ*_y}CXy6Pknt zeRU6!Q9K206i!9>7`_%LnO3%l8JS2&;p2Zyx@-6?iYX+3vNI?wwJY+h4D%`JLk+MX zl0FQhjrQ`q)13>cn7-qfK8mDBts$)ajFFKyAVIx7=D@ms{ZmewE|9lw2AjeKlit>D zZ*e|Hz!Q3qWJoL^9IG0Pu|FIEFlL;2$T65g&h`yLod~{HjaEc*@O(frV6@+H7fzKkntpa_>H{D3FSA*H#Pv&lm0DxvpfsrHnGInYiplQ~7ejTA*p5@= zi^TcSA)3ondY}$ru*)OKVV~CfnI9%ii8hL{1=poRoRPnI@B{!r8WdBJn%?0lni03u zuyGr#_tREeSBfSUv+_g zY@GfYfT5k0_7ge5oPa}!ww@UqPnHxFQX{2+w6$>jdWfI=VIs7?@I(k8EK<8j(9tV) zf3uFk)yrcFl{G+Kfcfn8>9cT59@78DsiNlL0n`Fn zQ>FU!BP0fkiB@cI)db)%Ep&J-15(!aj88SaWOAmxO4o)o?(FN-6bmga)z7MEG`=B* zl;7leU&^*}50oEBA^Vp?z8{E*+*c=5f@P1WAk{r^ZQ_JG3iAj&ZJA1!nPAvS^u?hpg4BHE@D=g}Fi@+f2OO_@tTvvqdX5v=o-C)sXg-mvGRp-rNH*8IOG3wKl#IilVi$*roB9RSe|U zOT#EkO=yB-oAEZW#O#qb$!Nbe)zI833Dz4(M1DJi6V~U3I>u-5a#__HO^gW{Wl5je zHmov+ON$P;G2sSDC-9i~T1=T70|=GeCl=kw4s}-Vz15gfdf{$nvH0)VY+K=FJF4$* zOBZ9Nb13XX3gyGKPGs%eJ@011#xmGTv@Kx)4iX2{aU>%(7+eC8tjiSkbQ2uk^5kE(IbAr!CBTa8#c=xfTITZzm;2@gY5YL#Oypths@md|MS{1(9 z*tLm7E#l{;u_1z#;wBBHnx42|-<^H4D>G#HfZI-BorDxwth4aB2;>^Su(-fE5OwYPv2NB9dS`Y`OJzxrh+ylQ>X6G#% zK${M;0D~p+#7x#5pzHYH%2Hek0Qh?s_RcJu2=!P?9qbJ4i$hQQlm~W>|03I>K9!Zc z#w|~#ad?~^dd^S#^$k#qpT9kQOMEoj&6IE?LT(+ySfGLB7LNpqsf)R!yNB$94$2;9 zO_r_9*X7}e>tExbwZlBStLwTNSszLnbbm#rdv-QtDglDlE!^=y;Ea^x27m{{o>xD% zybocb&0Kw$#fB`Hn7k07`AF-spmSV;0q}g#wi%a@Q`{X~WCX&a9B|Bft9euTU&!Xu z8rQm}sMy9PVTj3))|zbz--GKriy%kqf(AbUrR8S}Ip)bqGJrrx&%TFd4`{Rwqr@?A zL6yVr*!BR-9HVn-`Wv!!9Gte>lz9}U3NaQC4!{~ghs_ZCE@|CK>3qc>^1 z;ivTTQTiu+jy^p-5AH42$t?(IKrp9xkS7o~Vj&g#O(K_X{e&TQUYvF*WD|V0o)!Ii zvi~$_Xh2#*`Hddrk7BZXxq>2WS>yAV^UP&H9|F7{dS;A!w|6o9yzQ9w1PaGNRuozV zn+n7y#5+2zh_lH|q%a^8NBJ->$$0x}!!unI(Z}T-2-K^U0>QwroZZ)S>HSoc_v}rd z6`gCHic!xk_{zayNv=MK$VECyYdSm*ZhZQe%_kw?PWNzdWqBnxFg0MGW> zAF@Gjo*oF0PGyidR}Enmg&i= zI9>w|z=-lQTN|^jcd5E6dq>V1sY=ATsu@g?71Co^G0~;E;co0N)cJ(NDPwSEp~O_D z002UOxDdaRT#pbWVCgV=$G@(W8m*y}8{Nuq@c?N(S`-q~$f?mG*5Cavr1wmQ>+$r* z!+aMIIY4MWTMJRq*(y=MCEBluwbaUksbs3GbzNqlime8*kh=ez3r7`Nx+ zVvv=@`wJ4U=L_`sA#kJ%>u0HR*+GaO*qH-L)P6P+g3NZc`m{n;rd&5KrfvduwR;$WY_l!k z9RBt+)@crp44eiiDc`bD`B?%aO z3@BMqK>EmW_Obk=*AM=tu+&$g$b8NN*1zc4gblFnW5zPD@*uT^yn;3&^ha!eG9Pcx z>UdbcO@|+wcs84#+~i#;Mn9Yhuir(fcu7b*(E4n>)%@hSdm8U=^wD$-?Y@z%I7o8m zl~J{a*X-1HJmXI>YbIMX>j7_n1;Bt$hbSC077S){=Gs0Gh|pji3)C+%!M8kfuA$!v0}InHtWrj@PSqS(X#VycF@=?pYz(sLTca-kaCs` z?)+>Nd=+FT&$`)p&c(x0id9|h%ZA=jae;r`8ULDecrNT3RCXFZjco&Y)3B>V zo1`L%Bo84R3F;kH@#-vtYMSK=-wn&}9Bq5qS>j^6VWIhrFjSP28MTtqP|8uXA+zDC z`KNpRix#_{zp`S|CP*$+{QNv@1FMYKhEz7~;|LK{r+!&|s*6BD*O*gU8>c@l~oC_*I+C%+dCd3iO94YiO z3oyv90J`l`ZwicyV=bPkVU4hsSM#M9_=+ACn<^BG8tBlQf*S?s)&~tUt9LTXEfc!5 zo?ErlwSB8Ms^QRO{Rw-XnH1smPB}<8G#)wu-g+ASOM|oA?)r?%beze4krhl;{l!#+ z3l)5%Iyw68N5haI^!G||C>n8s>4s(9wa5-A>qF!Ou3S9A3b-!!Ed|Er3GkvpZmx3S zQKt7gm_?mO445G-=RNIDySm`ghaC%fR95Ub{$$8R zE$sNJILLe~y(^gIjMPCGDa@&Mu9#YnM{pfo?;Qzyb4=#O)O(smXUqvibg3gzQ{F+J z-a@3+Nkg15yZnNUdZqW?C`MuxpU_8wPvx~^}FJ{|@yC+#Y{uBP!69>(e4ZqslC zuIbtTJc-*pGUvGO6bw2Vu0OP4eLPQndmASBY@Qcg{|}t>O3l ze$!{*3gaMM-|aqgmRNsTZ5eR&N`~fplS*)J4aS=(nq4cZ#~~ycU4fuO+|5-`79%9s zk&TN`H9rz(KE**SmW~mbYbHDG5^Z!`B*zcTi8cn~xAqvL))34JhyzZ+;DWm20k0$4%GJDOQDJhz?g-l3?&*bPxU&Q zLY!0lZ&C2lhsGH3#z*|&($u_~c%RYM(AoSu1%ki0?!`+BoHf3laqI#mf*VrXtnt_K z7N`5>eXUc%e~9mg4~ioCm*V=C`tmcm%zTQhtX>c34dPATC4$S&^e3>0MsL|Sl^=6P zEVoW&h;$4k7nkT~;O+86%NWo7TH}YfefC{YsQDoVmdo!2{$V;Y`3xTU(5UYPJzdXe zh8I##IHR%Os}SGcfSV%)sFv?T+e{_*MG@^V#nw!<8i6$ueSfoiT?gC5+yzqjZ@>TsaQu-8bNm52EIJ@Tn4Q&z$@i_%l zQKl3;5QT3(-?OKK8HliG7dOAz%hUPy7Pls5{IpE=JitH03dux`us}xj879dn5{Qf8 z`5Z`LIfRxTwrF^L0`c&ordGZ zjd}_>xCgo49$L`>w}w6`CrK2*`%cS@I3nvTD8pY5$GhALc;}^h2m@e-3#X`Tz1*j( z#7HWVvm0<0q-_12Oh`;nutBl;eMDE#uFTx?RMC$A&*|k~a1Ory zuU~rc+k)~jyxmKdEY)U8cBVRVsMmAqiE1v}2$yq31vnJ)U|M(BTif5s{iQo!qP#j> z{x0Jn)R4CUJ|;E|&+0lqqaGJ8HXDsrRds;3ekNHOuAgv7+DngkZSC@WF)9+rP-52P zsSu}odzdXmiUawi_W2uC{ICh6Msh-#t7_;(R6fbl28xIU7)MLu#=~#;PFOvJT&})m zrUH(cQamKlHPUMS9Naalfu(O;jD*#+08<_q7&X#b5IN9q8X4qye-@h2ZeQqeXI2v) zfK#~LAbfj_nf}a2ApwGqy+u;q=dZ(=Bri91OJIXFmA(GqM3(}Px*XZ?Xft9#6CqQ2 z*5e=_fMEV^n0Kk7ebDro^DT&0eu%FmH%5;R!hG&pLPbU~=njeTr0eGAc`!;9f`yA> z(5TwPtu+P*ur|TmcCm`Gwv8&+7l(ur!4Hfh!O`*AcC^bii|!vcTc*xg09}+|Dp&lG zN@L&zkhH`k-?LE9Qp8ouNu8Dy)r|8x#NF1MR{e06DI5l7P7awB^5Cv-$(BNmhm4i- z2hl&Up>}&Bh}5MyJC0lQbt>_>BuR0`Oy7K+sq=36zWKwXsiS2vET~aI!V25&U|*o$ z=L8c&L|{MYdvZR^wY`NP4tYC}!!fodd9a*9k|DU3$@+Tl;y#_;`Cl3u!Sj#yK#?a`dS?5DCM~k(-Bh$myv^se>x^8m%gDHoy!rN^AhyQJP_b8s zuCL5-_aY#*@>ce-e^{(huONLpr_w|<{kY00dCShTU~DQD?d1mGt&Z+PNDfi zxmM`p)cF=Zs#Fj4D~tAAtzvaSb21$7n8;KS0^~gmcoPc)bUDMp_VasfRKQ}o27uh( zI>{~$wB6GYW`QC&p75X(qxf_sF-6Fg{*)9U-zq>4rI0#aE~07Uzw>VjlX8dJhox}PQemZ`s&^fwEfOUP3)fMt)pX1e zUR=bHCWY5ROh%FgepzFtL-C;L&BVeuY2o_NH!eD-;VLjV5&Df5yIdUwjTWR6*&%LX z0U!kuZq7cfiw#`M?biyL^U-B$MFgM=k<+)TN3iGA>&IlCeqlCS@;P_i4FX?6JafnH1p5KmQwpJQ_D~Ze zJRdmei~`-5nN>pF5GO(al?_9hsyVtdEa)H`AXbgh`xZRQDs#SV4+{nhcI?QBm4~c7 zSSrDNE5{%wZIb+!(FG*px#_d}ywpZYV!GXV&1!Z_#c{ZA5_t%Cd_QP(i2AY=bpo%U zYH^;A`h0rVXXd+UPY|~H+?}|5&U#&>^Q(xTnVk2rJC?CZrs`0Lr!b8$x-CK{n$y(b>}o^W^hcC;N&3N=w+sU_s^AbeiiZ7!;O&N#(<;)} zF9S=cGq@=M4GvcCg%I6p-18kBo=V9pLdk@@712Ubs9~|h?V-AY||$m zaA2@M-sZ8{S*5Lrfu#9V%m;;}ve6{eO7Y7ESQD?k1W6W|i@ntyUX(c8meNwdv;VcR&%MY?~ z02$Y$M?M&T^&;i;j&2cQM&dJ$Y!YyRaxxq%zLNmb4JPw)HWfYyhI&r|BH#1wiPIe@ z7S-TY!i7PhvJAmyCmU1D)t>(0tZ%SjY$V{jTegU*Xi_wNXm*40#x>DHYY%wmdwc|o zFr41+z0ZHK=*$qjf6AZ-Ono0>oVemtC zqXnr(u#VptfgJj&A(l>SyEe0Pw>~da`;G|XdZJe^Az_sdA7JVtWvd_w{oe!$M<>B0x7~@h6go#Bt0(O3a z-W(btMrteU{@jUJQav5znhSJG%L|jzQm)=h4y&mM&2UMN#-;&rlx?E4Ie~@F1m^?adu2IqO z)Z>?q{{Ca`9DqvaOZ0RmDP($+gox}zK-*KMwnbG5|zd}q4zpLJ!qdAE>0bR`c zBViX>QWYm2#J&Rm%Y${z6(y50=dp_ck>mb5!Bpzf}=_lv#Pl4Oz?=voz5U4BoQ!d1YKlk0HJ@2Jg=TLt{(Tkwl1e`fc(b_ zq~IoX6o2<%q_L3p8Yl;%b0+?iygpbfOQ9Skqtq_tX!>_}692F5(@}&5`Uqve^ql;Y z%D$QdQp)#$v^!HoXq0pRov4sNO+olo@iGt@1vv)zI#`UAEQ7FK(dQRea%K;XGeu>g zR!`E4OPawGCS-EZ@hBX)Ss$lzRz+pH%_X5%`~arX97qn>fEla$5iG0rohZ@O{T{-Z zV+5p)M&oPFuph1aNmd!KNTXCM+o*GAB4)!i9yIBcGl;C9y3%GVuT%-W&IV~0+c13* zwH?<2TiV&|J<-?~Mq9usuz)G)-!iBM$$c6~BbmvjZRqCH!5;Q_3bFTVzY{`znZ7yS zYroc08Bh$lT-?d2`M-#`enS11ph<}OuD$cYPZuroN0bLkuOn_G&lLOK)~=6>#Ig0$ zKcDx;lMHJ8_TjOpVvZue2i-bfAX7BN1XY4CR2T~A+6L^LX)yYo1;Jtw2Ar|AU(xX8 z{vr<|PWu0er6KY*T;Owq2oS)@D*l$rYZWWwU!@m#F<&2lYchr@graM$txiWDhj8i8!?nWR)*|+|^1w z2*ORkp2pJAkC)F#HpB(En6?))O7%fyBUDpQ`xjW?w(4aYTAV;Sz*bBPRw8cj5Iu0r z)R}d19ZTudlF?d(B0?z85_9AhEHp3H#egj*fr|01fSn_e<7~z5(@~^J4R1>k^+L5) z9E6CZTe8{iB#@4jcgF0GA9o;=@Zy5l_l?TDl8QXI|IvG%W85z6ybmW9RlwJdw})22 z%BL6fly>t{1Q384>?EDOWEVBT&U}17=&F-Eg2-{8^A=nW&sf3>C_@FbLhL|*e+8yF z6MkJf;r1!fGK0WruVUYo^vtfbBR1dmDyYLNOMV7aMqFMN`PT$vGN4XP*xfoPn>(c^ zNYhJe8T^gO`jSx*E|$b6piu9jZQ1>sTo37(QKKFI?)Tp@3Kqy}0&H;GQRSj4HRf`0 zB6N)RL(8^BS+G?Dutc)MA2*`wu=$`RBrErQOHmgRx{N5TzZz8nP^*H1D|eB5_g*eZ z2SfL0X!`D?c3BsVrev_k%`!s01#*PonaGW0#rh%*3Lj353R<{yrNmhp&xT{7qm2_} z*a8`ogRJ44?)y^*gJ_LiDFt)j*(FW#%9Y5;d{Vb5dxKC2W3tiEc*O6;0ojA2ZZ#}% zl?GJ3L#W7+yVZYZ0z}6YH;2@vR`w6wcFyh=Xki1G%BKS0z9tl@T7r^cHO{2kDz@La zcYHNhJ&jUZ#FAy+9~AdJY~ne2x8~wf1sFjAO`*v?G`UD_jU4} zq3;2R1M4^F2B8}Pd#w}d5bTOv5nftv@8Xm1BunM4G6&dQ;?Lbgr9@Qm@=yp{xpI`* zna+j}djuEwTMp^)R(m11(G&UBJ+^$r?J)#VVRO=&DR7CivvYcXcV`uM1(XocN1XF= z1Apj1^{cG^ge#9U#qG6P?3kuDkGA%9)U`@J`W6jIjPwbIOvvj0>q-Jg*!`~LdIrw# zpO(eGWhEs!3fOT|i3LKz2OGC*9$l+gtq$E}&@k$bEh}_r3?u+Kj|e>j^HP`eKhZM^ zNB3yG{)B{RvQFIbVllyXjZel_ZH@vMhbV(0r+#(fv^9u(+ao>DHc zo%bO&Ckj@~=d*rR>cXGpx22mUN+U?KJ0P9x(R12 zzS&~#POwIDo3$P}T9bBXa>@NAOfI3M>`?qN@4geT4*%<< zDdXRjcO{n`*Ca4)%W!wMD6ruucF%rn3k+``SpbxvtlBUs-hr5|0d&9P!H|ej^7vEO ziRezOfW+}zNBS#f0w~ku9;IN7hKRsvE<}icDWUqs^(oBCj#mBDKMx~;_OaP=1Iz;@ zkCr^{ds7DvvygTu=(}21y2sl{1fTPoBcWD5Bymkvj1H9AALY8d>WKYB zNK~LeYRSw78&9N_mC)nGs*%~ulw7-U0L?_VL?h?(sCt5VWO!FUP$xl;*OMenYQ_-c z%rYS>!YLgE?3^R8HJjjuL+mH=Gk}n-`rI6Gak*Z(AW*KSI7+jgV3Kq&EpkOJC^h4JXpq zO!KL^7OZr;RU`BlC~(SA8UP5a>v}GamH|nS;R9RLqRzVeP|z|IQbZkIzxTSazuXc^ za4F;U_!9Zi2aojAcX4|7(vnfj-jn_j@H;ee&v7{J1^c%l4f}Vi5103+L`iuND(a)G zGDik*)lEkRi|NheHWo@VF=`Bs^ z7oODiwsj7#+z|Cz3__7)gzYQu^XM#UQDW<`jDRJ~VFZ+)7j!a4kzf?u7AF&W#ulqUIEjre2-dFI-JAkWT7*&P7nBH}Dh z1^dhRu@vR>2Kk@hfC!k7&g^-f$*ZuaQXjLL2R2aT?y#+4*_9;P73nGhuHTgMeOhH? z({-CV%d^~1>Up8j+|>{OP@jC`M**VlTm`e&QPzwu^~^+$hInThVbSZ;kMWF;Qj$Py z)b-4ZeKaQ0B%HWkNO)mF_8nOB^sRWNwXByGeVJ3ltvw97^*z30y)cy`UW$A7DsYkF zv7>bCj-M(v%o}iL$V1(A5A{|J&@r?JT);;RD~F;<7$ULj`@9PdMHd%g0=i_3E=H}v zT}lEmF`IPWYIQ|TBzbrczb%R1RSU2jZN5u+MB^db%%2Cwf&zrilNAsonDP_92INW{ znp5Hl6^fSXFsd|5Pv5T}-b)uu8U6kCBRXKd)?8HV)(r}F-X_Z6`v?df9_m|fC6RUG zhiGb$aki?xgu&9E&v1Rm{W{Ko|Ah!Qmbi3%7d`o?^LEzb{WhQ(G@G z#-*{n(Z>4~BC2B8k_PD%1LLo&%Iakt+BCQjcRmaPhC%zyfPWm`|FqeXs zR`#Fa$c;E7G#cEQ({L@n&Muc&lDD!4)L7jeQLR|u?qL>W;QYM(s?Z+Y3}-4;M{y}E zlqVaclV(}~)y;28!{=@*ykXw?C}*fngiS2o(xH?TO%<%>@u}#`NR=uDeT1yCPfoox zo*^qRFFc&ZZjY+2wlw&!9B5O4mkkqNd9XN;k+RtAQDY{=9A^D``V`icRXc)7E_9gz z)znL@=X=A5t2iVKEM7FRMkyLN1Nigv^C>AQN{HtW`fO>6ii$yoAo*Xfis9noVxa8m z&@lh{c$Vog@s_`b9vHF`2x4$%{(Gs(?J#`VZgz&*l>SR^{6~$Tj|B4$Jof)<3!tc? zk{A=C#~}3Vh<5@anj`l@NcR&B5Q)pR_2`m^9X7%czK$YVVfAn8FGY!SD+h-AwZTlQ zzZ=73{<``~WJwg8IH9ZQBC>%7tE%~I6{%D4TOrOB)&Iek zBNqto3U~4Z_Uia{QR%RXTIF^%9Q0>o>GrkFt8Lz|MD4$n|BsRmg#6~qb`Tbr|2O7QMxaT_}tTg;gFe<=FWw9rRAN4d(K_Pwz*S)2u5ky_qLtY|gt z@9hd;0YhZ6<=VEhFXjmGFBM+TJq}oO+V#K35qMo)HS=Hrd^`l3Ehi674;{DN5m?%v zcj6G%*?ad;f7{w8c^x90^l_4>fl*u;DJP_) z`bb4wTK0-Lojf}~oiMqiI0u$F>iSLk-vadMxAGZ^+`}`-NMVr001O=g%%TP2m9HWkvkg(?N>fTjLa; zC!}_-zUHg#W5@eD$;XXb+cjz4kJ+fWUZ_p?vl=)2>T<8Uzcv>q1lXj2qe{_x(9#&c^`{L~9<0jX*e4qdTpiyRl9R={?X{7<_ z+3NZBWzfW22l%qyHi1N*)NU&bYyNTdP_VgD|2BP}B}47?PIjc&iMENSo|II1@I z4j*Vz63i7Q0DPFE<6XuW(G-Nzz(NrZ@^acv4I1ZleaYD5e*C=Fbke?=Jtp(|WmrJW zMMGTd`Fs%OCGRjhO91?t^DzH<(R|n1mzTAXG8)1D?Cg#y`B{8eR2X6W?8Uzh~LkPVio>rjdIVDT3mUd_&*aNx2CuZ zDr^lbv&ys4_V|&#CI4WYSIgm9_CllK#;d8hU?A#Jqy5CEs-tr9Lc8{A0-+(7cC?mO z<;%V!VD!@}O*HX@-P3L&J?%pgKha1XF~jwvl(WpUqjy~)-|4Xx?tu3Fnl0*V!+mn6 zf77i?Cf-Q>CVi#R%&kPV-0O29fab%hPBPX=Ju$WOrT5YECM&MNA++>Y zi{7-uX9=`ZsTgM2uQsEelOV@eLv{1Fz088cclS2UhjY&(-~C7THOdC1%hkuB0Jtx4 zj-ZSJlaQ2jdkRx~eW1Hlsaid?cz4>G%KDM%wC+Xy*WeeSgM)*i=@d&veSN&#i>UKq z)R%4LHV*F+2Jp5<<7LTymUD9P=qE1~!LtJh^YsT9Dv>|(+4GeK6!T>ZwRDx2OMq^h zy8$z~Kl0@uJ68+Dq_UZkkR|cSu^n~pL~gVMzBY0a*EHxe5^mxV-zg>W=HO!g>O#9l z6`DZsM)UT=eiE;zQZBK1QCMkK)*$&tBPSm;5o!JI>+xAQg>2cb3~(E7QxXmHBd^s+ z?U{Z3$nXHDS%C5-w&*DH^B$qWUjr=6QfPmVOqt7APz*(un<;|!p8}vT`fO`!eZ5j_ z?q}=I)M{xoRrH?gbBvD@ZKDAfiytYzQJ19;s_qVQ?6S(0m=CZxHSkS#3Ma&4 zWU_8d=`g34XHmbR=@fjJ`Z2hOdzSgkvFJ^cl7IpjFxH&%cJTIvpMnj zjzqv&@3LNxSR^Fj5i**bVR@RHHis64PE>!RO>m?~X3&#*dkYT$c>m*EDL+M`f89qB zG&ELr$cX~*_LxXGv=;qvqm>e#BhSqQ77?FN!agtQ{Bcuz&oa3jJ3Qt)<(i7LTtiLw z+zmB^1bBvuH4e{edaX;(srF-=&AJ$jSIhzD# z$de@i{yPFf4i1}OmQu$jN24acsZRJqii@Vdo@Ei|wzkTmL~v^3A=pXDdhs>pCZGwt zKH~B=^CmM}*xx27srTSU~6d z;kHDwS4SSZ%Ldi=vtGyyQ*i7`)gbBe5p&J)b81j>o1Rpj$waz@jl4?&hRSC9P#gf- zUvi!IqI;|Qn}vah%b!k)kqi@QTEUa`qd%I~sKw?tvDpgqQ%&EfjFR~e>V5y_<_@PM zK4e4%Hw>l~%<%q7wlH!&>@#xeu%6n?RmVQ$Y+>rOZe=k?c4qXR)(zZeS!E#IG!c(WlO>`(GJ($c>Kr+;7|*;dS5k3XQPOc3__} zM`TYyP5p2`BciXK9=K~FGWZjV4*HEFjYmw9bPA0> zsNmXOqz2Gps2^rp`!%ypRsQT8GaKAb%v&zg_&6ygyh6)GIKj-dg+J= z<&Vi(3kkWKFTI<(JOt4&lq`>tKdc?w1>u{vveQ?H_njnG&xd$@>pA+4^ADH{ouwZ; zQ7S0Ug=y*U>W{WJU(wzsVAQHEX4*NjT2A3-Tg&zFL?n))-mTE`*l!QTM*hX~Sbrf1 z-wqbjJ!4$(+TOa+nO9c+yUyBS&Zbj#)aq{TU@4PHZo+cUD{DMt{>_vLhsqX_=mkA@ zjaM{S?Blm`mbzIldt_zyaCVN-;uKTt%w9;4NR`RYBm&L`qvZYN9E6v%Fcl&poq1xL z6r$6@B#x!3cBOL7!dJ&xsJL~ z9R46wP~OddmcU~8L~>OTjdZf+R86o79hKc8=jKkUdL`kKr=q8FsV-_a+=|yA=D|bg z|HOiNIg|<(zAWtr7ivvtV_m?h)m`Cg)UnK4g0pzL#9=`y&K5-ftfEU^JUv6-Vs{a_ zuF<9m&4l{Fi7QJ?tluv+RO3w&lDf;S65Za0=gj%5E3u$4Cc$Lg2+jB<0e`U}meg=_ zEzT?fbJ^UYD3vMa)#;3@v`Py9F z{TuxPY8~|SA@cqcr+|`{SZFlsyKx~rH+NwyIo0%3?N^L)WSC%y_XdjYL3x2a72(rA zA;!0eLXEG+>$$5_O4B|bC(@y(J6@)y+PlqVAbEq%&MRxH=Vny2zSl%EwwZ3b)7e-_ zi;*{G3(MQr)4-LlL4%8FK=yE$AUdh!)5(wwcD2Ct>3r#`X9|1t`tojSLM~~$(X-Lc z&Ko5c^6yW0}BI=ua)KtM}N zqjJhJ6LDIt%gcZ_R&r|44G7^(uTI4J_oHnZbp(HR>+F|Df-}}>SkrOl9cS~ZnVG+P z;ht3J8CwyCIa9noIXCe9U=ki(kq8nF1_p_}A41C;Q_l4MSc~%7JIx2TTS5BHsBsIjv*V;b^OHkn>_NZN6}n{NsHN6v_1zXXd;NQw02 zgdi%aR?L3R1vX}dhTy;Dp>)(9_f7%KMf44&Qa^1RP6p+^v#EO&3@;vY+BM~jt_2zo zpJTJYHcuU$f@jIJAkMNoKD{};a;1D)sTp+VYJ=4p2eL@A;5|Cc+I-4@X|Sa<*9`Z` zdb#3T@d*j3I|&I1&`esY{``E8$+6pf{=5@b*xgjiu+1o}IQQq{%gGB}nQs_29~>Vn zQduA+20rg+G=ZMp6n?&N?&>2U;9CzZTpkQxPs~*P0Am?tDeZ=bZI4?j2m#8U#2Gv) zMb?d&5ec<;MIw9TYWIE~_?3)AwIo<(X#Tlf^3Zi|s9k}cPLlMijTpJM`_UQSP+ThB zM**qh)~msl9E_YR>-?)^2RI%kDa12*%~P}&s^R{Rl&q_;v_0E*#zCT7hsGOn(xB9R z2?ElW_F5D0@8o*aPy&BB`9#giosvIf6f)P0Gye8+q9xGRkN+hi2EkJBqnH7mTm)#= z003`7ce5*4?}i^2Jd*}vbE?Qw2t@D0iHr7cqY=~+{Ykn9?!sBG1;V;fkfK? zC(4~!R}tlJbB&CDcyj8RhqL9=oWiNUx7IrGxu(5Wj6wXP0p@4qv08b}H3C0X!pd=} zHHuBzz~7$`S{)P>1EU`czCZy774>g#l$@|t2~jP!C(ZdSh1QKf0sy94U63s1rb*%}Qd1mSrqq!C~ z)c!}O$OZezT>eLCa+H4)l~I>v;Uq-qUJ7&iwYAUG^ZXGRT^;X5&Rk;#h=e{7W2i3< zPChEx6avK-TSYF7R5!4CR0eL{U&^{+D#{Y(48(MG4}tcxe)&-uOfM(6g7H3`T=WfJ zf^~s#E_+&b%MSGVCHme=q_CR$#TIPhyLp#V;Vvs3JXF-!%*@ZAQA0yR-MR5B3S#2F zGc%|*Ph5l=8XCsN#t=0&2@)9i_Mf~?B8$bl|wJZK#kn@iA{!bqO041MT_kS^? z6OwNITMGai%|J{HD*r^1WB6|cSg@Rc0EpbH!;tWBpLgbN}`NC*-f0)a5NyIX+4gS!WJcPE0o2M_MUitIwnhWXUk- z27)9rT#25*dFXZ8?>8Nc|I=c$@`M<%kHvIwx$NPLqty-%@0xV@x8UGduou@+!=cui zbp;x#<+R1Z5M-zCLg(`#A{@)YaITY?4aXLS6K_`!MRQ6f5IdZ8zgz zk7_kr47;cBA(sI?jz{T--O03%&yUCPTGNqT{#Xo$L1DgvYpYGgl0>f(_?nxSRAy*r z8QtHSs@YKet&(tk-~8>r1A}`hVuKJV-_|i@)$JSbP?%?x6keEKZ?^O+LD(XB z)r5cbz;C%2ycCgY%CRZOq~%PL0S>~cA^tQ`7Jx{VIb+G;ajhjJAGYWeG};o=3&X-k z>UuAh_Vk*-4;8r7`QkJCT)u4@$nXCx+s4ovsf{eQH$rka2Z124qH zYzi{4wAd~5grg{`O_)ycNfJuQF>6HNMZ-UR{E~dN>ji0d$*8QYNR20@HM4u?1LiwZ z(?Lv1g+4~v)SDY76)ym{saWN0Bi>R13_G;cm%G#{0C}GmJVtA8cD}8)^zw)We#1C4 z_m*y}Hq?ESBdNS522%z6lU|DGBFJGtHtM>w^SV6b4~fw3une z$e?-L3vTaE&0N1sHfn7QM-e+8=$+d8^}2fhONAIrzI3I*qZ78*12-3Q#tcjqeT!LZ zU~*6+$2h-;zmg|g;r4LkWdhNo{^{m`+W|@baLRk55cd-q$tc zt&NQ`mq_vgOb!-HV`U+3ixA~^JF`FJ=rTWE6Jmazb?t?MRrNx*Mh~)KeMw#g^ zb&h}uO2zE#UQ(G-W;Xty zYFG`iIYMzve>w_MA8!5-=V*GhBfgW07p!DEax`V~{N&cq@Il_DdhOIB$CJHe{|oKs zBtJG6f)efQTvu;<@*F!DcQQ6E#AvdbN&6l{8&V|dAowU=7#Lf( z!1Z(Qqen8;=&4~$PvbC~P58C8h?F$UmvFFVO2xrG{TcawHDXtBoOHI~?hY1q)r;g@ z?>B}he~z^86D6f)xn{YtRn4yeBis+I&s&4ny1sF4KT_4J-qrkeX`YHFxlV66v{V?I z)_t3_Buz?rc{js!c(4WT!aD)Xy6vx%@^^f>JO4FCMWsH>TUfZ5Q;+fLKgBMudZpRM$_((N?f+@oX-N@lerK#jTI=( zrK}4CmHVoh4s^4Wdk%E`VanL=C|3;>p*X-_89yM?B7Yqb-$zf zv$WW``j4LQ^PAIgi7PTsH*c%bcf+e}?|9jg2bwpJEF7HFBTv>c^G%e>Z#whHQt>y( zW&RYURuJ36m;$_v`sGXbj{h6-zt1~fi2MH;1RVB9QW3alCTwIe9i6n#l$+dW|!3hSEcw0NUF#cXD!yk&HiG z>?bRPgW`ZM9UL59y?TZ46>gsEpS=F7=QDaA-{Irq_xAPS^p*+yC76s!o3JJ;1fS>r zTVDqY%rqA_s6wmSZbty)?9k9%kH}5tn}4_e8hT?FQ*hIA%>O~0^8UfI_HRvaaK3Ea z0|S!4^{5}Qf2N)q1n)Yze`dzE57&6W@@1SX}}-(uM>=ghnEC1Oj=mbVB=ne{y4NK{9z}`eh@qJMYo9ox`olN&_v3=X| zJ|#zPj~g(5R3>6b$?)$%_uhXPKQe~k9;WtqI!ir>;z1l(>d#VdNY{8Sob-;1x|$gQ zZ4J+Wd3)kjYHsPId(ZE8?EMh zc%-D_2{ky84OOQ%Uw_O7L$|ogz577NucCOXDEu{q8I}8RD(9}1U8F3gvytd-4*CRbZ z7u@{WV^>_)tFH7*^oVkm6cfojyDeC(bqzn#Juq1CIf_b8hPHUz5So3*t9FC>fC}{n z-+YnlH{d#ewprzV{{hp38oEGy`G7tR)(5&GKw$i{xo-if`a%8P^D!LUKXC!d7dU<^ zehD5sxjECzfo0H<->v7WReUg- z0-)NxnL{{p(u7U)EKE7xK)buptlq(?8qs%$5gMg(4uO;}*m7q)OUOLt-pc5^X zSAS)0n0!=P;y6Wnf*6v7=;9HjXs5TQF{N&v2rfUHVx5WbYxBZRwGPP{o;BKs5Bh4_ z52<}hOb^aW=@6bZrIS*6$Q@pD$!(=iU^PY&C6Y%G4Nqk6134Id^4ICB%Vd_IPW09i%>j-m zb#F#}+f+f@K>$oYcyVQi?18>FW>9?A1WX4}MPUK+N*0aFy9)RPUHKw{Gcp>lv&5V> z!7K*Agdhf@t&gMty_ok;AMNLxi{kR2FWj|ueNQ|hVlS5WlMzibs0wa$gU~BE6vPF% z+l;mczJkrlQ};}-Rz-z@x#7H=P0%Gmaa7NnyAmCi=|!*phBAB>R*Ek!nXD;1nfZH{ zoVgs7rFVP*WZAVXZ^g@IiR%YiT3=z1#cNr;%iDEv`#cKIlzVZf#&?`y7bLZGxYcn-Pw5MH{F7f>cHoCn;9^r&0ZG8;?Dhtfi0UbP658d|hMb=s_vIM>S?O8v1eKOGcZT!1xN%>TY-t5|9Dv7wY zMs@i2@1}`S33fMN9oN}xNU=%V2Nw-McFM^}%B-HC`_bdey5eE0FIT4wgC2n~=3*Af z`zeyOB6jEd7C?TB!vZb!g+>=I5yNb+rq~HXO8ryy0%uu5EhoCqGb=hlx4BeFs|V`T zCGH;Q7LIhv0)Z2H%aP=%GdIaxQ{JG>oQ4T$=aZ}|uf-~Dk`&H|*msx*3G@$lw?pc+ zyLNTwZT3R#C3UvtyBQ7C4kV29TJCxcaZuXRViLPoR?V{|{0;rw&=H%GDB4Vlpo^!h zj~g3IjC!8Swfdc*878dqbJK14chTh~&+QYgiil4{@|Q!2FPdjeJ_H9-^Qa}^93)ah zY?^iq++JD#wcb4bt3~~aluOGs`;z9odav&??L%gF(QaClO2ba8HJ2Gqh9Ma~G#R2A zzJepg?Nm%VgM$RB9G9({&x@R+mqcJ1o{aWKayFFIJERhAkAdc%#mBaTpX4hVd=oX3 z@o}dr)PL(G{JX!Y=k*8aKJ`O0?WcFce8g_slix=2o6@r1P{%l11)XMJp874TKK{@< zcGUaQRsX%oMpP&JY42tMFVWilXUZ<2?_G(}S1`(A_`y93jAsH75R-fbZI4FMp$|Ll z;$2t-DW~&(UnWNdiYU_DzXs!36)dRCH`?qzpp$4d*p`tEc$(2O7w;L<`*|=a$pNd! zhDZ%}<%?Jzj5>aHN_X{;@WilHUB8!dW>ElXiFPZepkOg0WiB!@wD?)_1g zz*I1)L~5Deg7ryGiDy-@jE|Kl$rLVN?3tT>m64D?BmR}x=seJ^&8 z8iabd=(p7=EwUZ=wu`32Iv2Dw@|B0_Ono)7$@a%M*!$WJ6NCO634he1-c*tRe9#FYTOqCEyH>GpBmtE_sOo{c0z z*2nwGG{xj;0oQbTsjN|=|HM~dEI^%>4P7@m<96qFq%4G@{5b7uzBSEDFFT&tS0G$^ z`iUPU795VdKgXx;#*X2~#(jY#5_7&YO1F~#yMrXt$YqFQe+smJNmkC2vEw$_bK;2N zo|Q}a4m7LXGHpw82BVD&v!TW2W>+0T09AJQT3(rz+bTvDIphBLXpxWhtdon2)x6(K z``Q+`fHsQ1t+ON|s6_kn1dDTZh~dRW;aiy?7fYSRI$=Jh4%p4g{1^^WQIL^?)qG(F z_1WeTekxm!=-wEBx05{h8sj2JIh7;xQbGIx&0ii}h!b2AAz>5cD*7m&c4IT`(C5Tc zl~ZyAdl1juP6Q{?hr?3n|}&^Y|X}_AY5B2G`Mkv~%WK2X?1x zEkx6UM(wV^d6i(jLF9IV7%h47$STQ!iSBh9Mn>dnT_)opk$tA>4d3(hS;MAp!aO&J zQ>Q~wj68`C#uY24Xp3V~Aa4WSy)Czw(Cu?F2V3F_m-)=RVk$@Za+CP)D$Pq*{kO=? z^fqmql+}v<_H$~)(_3ZA2dh;yO znAX4OFr#y%<)$5a*0nNIFMJIGtq~$WFZR20*jObP_t>>btN&iM6(tUpHk&7GHE#-F|+bIl;n#Y5p>HJ1;dYyyC^eVSZ6GqTUp%U%;oi&DpOB>MjGc>E343M9O-oJrB=(PlVoUV0yKv#3MJURXTd{2n# zob2yFJ@uw;aN<(xkFQ2N*S6T4Ts)rFZ^UY#kd;DhMwaLx+sj4$dIP@-Pg8*B`FRZ< zVgE>nwUGTqC};uplAz0NB%IgjQJ3e}y`*u`IADw#XNb9@JZoH@{>&==I|jF!QAiyo z3JN}$vT|QLb&4IQ@6 z>6n6bwpW4L;y^hmJIg zUkAIhn)A3gHh{E%svvxz+j}DS3zwGX`i-NTZmOjvPw90QNQEs&E(%X%3MS9-C*yC~ z*>VXPSSeh#ii(*O)kci}t>N{@sr-Cj(>DL%Q_lg{^Bu{e_jxX(YF!8s6>F4T1nMZm zE9vCgp{4!fk(WT7rG7BBgTZ+$E2{R46RUAae08OB1j~cy3aFE25+__k{h|o11{3Dm zlSIQ@wIW}jIStwBBhlCGG1&SVMpp9#YAtW@(>{#!8Zcnedrk}KuYxdZeVmEVQ1|G4AP+j2rf^5SaoE&K2j>QvBu7)cRNz7I3Obm++>~MlKP=mTOo&7#%EI96!6_c>3IS z_AUbXX}Aw)tXq@4HFoEPuDoKwr^14PfsrRD=t>qb;fikE_;8tm9=qEz!Ir92JtYRnP{#tKwDrOK=SHNdRm*R}X&k_M>K}FV+~} zE&(g2PEDD#Z#kFW-dw55@>aGQL>ACIKR+y|xy?#uajvBVG%OdVqsmW?E`P<v@$T zpMV0E3K_yfR+jZ06c0B`DWJ~hpR~{Ep|^M}gA++A}G=>*2r;({f6 z%55!tN9%ovw2O7$$O5vbsQSwb_M4$~zmj}R7nsP5hvqcZBLyUjcW;hyU-D8mhtms2 zJghr`n{&sOV6T@PmuhxC3KsSzx;M%zTPOK0!n7+#i`RZ1u}c*rZd4P8_1Z2S`g|iv zB=Tr-7>~Sq6E0litK;K$Cx^9n0K3b1E2@NN@BbnKo9ys%(YDwkz7u{r2cf=vH-dhR zu-hK=3=5&p4X2uFso4BX%=?Y*F5uWqEGv`}8l1_T(o_SDuhFVrNnkn;!vM{cmw0X# z&%bDP$r&REn>em*7q2?c2xv6bx2lEU#@|w89+wc;l$H(LTK3|R%{)>*^-7nZEV`GD zJPqM7>9-KAVO3_2#z)ko`cC@$Nf?UCScn_zypq>K!JvsXoM8Vv!hASO0>!eK_T7nZ zyesK<)uWwcr_nBfZeT=2C?5Q?(MhwCWf;N)ykPLeI6W;_DoG0GQgt~lDD2)6#(Azl z3OTXTwXp@0vRAjmB=df0&}I*^`zZa33cvC+R>!>~3aa`SFqbMyi1s`20%_>F%cxun zIa$zB_m^o`P~{<_BC-3j&y;NNu^kfI4?b`k*5~sA# zJ&y4_CIUFMFFN@mE~ggrloHajwF;#u%JCNZHgz{-<8WN+`}OSZcYWH_TuxNm{p!Z5?8 zc&lXc5uWFS@jCL}U!CijUHkq)KB&(;C83%v@$urE>cDBMRM@0pLq?8>=X#~Ks%>hl zG-drDPGYoAqqelLznXx;+GQhCocVrK904GB-BsLCe}Iy_JoGI%siX0ItXfMoE=FE} z4%#U5sbRToW7a@!8c%u?LF~+&W)P`#6GshLWH6T^q2Ys!O|$ppt+W;}-qWhTL7oJ5 zr5MR4^cu&V#{@!y>U-7Nf6YiL+ml)#l$R2{n3?Zgg|sr1HB_pSIPFkf4iJo#vD!8?WUzCMZEIud3HB+f{;F+h{B=; zz>@!J`au5ER@FXT!(V~66_%O~JG)%`o=0eN%!d2LSo-MuFb6{z;DOGDG8`OGl#c*G z#?W!LQJI6j%F(f|gmMPo=%7@h)SJtCjpN0}hsX9juWT5pTGO$o*S5RmDDlq!5M7*G zGP+MW6;c$gl9ubE{;ZVq4gO*OHQj7=?k;OV=x-5#G^(Jdz;w3g+m?!x5{@Hnc24vq zuOkD;K5Mka+T8ZXt*=%j#ZAv=O=h}E#^aVV;*-mVZW)U$ zh27yG!(@1mMFv-TZ71uVqcW-R43j1;GBs4&Vo@ zro)vxd3)pa^K{>q)olq?Lrxy*2_2P8_3X|D&`eZ>TX4&oE%-ckvrQ>210x;I@d{ki z&jtp~bY`VvqkWuk^-5~Z`^~$$UCBgKALNcT%1CuZqe=HQu|BL?A2Ay}Fidqc zntXIIBl|3IqPx1nzCZP={;TNlPp?5!WtXl(O z($mwgnFHNT+GozVNeLiO1BS1Ge#mroQk#u|SUz#&gpDlvw6lQ|Gg_R)ZJva@m{owl zzsxYn$(q8}I(bR~K|y$9zyIboYv@=+L^@T5VljpGvg{ctDbm34@$my(jSgP{j5@p3 zcAX|iU~gOF$TH6$?~F-!C|2s`reRtr^epjh^MIPoFw8vh(qDv;mv^DNTLciZb!zJ5 zDXn9z+b`98sY8P&^9IB0&=Q>Hp<9+8@H1Fa$+YvpB(_OrCQ|4f6MNZ&_dzu&P9459 z8K$Ni;p};j&}AT#Ok!dg*ZnDr?B}Tl;LNX4eLs0kLAPV}3KPfgO4kzf_?oiCBdDoU zzMyXwZEm3>CG5wihXq1zzb7mn(q`OKR7%tZ?c0*vHWqBihRnR+jJH@^OeQv(4yn-8 zeztg`ojkbS3(8`Ml9zFMdk@jb6kH1ynyI%e7LLnIb%zin}$7?HFf{zT-5j%VAcF705C^!G2&`X&Lws@fv^8@1_ zxQ~M5BLUeO+_TC{c7zjo!n&n31qVUNyK`W$EZ}heH4eL`ecvmsV5C5XOT&6l9fh%(Q1Dn-+h|wPsvAC zAu{-a{k=Bh5qUncDi487lWT8<+CP6Q;D52l|MTO2!Cd{9*$xInqDcUC-{7hMc)f0p zt`BFOG(Jg46ucwm?6iZ24d$QZON@y4JvjJvla5|tBrVf?gI1j-0Y3h(7x2Dt;(tul zkNkmwfiQ!ne5Rl(HFm&mAw50)LW5mq4+0=&ptG}cZ0sGy>odf^kV+&ldi~d^xD^u_ zIcn!Nj2y7B+~P*f_qRpw({p8Ao!GlSty^VfWxcG@9R5FI#kygA`(I6%xVX4~nJ`&d zG5?=TnEx;EXaApJZa-ebe`j+UJryDx|t z;Q;4XJ}DZt72K2*9Qz=`LiN3bLKFIIJ&}yX79Ml;w?tS#@$}UA@w*Ii+0hbetdrwq zZ*KPQU>B(YH=T)VkMB?A@7*%0oM8d=m+wR$87b^v|A97r>QWFwh*Zx7z3c-eHN=N% zI3Dp2!~{1sw%R5o`07xQB>nL2qR?A))zfsl`N3G{Oh8NKvEXnelq5K1YY~12_xGs> zBL-yul-w{{PyLR_AraPZPg-2}hpvuC>(9h(t`bG?mb6iS)-iG4 z>3=V51ecsykOu9bSi|63YGr{R0xO!aOH}@6h?n$JtMP@$Y2cHq?%TBE=`7(gURkd@ zV$Xzg2lXzme29%|g=-tQ9`Pr{tHGXg}q- zx2NqpyB?4V-0T(@)&g2zV|)BQ)F$;V@I zq|WsI0JEjXZT;H$keHC`y8a!XoPf7^YtGKBjM@JNFh-?%H7>OxV+)HW>(T`M83&M&`$G!@;c0yadA zLjx^Xq;}Xk!UqOE^F8U7qM&E}!3SW!t|Ool|M-O<@C5>V>d2AxJ{WT1bI5OCq0dDl zsxrFr@jk8|mBT2(`(eFO^@<`D8r6c&W9{wB8-Njv{{ zaF_`wS=X&jyF6{Cq?%JPf%QHH(iXYvj4}elfO{6*!=JRRt>=x_@M!ulF zoM~Wq)PaB?bR#B2nsuPLFkiHxUX3ij&bFa{d1Sh-JV+ZyMmBIt;D81O`qGP{02l_m z2LuMYdpMu*IV20ak_4?=Bo=CFZ2}|v6+Y9~TJyBeuAh{g&Zae&)WfGvc8dc6$_2>c z-?Ei9N+kgs+cf)`;;jxBTh4m%1lE73W57-5c9^375+N=!OzoPQ=J^VPY%D&MhUPG~ zv=%gmDxtmF;dl#-X?+-qdEWR#3YDqm4xQqxH0j1Jp z>Y_=FJSyKDEU*z|Ub387bfxYRu^n;bYcAY)d|eMc*C&!Tazg?0mf>J~g6Me>c|(p?men_|Wi&0TQO?6@ z@2*pS8hh9}MZFn0ba9pDb=JKCg?lF32uciYY5&5P3>PhB2jc(r|yzmxUI=h@kgyE|_q#vB5A#PL3({Y)e3v z3FnKZcXKd&Ace@sRd#%Iq*EZ4*}|jaVJQ z9t~7GY=gIxGuz%C!8FpQVHw6BCE+?y$=Bd~1Fv>dgPhYVpQo0i=bY6f ztM??R)KO1aZ@%OrRDA_}mG^4&jBbFgBiUhUU|BgiSnn+xJtOkTw%}*(KB#9+W#=ZF zyfKm*71AMbvi!gqSYo9M@+v{Fr8=j^o}%+VrQl@eU}KA6l~M%T5&snW(px?o3Z42O zOv+@XEXpA52Yoz}&J00X@r4X6D!_ltY-@XDWJzLci4_X}fh$R_`^fclQ~64U$V=94mOC*gfd7 zYjpXO#sF1d=ZMGg%W)?w^y!J`jRfxTo46bmg$lU#D32tZhbST**OP_25mDMOsXP1Y z(8GZpXfl0s#e(0~z_>CVW9g}0nxu!_X@EISH$sQTT!i{QFgMuu4228=?nsujC? z`SzWmU0d2FZud zt5)0F0kSKh!1fJ~%gNRCq|tgd`zyntij&QZ6R4buhA3uZI2ght1Q$==747GfB&htes35&1%&O)&Eb;@f>ciwIr6nF546_p;BYvCC1vFmnTvoc;XRGo7_odd`{+uqp z?N`U3BDzd>BbAa#QuK9i^ja96tom!ZBCCwJbUpHzT}rjOfHw!Adtvd`a-M2le< z-UYGmd-MB9Z-^8ZYA)q@FjXv$OTl={fG@Q}({9EgBI>T?i>;?W?%?_4_%V1mY!*Q| z;qg*!9gPi;VkfZD2(21(QjqQnv*+Bx8n~$DXB?X;F|w5OZo7|-%Ut41BrKa^Rn}k+ zM{nU(#fMeem%mSXTCL2nV!j4}Z#ie?(Gux4_mAo2I-0FFb80I%lVzN|+I$SYGv zP8BecD0|W`oM_e>k^Wr-maDiRXnEj(lF)u~Rd=|Jp}jyMD#p&?YS7C+Be1h=+xX!+ zO;5q2A45B>B6poxQf7J)jk+R8Sl%udlX3_=N#@-&^t5ex1^3taI8v9&PoXLXTO;oz-o9FK2+@ngM3DIu$-s#^3D6giym>TcvSanY~PS5)UHe=OO;*9w=*YPO>UBV`pWFK1T z3~#HRujbjTInVP|2Adh_Mjtegg=(R91u}ecsxC``WV<)+N%uDt!1|Tc`f`m{(-uE6 zU8R*}*Q+||#hE%a8ePBy-LBfKu+ZWv_g9M?41rP_Q*bt-UHkA*oQzXkoSt9W}mR6RazecMZMfemITVHnH$0krWoNk zX3JskpThb%Va~N007eE<**wAhGX3#B>F>d8b!-670;NKC!z;`1v?4D@=(tOc=PLpC z3pH`zqdJ-E=k&C{%t|_-&z~w(kR!qSCE;9dtMA4SuYFR>p{+W@HTXRHIpMzH3(>53 zv~}f_P44^8$w^YU4R(DP+IcPDA~ zwibsS>ko#6&K8@}k^{m4{0%W9J2$$W#XK_|)x-L+5?~e^TYU6q!%zfNRN>4kT_S&!hFL;0R0(+hI zEeX{vSQZAEiq%JDhT5YslHJ*vDtg|9FbHTRXkud%`Es%5d^af9e$qJHV#+Yl=22!W z6AM91%7~W(rYOmNPxFde@?}-9L8kIpYS|ms`(0Tpktv0>gpy}{^RmofwK4HI?{b~+ z702?&y0VjZ43fK%K@kJv7$n6*y1|5jY<}N3tz)Ro+9r9 zs0h?P!5Z(w5hto6DN zL(Oj1n(;g#tudRic8#!;LrymU03hDU3Iw8DdI$*a=j(Hwey>WoA+}kJu+FUY*-2E_ zw@We*GQDiql6X?*FW3nA>8N-4Drxq12@yYj`eQgIRQNy?^qRH-wusJmGs%2;ByFVm zN<)_7^m(}J00o~$9QQT7Sh+s650*cZ99Wn(CvtV$EEteTWRc&5*27%cJIa11lk-~Y zdx1(CE5g1}gPpS2Jg1Socrx_t9Y)UONw({nOzP=4?Kt3lb=y46Ag7Gc2Ka!Dy9yHF zZK1@>1+D&)>asLyhmWAw9nUIqxti6bG4#7Dh|@a_Fk|;FvE4%j+iS}T&*x!b#yfA1gI>%58KJ`xc&>EKanG}O04BI{>k#7KSv4Pu#OvqAGSBz|(DJv5&;-Q7>nWOseMpChmAp(GBqJ4M8aaZycWn)Tw(bmEjV zpLbL-QwFFCvU9PQi)dER&Rbx%@wq)p(aMuYaeJN7U}&n=$G_`AehI|JdmksjZzq-mMomua8- z)vtJ?WygwX)OVxXT>E!g@MmN%f$!BA>q{S3GIA7+_jRMT1nL`t{*QSHe zi}HthOLS$mbTX;e@4DfAMsdCTn5ZLz-EsP%bHc$g^+#F`L)k%x_Ye=YMfRQhJToc7 z-E&1-!!cc|H|Ng+j#-#cB{l|gG>hMISF zq5>sglOq3wi@7#+CQIV*WrDs7IKq65+^&SqPN2(J%0 zw#DzoJ3LR1XbWtmbmQH}vFn`77nKe5ZUiY6rGVX`-vg)PLjhCQ#KRrUWNyyb9d~_c zDVcA+M#>H6VVT9s>ol(^b&-=@?Pvo=LhDM|FX|B1B#?vg)0G%!oF5X>YC&o&c-wlt z0FDG@87iJda2;&DU@bxdCl;x?XI<=^M&-_U$;Cq!UY>o&tAIH+$lrEN(a+jlbFO9( zCrINXE5!|tH7i}l_#rV`n9+a?7-D4miv(?Fv~--!b1O&pFc=N9e=Oqlu=yT+1kR)< z*%GaGD>7|N6b)fCkEZ9_YE7+3|Cku0Qx*AqWwvgIiR~>%YI2pfz#SG6Or$l#ZV={; z!MuQjR58z7lq2(oOjTvza*W(2wb%w06%&6OV3Nm(ReuX9vnj!OI$PHcDNCKL-u98% zHKFBN^i}puQWbC8i~w;GG`OnykR=8K6J_r?>u{B?B8KGS9KAOwthRP0XP+nAM*J*n z&2<8*^k6qt+byw|n~2HWUGkxCUdgn0F)Rw!m6OW!?oGev%F#A4V#%YTVc`2rq@s0v z0~4*{-D4=CykLGOaRd7)^RUn_58+k-3q-i*qkJThH$QNJyx+3taD*%IjsQ0+;1h!i zv!2kozQ}Fspf0iA2*y;Ye92fGryCU4;IV~PxZpwV-408JeLkSw1Nlh7T`4Iq)6cs zlY>&>;rwj`v~3BBe@=~aC-eG($ESxSd&Jr5hxau`!qe{!3u3LmS6+XCsY-+z1gbqxAOzuW(x z%KoZRfu|de=-*ELm3R9!qWwQz{JWnCWGJ0~fBEM_C6XB~=>NCE0Bq!!i6VV*(a~RQ zY$CQ5T0h--w+94>d61Wnn{RAvR7U@qt}NtF^tevkX?Cfm7IbUoCKnOMY_D7`*SGig zCx83hetSY^n}&mn9r9z}Z)Z@uCW(?V5#nv!1cnpA6_zTC`gomqS7a-pqn;hRebl_P z5mc)Z-OF}wDt5XQvfG9F_21RcKkJ#_wP1w>x>!#PH1w=_NUVU&`J4X7SV3=n)fc}b6WT~BUl0}r+eU@BEYK1RF6#I8~3F3U6zI{HbY+0r05<6%FEzkqXjk^2y1 zb;k&Y?}cLdHpHgp$w#570WV6&xnk$1M+qGTiEjN>R6w`SB_ZMF^1$k^WdcnD(#PVY z?lkgJL3;5Y$H)dUJ&v8t;;$FUMckzm)j_;arL&FBB9(?WSy;Tml5qBAJL|v04(l!a zR4O#lRs~6Nzj{GpPC9h@9z*Bz9Z!#%N-bN>Eu+L=y=b)0exy2S&)8Sh82KNIyhk|s z!8yw!H2boKoWzdZ_~3vm7-aDDOr2p0Fb*{_ybkwfW{T z40>Jxi=v!c1P8Zw9Axq<>oW22@b9c>hx1$vuiX!PnY8&KD_` z<73mK#HH|qT>ul&N?me+`l0V-q_1RJEbt^%pwI|QUOw|i6!(XO6b&4l*3FIiSY@Ov zzkooh+lQ7D&t@$gJhh{DFL@nR!#8wq8xv&%Pl98|nyqH^>%p8u@TNU<( zFVXXYTOf(9wg;3S;oz=xyi1r3PkIN>bzni{{@qEmc+H6R-VJaLD9g5av^t#IXIR@pV1H_<)tPq^fLE0wp-V{uQIMb7_4Q|wW}UI zIZ=OwFSEH0qlN6wt(9ek-D?Qx@tBS_IU~o9FH$i@FrTnG9S5$BUr$1w>yBK%%C2q% zQB%MifDWcE-%3}bm4OICrC$ANH%FHf%kUN;{41e(;*tU#j6E&oWcsebF6nF4h zn`5p&8qVH@RqjOa;%vDzv0bxjo+t_TICTm$mLt7)Cc^@k{-e}Rzo?|k7U%<o74_vIwo;OKaX1Idvqq?Kc- zqHWr53~Sy42!pWjEg%G(_Tayd#=ps-P={IOfy`oUs&;iN|H(g=La5Ez91_{Fi!QTG ztfNBcZ=yCm82tk7<4+x2c(`AnP~QqnH+#BAj%aaJtJEQ;jzP8A9mL9EtEN$d+Ib3lhaNWc>}V94Phh(Ijq9rB0lyU>nE{__ zJVR9Tq*Bn2F$=I2`w@w%dnjP_(9QYV1cZb7%~M%aVheoQ;jrDrNw=7c(Hx1udfap$ z&gu#s&Wqv5Ke}S_l*<^M@};gHTlf_7Gw;_|c{qRRSjx>5?Xxo+l>cl?k!=yPxd7v0 zJ>A<~ca`UYTb#TBb-3B+5+hPVnLXnk>1M(NZG6WZ$Bb2Z6LnW+Q{OmmWb5PW8hy%l z;*@_S5!>$eyRb+KOZNTQh8L%v3}@!cUPt1Vsz;Lih8FvMd;Ohx1tD>U!JoaG$zbv@2+W0u!3t z8DUKJU#6MxHhr8WQ8uV+Z4cTHyN zE8MFthMYx!l?-Hon4#2(k@PrEr~cEHhu)0q-!t8? zcD?*B&c7`RxnK`0>{}`uPXE7{d&{6mns!~c(Sc#G!F_-M1|Qtr-QC^YX>9Ppox$DR z-QC@Fu*O{*+q~;rYkhl1>>Vf0iT&eL^pDEus*0-0s>;mgzVf-XdZ+9(P*Qcf87h?* z%z8_6GVs~I&2)as(Q|q!{N^(a-ZygFJ6W4QYNT|I(|>H(4DipKUP?Hf`&k(pv7*s-%pb0yZ(`x0t}bv$=i4WWFgg< z`QK|}Ae6)gyNE@V&Uqh&84<9-nsbe+F%^Y&xiW=Ma(q4X-RrB;E+Sf6pWNpwP@RMe z;6zJ`)lc;Y^n@$zM+L{If1b?E>#d;=G6A@pyOdJAL3;Uw=LZ{p(Gt$8er3B^4 z-MJjCO5QNZVZ{@T=EZa7T(eKeeTtJ%)@z4A?d*ja1&yPm!puR=u0-YULsRWv`XH$| zyf?K=+Q$d}t{}FtGozxY$6%z9?+)g;tdSCSN-zCceMZ7&&JF0`g$LB{Y%DYgyleXq zV>HiSFxkj!eX^}-%$8VZvNAo0nXb9lI@+w6&npddV*V{!zH_Ljko?x`)q{A`RXrex^q|4QM)QdAWiJ-lWhK7pF z^OX!v&_Hapc%E#eO$$5B3!!1-S@W`?Cmw=LyokX8>*+8t z))R5nnQ*Qrm_D+D$iaSK{UlnfD&ve2OlxSLnh;Md2E&8dc(}gQmHeCh?5M;DZ-{Pm zjk~G}A*8!8(g$Di_1FU}RVg84^w=~Jp?`I(mPcJvU?Y}oN%E!u6z5)~t6&bBLD-*S2-sHOP%X6VTDw9fc?NX|riPQCZ1Xs{w zU}h5G=0rIc-{MJ8T*z5*7#n2&IGN4drYu8row}@};dxbz3h#84#B+aXG129P0l-8j z!v?SF()VSbEGl;8H~C)1@|up4sp@KE6alPWqo?h;U}89ob6#x9#`59BZlt6xfk z3m)gb{1R7CiBC>j;x%Wls!V!FMxikRK2zE1iK;GKfv~DPZCNg>OrSDuB>;Zl&d!Ps z*C>Q_{GBt_E{_3Uw9kp&b~3|8b;LIT&udq306*AH7Fto+<@&z@A^KiuU%)C=pYVO^ zR=sFKKFeh|UpYpYfbM?Lo~-~>LYVX-(;A))lA4fjCa+G8IqM1=ZN7c$4ti}}i+f#i z!mY=O3;Q$utFCWL381e@!eO%Bvd_gt%Zmv3@%!F6qV_x!*nO=F4v}-=_1o<(mU5C? zAGSWDH$0T=n%~W>GCEW#-TAjIbSxi#;DeGC`n z(#)*0;4FSYum_G3$ta=g#Ly}P6x(01P+GEjNsuKX;sKN?)gEB%n(vtfNJ~|_mh#^! z2-sUdpnJY$_fEa})M+f8I_HXD@H_XphsTv@6Fy9$F3xMe%CnUR>R$VV#Z-qhBx zqIYePw=KM&i}q`>kE}TV$_vnDA)~3Od?{l8(G_Gu9^t%cUA^YT#Occ)2{brf3Pwld z-|ymXx7x^TtD4*UB6I@gft5QwQ_Qug%b%Jo*YG$#D5F{J?`hiVj?Fmx@K@4A^)+1z zn_$zGgkD(v^eAy33{PQ0XfoE@sCh*hw`(T$vrTnD8<>elN?Ktv?&+Dj?C_D}`6%U+ zT@jzG+K5E33Yh{ueJs=`Kykj>2#|2UfA~if|5j0_%O?hxbMadV%I-w0_;`Fw&#RQr zEBjg!SUaB`{Vc4^7V~~d?B}+b7T-2Wh!L6%4Z~B{6az_>%m@WWYLf`ZOScAc;I&$J z_)|lw9CqR(K3PZ>P8ZizEtd%!YtSa*{}%5|%$+~HvI2%erMKV%Tr&66g-0jQgVHBl z99S=u=DPntWXF`V$7pVR*uwPPntiP{{2B=bu@E%fdixtu^r+{M70>}%N}8UWyVuUi>W;1-8|7R24yycS1Rp%SO%OUr$4 z@C>=a4)^8;@B(9SU%NnhGxWQ0whXBWDunIdnulQFBUJA|~0B%nTcdv2o1&c;?0Qa*(u_{C$^?rF6W zQgEwA0Zpv0*Z7Yox#y*9NzO0Hna&PgfFI-8^|d!}QW#`kaRU1Ve`{zVlt#Vkwe zbLz?HTI8u{LRzzu<7YQaNK6-8u*Qa;DJE^P!sm8Ai_akdxKMD2avG`-~A*U{+YDM7%|nxH5uGYZqH+5hODJV-Alu@JOjdgoR3hUR&$r<|(3pRFm4fw(pUnpSz4p7VJf4W=l4 zBveDzX?lnZyYpC69nAd0cwaqVH{Y3M*cpWc|oW{#~)?3zAOFD>Bq9?hQkrL2A^CCDa5DYC~{m|{F?V_M=tu5X!BBZqP;F7j8 zEDw{>^x_WKwdKkSa8c2??#}k9d=Ldm10kPeOW$o4mhSGmsWAjjW9CR zcE;lAVQ$=}o~`}-1K>|u{=L%LiVS?6NvYENce*Rl2C{7p{iNEZKqz(orZNtz=0Q;K zmu9B>{&Ij@9& z*=W-}ev0T69xpD4IF`azuDdzEcIKTtoM$X+3jgt3tnII=@F@w8bCE$H*|#bES|1gK zZG!ue#4nf|HOb;I(bI81P-RM{ElLxUySU`|_Bi}&L(-LF3Y}PKU_9RW>KE-A`vGRT zW5MOtG)2>zak=Z0`zL0xF=g?935Zuodl-DdEnVDH)ac-@kaQ;Na{tev)W7GH__^LTA%WpL}I*4IqQ(H$j&hn+9G-+_T&Q zxKAS#$zMD!UBmQ%sPWL>u)oChU)?_l953Ol6uDMSltp_~9g@Fzm8ZG6-p15rz1i`2 z`O2#~iA1GOV_l_?4UrEH;=2S3O9bc+RC1So6O%o}vv2S{5t{R~Vjw4srTh z3#gVa&J3X0QCd@Nhv!SvHe22ia6yFgqe_fa$|Wa^8n@Mb?a2fhuTlECR*inL&w!=B zp!bcGxozq#-db)gAcNwc3NA|ZX1o!|bU@%eQq(y;(m(Ac12kXqdK+bGLIGeTDdW|a zM!HkP*i))S5|ssEQJ@8;`B9)H{1}-|YL%y(5?elD0o)h4O>RrZiy`wg0f2GQ$O zO-Yk=)x!r1L)~OCfc+;c8?U58Gh-GAjkWen8~(+|S)G_4Uo9G1$f`o6$b) z23$Z^(;F0>iANQERa(nk?yA!PzZe~vLa`sMzHf7NthG00PJ)#Vf+F*Rm#@JSk_c$Q z$2EPuC`(c0L^8x=&TuC?tN9S^W~9ld#tXrs4J{E3N~afcye>+6esK_ilrI6DPP)?gDRyx)+A?~G za=H1;T-xpY4Go-pYt-w2m2YvYcZ-FV(*FSsko2^hiwgH7P>pb4>o&pZnqT)X7$U>) z_#7EEqkurvqb4R#%T3ZCEmVxFw^8$ZED?(Sc6BOsbt(~AenW)sXk`87{V6YIGJyPP z0V8&MdgOgN@%2iP1uhKn9d0MwBC4?@2HObtpO|{i6epIz{-F0KwsV98n%Ad&gipP) zGz76d@F$OqQD2d3C?h&hAe;V7Ek>bbdyC5Wq8*&Hve`+RTq#*dQex47FL4lU@;?)GU&_*kj9-WQHD zCvZMO@ze=aWz4^6NU0#Gp4QMSrHOB;mKwfB=tk|u0-2W3gQWjz;+=@JpPXLi+|x>}wnl9!A z#E*jDjGXKS{Lol}EIENI0Dgc8wK4jJ^5Az#?e3CtXBZRnx4$r>UqBxjZ}^M270jUp z*N~;Mz6gT{kJtMlTR(sDGya3Mc_NBp9ruxmlacRxlwRWt-0`}YK^c~HG>N5)E_0g{ zfznsM5rbcfrTM(9$=qZUhIP)Js^oXh(>ajuI2-B9@?I0LCDa5|&z9!;=9w4Ed|d9> z;-vIJm0BpqsLXW*F~1hBNKIvS{5;+touECXApxap*Smug2yIykV=o$veHNiIGfb53 zJD!uL2qxlDN(Ily853W#bU!0%v`)JoLGl?;;?<>Fu3m*BzRM02mTjLvu$vs|WFKrH2>XW z@h+VM>+OWXAcbQEYcpErCr5czlTd{c@t5u#y<+Mf68_?L%B}Oy8^~u+&#Sc%;+?}A zRb~sGNxN;tFy0={Roms(w670%SEVHvSBLa<`I4;)*UrDCF7{ow3!c$Jf@%yqIWC*V{=bGKGK z<60vhZ|N%mE#%icOv`Rkke83qqjqFlD9+oAhVd>Enc+hRQ0ottP_{0&QNu%G`M)i~ z0m5KQ`L7>-KOm?0gYRJxk3ZwJhiz1T1YOxZb+}qRi^Q|~r83u}c|4c0eLtna4%i|V zS)|T05p1X$KUsJQ+p|L_G|Qxa%NE9t*#0SL<;^T2G?HQdc)C#rjWU4Mt8ewDVP0D~ zo>p_n*tHN*;9FmNfN{4*V`Rl+Vp!=9O9%n5lxR&ivK^gkh!anFdzA69-n+eD<9i|Z zbZqm!!z98f;UO4qMRCPjoy5i3?u7z?MhAb#8o6yqTdX3PrSU-r6q#LK@#SY$v{5>I z*C}OCn#)`<0VoMg^JFlP+^6u2#zXe z%VOj6fAmd~s~!F7+E;>7`qRFKL2%zUl|iLOt(U%Q^nIRB{&ULLWk{&laG39#k@P=a zq;&UY&(YL)6cXZ+j?eP`)fmM`k!K$2DgQG6^Ef!aU1mSGw>g^6|djPB1`rXHd9B=Iw z&e;!Pt?(DveJ0-m*F3d%cC7Gu}OiZlQ2!Mdhpw{1K>pIgaOn&-9X1O~odIiwvApV{l<)FQd%DOHtNJ zY6vnltXcLCi92mN$@Wx%J`RsCxc@vm8ci3g_V2ADq{gqUOD^@>R7!dDK?FV*vo>LA zDGnE$cttkaf4PUdV7XYm#tOWWVeYNP=Q7kM7s$ftns*6}VI$eXc+ za#o)h5N>BMcVSITl)+<0O50Gb6lkS*9H`!Xfm4MD&r9HujKND0w6s1xSC3qmjxAf9 zqu6UYHv=&Ybeamg8mbyYPh=)mDeDQe)v!bv?*z z!W~M|CXKc5b{*fOE{RBm^?zA6YPv>DC%|6@d@f(f`a}!#VOF%*l1;dOaRrAUb$2(q zx+wK|65ui=m+Jy;G~LSfgz#jkPR8$My(K7GNVw8+i$u-cW4Zlp3{9-eMq#%tWkMhj zUpJa)H=Ac0^*$m0x;+Y#`K$Qga7Sx@`)1V25J2T}n?#EY&wxF>8JXev{0OFgX*{vS zt6AaINp}k_*EM9RwsJX0>oXXdMX-~lxa%)OabkGUT&Q%deKr7Qjz};8^ix^Qbu3Pj zWO7(}(l~0FF0yoi8T%d0VSe8vD%O0l-)IqPU;4IWZ%%Jlg=$loaFb@g6sKdopL+G* zoGfk%6nBK@e|Hm9eapMeSei;o5_bgn=^w)1sy8$4f;v7^GUerEDOqdMN`*&pkmf7G68hhD@k*{%e+W@CSSbfcxj;tRW%YOVeOab|THT0I0 z;XYbU;y_-5Q0^id)AJ3_^Xo;#wK)uHQUtXupYRaorrnrr7Q_AV4eF+Hf-X^$y=DEq z+P4PiKw&rEd=N)=$Xi1Uysy+P6$>_tl^_KZIl$)N{LIy0DfB5$M(W+Z4=cxK>Th!n zXj%8XsZqqoTPZ2>J%1u`@|@HDdAM|r@(DjSz$qLj99&mFuuv}$RNj-ywV@)Ae*E8{_t?IiSFo&-t)xB7(w=?7-iSk#)P}SOS zV>kzVcd6l68~DwXzWo=G~XR_fWUNmk*yz3@~c#12R&Qz#k zF}C=_V*Ck;0+ENa!A<`o9)&^4W_K{#f@KDy{ZbwOnk9D7z&HT6E{Eqw-Q3_cQ_&l2 z{=HJQYiNh>#S#t7zuGm?CX4}Vf(^~j<2g|ubdNBxQqaPoVF5>Ue&vLM|lbaUBENq zY9a}T&!C8>xLX?4<11Tl6yO!#rNSMP*(b_+{yfXDK3L~_<472Ek>DXPGY;^PU791^ z?xo^~5)G!&lj=WAmU^8YIvggXAja36sIGeO1;w?9w|Du6Vi1xs?1HPKLsW$Mp%Yh! z_tSm24f>Eb8>0~iPFJl*YCK10rM7rDZ8y6xRu)U1{UUx%3%?}C{YRs16aT=WN>ChI z@TGEgdNAzC5b`0{{li8b0>h}^Yp_upsRL;dwQv_%1eeKCdzEU}Z3PCZ`2Sln3`C(4 z`_O^@El(HY8vv00XX!8SBQXB&%|}sTxBp%Gf4wtWI3UN)m2>~Q0kQs_0fQm&H*2+j z0cZcB8D zYren?-Tv9`ub+X+s!wvg_Y{k1KKjB;Mqe#eGBNpf_rjS?ShQF%5&xTSn{FyHf=MSQ z^RjPRu)|OY=WI-O&`b645}$kffRhH;a`LC{CvetFEb>G<34-b1qT>{^Uhj74>QB#V zvJEm)SuN*_gWj9(mgW{po2%cv8_9o=g6i)O!e{`J*Cbv&)04FAxGdKeYi8idBE`>P zc1#z-<~3&~9H8JPY|q=PnHKNW+s~17k2H3p`3PgdZnlfVOH~*3)k7TU0P{HHTRX-} zdrO1+;(F46R3tdLSl`%B1Vlx~2fnXVYW*VIboS;NW1$~|qR95ba)i&_hj{;eY4RL1 zsD4K3&MWnH^*n1Xqd9XoRws$;AFrv|>7Sp!L4USsa}a=9eDFA24_=%FWMn3{QIt}4 za9mUBbp`{6o<~3sZ5M6RQJeP=fT3;mba$vwc^89Qi;eQJGw8j3A;;?Aw%!AmlZ{4m zr1ZriEyYJD+)mH^RwRTnE2l426Wte+fVQRfJI#>UPFRBzFf_{!7tQ^fu3C4wJ6Ngo z%GnE_HYNG(+O3>w*2s|~xTOe$e74^1a3d=O^2Om6`FDkH#Y}3+!@~xAhtvOwou4#R zHlp`_Bd3^AQU?2kjRL5Q8l;SVV|S)66FK<`!K0#T%{o-z@l4(jXe=XEv&=N%5C46{ z7={1mN5n7NqC@kwu+i!aWGfE29FGbt`YyEqI5k{D8SsGx$YWtUBd5+S#6j>E_;O+* z4xX7#9jXK%Z~rBcKKh`MS{`&@FYA4~cRZ%=9K7mH-mt_zcrjC~P z{{eu^b_;n~DmAOeU<0KDIRQjM+>~WT)mP!Xx-xD?9|AXWi#(Z zF8KGk+}j*vJEWBa_TS$e>doGkL}R{u&7SpxiEmq}gWyhYu6l)f(3W~QGmb+2_xCBs z?t+i*FxNcQV9{m(-04b`MIidq9o9O=tA)%$-s!e6o`7hp*>zV{_^?^Vs@=>+Z@N`I zJ!-VI(|PZ7EZwt0^RfOd&*_A87qsttb_ty}2Dbbb@l71|W^5mrAX%9^_TnI3*f^fX zDl)z}%X@mGw@-Ovp6id!TX*2xR$-?Zs;)58`fPb{PsSI23waPM&c$09r| zv)=XX&^)(rFDdo*L`&`5(`I|7^x;nn-o5Q%e>3Xa&!T10p~LlOpe(3Kx6xswztaY< zHHB$^^!_r;Led)pTBa4}_VGGx3SI1nC>^;WZbsGe#= zYdo(>um7XH3tt?n__p!qLi}pV?JflOt zh;8=T!R2}^^7t(t$J04X$a~Bw4VDR9dSf~{fdYNqPf!H9wi(>k!+)}cED*^nuyrDl zrRzLp)*YJ=i5l(@`;2buD?LF6NVgYUCTgV*?Lh=KxoQ)L(P|iLq6#l&8!3h>x2YTP zE~8v1GsMX0a-tDKB3 zD96OwC{DAu9G#~rO-?*ch8E^2F8>!cGf8^6!6kRhs|1tAc@f zM<13FAGi=Q5L&H6Y;B>Aq`m}akR}W#CjmeSAHYWftrbSF5t7{UOqcG308J+kH%jHC zs(jucoRSFKV+-hZJC#@e9{E5`=9@d)#S*xDKRxF9Qkaak5-w8V8O@O*1cAP;w=h$- z6siM1kV#e&q07^HR&BL{E`Nk2=5c5FG|<&a5f^r>`9a|r59ez@t=?L!81HSPQZEwv zWaK?ngvE_|csY(7j_{=3SspRfZ-qvN5Q36?2=&)Qw{{^bqI_aqncC{b+Nt7pDP9SP zbaGo1JyODZH?@;y)x$6{Kfs%wuHcB1!-X=PqLi`d{yhv9Y|X3l(XhXHYaS059LBFB zP1jRv{^Sl*x4+7a63FJ1$O-Tn?YA4FJAPL>7q1Ud93W%QES5$4Zf$owII);W>OD9p zc*NUGd6}VNCSlu94(H@(?$m2lishu6!Y24OwLA!)7yEYDoS>?mh`V}wDOr*%u*Ydu;uXYitFuJKAGZRjOP+7m- zD>#~1JhS}8Xs$1{!z@reAAhaW@>X=W!-yyHGw_on_;|K(;{x5LJ|Byy)8|AgCL5X9z5HZX$uEN9Lny;`Qgb%G+b54))rH+ z#hmF+mh0sk$aU@Bf83(<)Rt0M8f>5-Am~n)Hj+-eGUu{8VwC#>k(+uELdP^2xg}f0*_^PYcr*}?8yr(_^2@# z$-xyLBaWPFu{m@REMuwpZ&WYAap){~o%wO2!H9s;gtBLUHf>;PHmXY5mX+fBB2xPL zjngxcHTM?1rOaVla2cknJBKfS^HNepr$sn9G=iraT{DUgC1<$VT=mp;&0Ey^BPl63 zm;!?48t=-!@A&sQKF#IFBX^Fj4s_a%%8)CIy4p^*87>Rs@?h0m=FV4BxUHzh!Ctsv zodto@yDkb@EG!gyuzD0Ynx?qFpNyYaK3gPp|FmE<_ycYr7J-s~qDQ{aYyzF1Fmv=x zxB5i)paI(w%-3j+>X@ulGh3gi!dV?I&)DpZ&4Aut{Z#w2RvHX%B9p%gG~7e@#B~_- zN~DRc?7&^HWs65;5%ipDsyLl_*nW5{H8PwWSr`sM0AdQv2-`E>*mcM#a8wh8JKBYj z@B!Izx?~p~323Bw?2z7B;6;M|`4-X{iUNR|uC%P9`r&{Xm2t8inNBG6FhpTRhDJ0j zd181{lKhcfd=&R&R4kgRp~8b`;Uu?zoCz(@AC<0fA+$#4fnUtXdb|A;ILNaJ(m4ud zdNpdlhif&dIKIYuQHzw!>@MA-d}4Rv0336V)PyFaQGVt~JK3?`EGqoP0L4$N;qa}Y zwP_|TH5Z0#BG4_8Sn3|jujMqer#ZOxKL zrOycF)LK11nx?~6GOmlx5!1DXLk1&BCOjky#f=GNj|+*o(q9N=+h?!pjn9FZFk{JE zKsEU)SiO{&pSl*C(-VOSNyI=sd?_uYe_=TThHTF%_1aGQeh5gkzKmb$S|KLBkZ1b- zS(KtNRU{J=OJ|JAmXsDsWtd8^5Tkl`dHe``JZwtB$jYoWC?u-)lq7XZ$$#dd#>2c1 zuvT!vn(PF^!+2My_p7CLn2Z*UZ>6l>F6`tHUDwq=#g>p22|~Vn#d?*j?jTWqbGNO~ z()fOM$me*dXrurucUW$@L4edsL#%1f!v zV54%DwkxCP7?Ya0X5Nf>arFBSK=$<}_we>ovDMJ?$8>WUn-Mp;UFx?0T4i|TuNHX7 z&r`Mqlz)jnH8ES&b;A}R1brN@CZa{&O~VVjM=)!zxK_fypym>&F5z$GGbGY?`$X@|9n=8 z^QSj4Q3P0OJOLfh1}W#f(ZzTQP9!+7h0X2leNN z%Zq;X4-Pu(p;L>)Q(3<`3Q!)K9Mk9f$7xtB_7l_baRGjDQuo{n9?bO_hooN%ipn{& zQr<+Q=7y&>N0@G^Q&vhGg*eT&XiIL#Bt$ni(HtJs$zNabL)z-}MEg%=r>9F_0w{JW zm0<xT4DF<=aEGeB@M}O%%rDcoY|}UH@$Mt7D|qKf1#XZMcelIw`F^4Uj{)Jks4l zTk}{hkGY}?`p`O_pK9igU>9_x|CF8z;|z^AAJTdHTUhs~<(pv01E9s!CA3!-C2#k6 zlu85pa(t=Rpk13N7Y#mO1?Ccv4jH}SMNlAQ1NYTvjnWaT43CUPHkWtCkcf$}iLN%6 z&yi7{ENK!fvV~}dM*ZuYk{(7C@|mO!3U;fxDfPlaBGb?r$W^P={j45WgTn0ssNuyg z{~_D#v*+070|5UUTn5+AMyD?&8Rtn+@E7A>)B4pG+lJ%2GxVzS@5O#`qI?qvDh^GC z?`5ZnaYC{S-HFdpm0KV9&SnRiRydr}lxEVS%c7izLT#mmKz_T7ivQvaYMwOd58AmH zie5&<4n?`j^y=dlizS6I|4%F`8M@vI*ZWM05>dp+j%ZF{vP6p($fOPcH^6#ot(txHq$tp*XUxQ=FRlwCt;7xj!i zkLekqy;l4@W^1&y7y(Sz{Vg%g8uyAZTlbf1RmK8xVsqKU1L)f2w*C|?0{DL^j_xhE zFxnrW%92|vkR`@IKntKSe;_JabobVK zO%OI&_Ik?RBF9%yx|oPGRyfcvz!m}{bB+xS*#2oP!S?A(NCJh7Y5)09x+_!Rozp&= ze4j|nt^4FtwC}ze5od2%7(h7#R}BpEs`&k;D~n$|KWeqR@t|Sphr9&E`5;S4Fqgaw z?Kis7barSC2gtw4mhf>A`OfZq&F5_5+pdfViP@zn#O1iLA>5L+5)b_9K*9B^rZ9&c z^b#;Guy};Lh32Y5I-8yhi%s0ElSs_zHQDY zk;|E3;I#Zp`gjai8YGsVWC5qLTgJue(Bilqr2DL)^xSd{QQCG$!IJ`f8_TIRIHOxA z76H17sNYr$c=%eb9bf@maZC616}0N;Szky)=?ghl3k3}$EWZ7RqI9ASorw29u-;5B zUwaEmrpJfls&N*Er6}JWW0&OaFZ_8g2)7_6UWjiv2VP$Re&3eATbTD7`-tOrSwEr9w?y}< z(k~N=%pbR#yC!oMeh0+&K^7=Wcw5tXNTv>$9(h$rA0K54!MriGa@6EJrXnh;aVp<0 zOm|OnZ`Js^UjB>`)%~o~^lg~L+*FDfsw^-4hhi2PQ+n_+I*-0TMI~sw^YRLxt(NgG z01!ii6mO!u5~!l?y-r`MtzbE%d~G75t>i>L454l+dmqGhiYh%h#4I6@2x~ezryy|@ zw>C29Vuj7{N4_r8V7cKif|Js2-*iGd{Jfjm>K?$a;ki^I!G(qEJY#&BM5prT-tfdR zQ|iSrz_8;VH4}>qtw$iRZ)TYm?8kU-#f1r{S5TWCyoSyLk!YqfmEx4sFTPU$RDDzx zfF+Y*59e1tUmc&kZ5~WmB)nDJv2s%ALiM3p+@5p~hRL@mb#7p^60Lr^Z!m$1@rrHJ zn<~Ad8FE|kId?&D!u^QlU0|-590tIB`w>=H6X-j>ptSzKGD65|i~nYXsGA-yCGL+6 z%LiGzgSGyKhuf&kL6Q)PfY>VEk499S&m$XzK$eiv6Q$l}Yk?eY?04;{%=d(=BhUST zE>C2+Sj;kEtuh)jL*U(7BL5zYpS$zuI;R_n(7zA_e664XB2X&do}T-kL!SBAYPnwB! zMUnKl%5*m=fS*<(*=bzHqgC|_kWps6t8OVXgF z=%~huMMg7fp?j@R2k4(^1OZeIyMy#azNgHpb$ zJ(9l)pKTO|B812+2?%P!UB0)9%AX21R9c81an){m@y?ULRapECLL8!F=cHki8D(TC zAd<#&+FaGH+Dj_04cz!ZKd8QFD@VvbWtmE=oF-n5RbBJlAtwkHLN|8v?-@e;cH2ex z{2qtZQvckqB8Og(U*P%tqwohKv9H@lX#2oCn2q@LhxVEuzuwO6Vh26C-U(PEajn65 zoU`lTV#y;5R#? zoLYXDA2R^uS$}8GU^bjcCYdnbR09GD7km{J6+|P=iHWcGHM*=(4l~*@lo;BQ_z@93 z{d5)m_0r&5mxg}+wuY)g26b)0#XOCcHzObAYvZW9^7H716}B2$4R&eoUyh7d-aCF{ zh^^jY5uP-D-;lw;Tg|XSbtBT-FCN@2G#UEX*MEY!$;IMom`Ip1=p3?Xj<^ul{%-L+ zrZ8jH2vEr=V(fDrua>Q_8RHX&5XTN;>O5FTc>Lv5%@$6;-Bf96EG z)I8#rDRxNqr?t&^008na&KE2uOVg5|IrWw(QnnB5!stK9A=@;3eVJ5O{4Un|XlgYv zPG*@IW_cp|RAw0{871}=8Znxjx)LhzAW-A4FT&T04#k%fNHgEvsNYsvW4};=##>I2 zQZ`z;21=zT;yhdLax{On=Yk(Dl;$^FW}m9ur1@fI3~&1}$(yIIiwa~rn%;Bs9m6$E ztqJn-Y9q4Md#j{{oX>_?T^gp*xn#bcTXM)SmteWlY+fq&veu(q;dt25@&2Ncp?ZDa zV&7GUd)>K2U=AF8XRfo|YWVx+5-&CkcBwis?@=if)%(o#vRSZ(&c?L6>B1_z*4*n6 z6M2B#?5>v=klU^o;`FjYK-8?fx1CP!nZCQLA64!5fkD}j9(u2IKXCi_R zS+GmB`^SYALgiS`nN?)A;#i4MRs?(s7tQEYmj;GaCiB7DT|h`4wv1Th4GWl{^E0sw zUIZQ%gX2oNfsgJvx%hZQH zHBAkT1~+9@u~w>y@n*c@7zd>s5>t#4nDC8_F=&2yexnArC51$-qDXGoZewcFD9n2z z^t8Wx7*do4;v+f@ok6c)Ozl2sTBz$e^E^o*pp(P0r`MbrYiLDJl^RrPrz^*7r*9}U zLbo|eQ(aq`S7lZ2(lO{MArAn9*Dfr(0#|S$;asW5*Ij+agLb2R{AN zp{9t#f(ZT8GGx1t$IiuG4S}v+$A_$W|6-m$+nMp>?CIPQG{y7FSmNx^fRtfPo4tB73W^d*T0^jiPu;$#A1_d+TX6fU^UPOIIaz@`g{ z5b|gaA+5LIeD_#u`>^Asn#QY@{o;#8N57^1((yQ3NMBSzUzbl82Ia@sEmtLFW@aj! zuP;^~cWvdfB#sT7^@&!QkkP!X&e>h~IcEJstwOm6Vj|b_adOL8o}=zd-3{tgl@{<5 z0muJ={W07f(hExX=eU%t6-fmmVws3sufx^*wUbFS-|C~I0P`l;0t*Wta0Fv#BWe5@ z!FD6hVV{NLKkUzs8qGKZa|zzGHd~xFJvC(0`AyoT2+i_RwZ$bjv;oGc4~?d?vIIr& zAMqdK#9ugbj+N=bmdDNGB>3#;b7O2!nOxU^Q0kh4_$6+Q}1} zVB}2Z+Q>-1(7F=zJ#Xi}?jsHA1FwrndcoHO6|y=%10CuNJ*PHwi4LWZ8R{{@)#|g} z%AikQo%$Wsq6g7To$Yq4uN(FnAfFfWmN-u}T3`0SM{Tv(DIIDJ#AB_&kJwDYiNRh= zme;8%-uDP&?kwkQ&b3dY8nNN;rC_B)u9^)7Zb5X^r=j^S#-~PI8S=>Y#<)P@TI-+Y zK8tpLtuP~IhhJM=&k4JXo6w!smI6>yTr0=7W-D;2E^?EsG4fDxZ#nKL zpzH0Oo6c3`=g*WKAn9u?qN}m#0cE;CvtNR7PL`%L6>c~EyDjX}6 zNHEd=7B^g_?!StzlPG&2H!p~@^r6eS_X=h}vQdzAWOcadQIm{q_IGQkE)~TdF7rn1?>g0kX6 zxnfCfH@f(-h*WgkJI}mpzSGpjeq4-Z-wr^so6ly%%@fzwde2bRP5Ffo)fhI(R}b_I z0(i+-ffTHUF9z;pcII=|!tR@^p^;L(i`XsRbFP9_#V`yPzQ(`Z4sRJeoYY=8ZR1w~fQ;Klgdz zgD#vKAJ&JV;#RhH(_MldR^aNAV}rE9XYm@T_oG#L(e@lSqShIH=HEqf8_Xp$#ick| zq4PD=tuMEi&%-T?KT-W4?i-FL$5E9WKiSdf94ESX zX)a;#N-Z{_>phNBkV|J^`@V%&qx;Kwh<}X@2nbMrLC(U!X!m;h`uZ%=n+fAr1(6rY zh1v2{=qMWPd0+a8Yw2vlnP|t#p9~rR@RHON>CLbj(acdF_Ig7@E0hKJ`33=&Bzn|H zcz6gh0GBYbMo|Q!5mGuWjMk5F+P~Wc?|4x=a7P^KzWlR$DKK` z(EtuGq0HT#u>8`H-Opm2t$^*UT$Uy=iWj$d!c$&n(r=Z3*kGT3ZLl4~EI-x^^Dk%d zk2C1@llS`|;{LV#{D!{$U+aJ`a4_uuU4RmVK>Dx3|Mgw`8^xg zYnShOHM`3jCZ6cxN}T+fhR?!YZFLFctQ6hRR}d@YELA5`)h=Ftb znqzdWC5zixy!k`2&Ot}WvZv?tJakf@v30J`J2y0=+Q5^>`B6G6omL>2Xce$3GE7TS zXHf9R%PH_bh`Z~cHv4T~;8bag6)0{cxEFV5af-W>0tJe@OWIOs@B&4HySqCjI0PrS zyF;*GxxDAS`<#8}?sH}4%Ad?+27B^*@~qGLu64k$(07xdVwfaVx_%+<{S9%+e+#&`KfwMvlj1YFNBWGLtG!vREyjN2QS zK2%?4vtWndb4#Ja^uECd>mdv@*G`7^Lxci41v*3ks1QeGXGnbeGW#8Al>Qg3%srTi-%!Z7Clo;kZ)&^hyj*iXYyyCNaY>62wi>@_pNnzVQgVS5#zHjZPAB ziK}xQn|>MDlC#zQI7PV|DeYtXAbrl5q?{^+h7s=bfsM*ksQw|?8V=d z)K>gSLcOitnXivE-Dw67PD;nlbI(TA(%#YkLd@*pG$BqsUF_(TFAV@4%qYTW0Ffca zsbg1*EZ9;WBOEcz0YLi~C`5oGfw8`SVxQs^?oiYu$wxcgJ1h-Le#s zOyllWx{hABYF&@MZ?DAg87p8EGSt^^Ro8ZBI}P2LV|s;9x+mY=>|mne;zYCtPsL(9 zL@a}QuLCwGhOvhY(yTA}9(}3`vF{N%7+Wd&lP#PxwGO>Z2G z*#>K%*JOkbaY(#}!#SwOiFIHj$I?okojHM?3_HO{dyN(VP+ILPS!qjcJeOd&JlPX! z)X~s!+Keh_sfDLdI)g?}^J<9h$96l;7KI<{l$4^ZyL%n=>T11YJ;;52)4LTev@Dfl z+egnrbNyL5m^QZ z`y*|3Q$XhY`osZIv;(cpZ;@6%JumqChN1-U^wstcB{;XKvtGquc50d#3;M-zsqD=g zDTc%FV2?Yx5xMk!QGc=f?W$%>Jb|WZ*NZx424==+TJhi|HyTlyzG>lJ;mL}NUtG5K zv&jcm{s)F+;5Uf6mO}NV)s3OwMP(C@6%e)O6pzL8i){$>nqaiaDw=y?;e|R*FV25~ zgR5usnucjp6*QTo@ltFx1(=;ib;_iI2r?4i?48ttRvy?jmJ4dXQj)sHTnnc*^i1RhORLEj5mA zbgsP*Dcr(%IYOI!+hB!~a3%Yy6CpxxRH#g}S# zHyScZTu5Ub3g<>*lg*yPOmTjf$B}!$LFSXKxuVLXP{cGe6X+lzmF@55yEvtUPGv4f zZWDAtGP*F3tkbwKRIrL|pU`%?AHbUU5QF|z%(llWTSji{;n9C-micNS!;+x+i-Chm z_OCSah-`*Wiyd_ZvlD)INp~vFw|?Qla`E1R%!|d!KpiGr2;$RL5FLbF)Hi1xB6#g~ z$X^J_+qZjD#pFYOxrkeD?WQ42zqJxkgGW;LpaSXm=DlP-wOH zln7-|M&0SZSbX%>eZ3*vzVcHHy!pXY>l4URyM&A)=U}PX42zi$($UST;SID1;8- zPEN8qzxcuBOzPICq~6M=ZZo&}^t{<(k`r&|w=8z|zD}PIj}zHKdpA9ke@AfGZQh&8 z{$eG5UM*|k$U=sA1_pMK=%m|4jWySPNunydLIEXPjZ(h&it}cPlY1cnpMgco_(1Iz ziTUT6c%lOMRVRTFcb;uj$+M<*w)8H?7Aw6CX60PF}fUb+rcI@hi-7&-$OF_Cg+g?A;q7!u)`Rq!!{xSA4{u|rR4>N#M9uC*hEQSaU!Fox_ zKU}<|7sQ0KBhqee7Y9>Yx&u*urg72tVn5m7nl(UDp?fG5ginv4od^wH=79Y&$ ztBY+FHM_b*oYryXIB0{ae`9z`VWthQQ7_%|c+&GecPm?TGhaVUM^dmY4Y60^VC{gm z4^YC`SAG^g23kSUoKa+c?^d}?F)%9l=oE&cPT;MGS3SN$rkEcik96Ri^_XeB)AEIh z%^c^}wbEuZsd;6RUyA~Zeo#k`AlA)i^eJ#R3J4xsDS2q%Fs<3sMaNI-o(7u`2hjwX zcw!c4j7PA17EPMgj(Qz6p;qW$Eq?Bw>ctBHd`M}G2tQbC9Su}0`t^rFR9E@XhH9U2 zJ&LD3dbDrvV<1dX`%C!mJ1%4;%=Y~+H)+nOa6b;|d8R<^j$A|4L|2DT1{(dZ<*#>~ z8Gs*yOAa-^fx$;c<1iH%LyGxVz5bWzKjUC{xWT)^48{W!*1=NK$TV`u-SJg)a8XwE zBmA3s`DxbJAStfVz3~~5sNi2}ETIj*BoMdT%}bhgG?IqU#Ja`W!by0d>sBj7 zF!x*$0DvD^c(_U@t}kT{c*bR@S%vSpr}3#q`}$vwC5~5pMs&l~(q=p*=S}}ainfFi z7w$8KwsCIAF(qj%>d)NW;Oj}5H!0fh0ce>ILqGLqb|k!HVYRkNv*Hk6Kk1WnhtomPTFf+k>dJK=kM zi|mePExG*%0Bv>K4Lp9^WhCgn0`1{*rSBpEnO;Mq==J2m2V48og*Dl_k~v|^Pa&R0 zd+5WuKkt>1HAFI-qMuUvapQ57B<4aEwWYHqB;d>&h_r+E(Wl?+GbM5GCun6+{?`=D zWuo<{3k-7q4c?e?Q!>-B3m4n8)h;M5 z_7=o%__}?--{vH^(~!8^tkt&U*TQ8oDWnz>!m=8Ewc{knO2FQJ5(taC{qFPQ=^%Q? zU@l#eF1JC?cRKE+thg$4r(k|9p~M`-o!O^B!w;JF9^K+T5pE(O z>w!yf|K~$$tNZj>%s^+HC7kL)vWPi1WUeQU1L>8!Fc{nsceR-cGM`+3Ep!lxZJJo| zl^OuF%(CpvD`|>WAG%O}ywuXB(ZS7>ja?%mC95!+TrZK=dyWNM6si?5g>k@K{IM%o z?L%D4c1<-I@8+l=#Tn5Uj=R5aIPz`L^KL`=N!FYvDZFr&?yC+bS2nn_imywobvj;T z^=};pe68?zAn$F%aO>=tZf3^z!0ray)SSGd2gvM_jtp*}3O<_tiBnFSwW$)4nIE? zzMf536Af^)l_%TOyH5kLw#%x}Z9hhn)DWL3zxA*tnd;P&f^tXeyxLK(&`!iXcxc3) zo{pnQP0w%mNJ|)(fWZkZXZsx#v&eu54I)3)9tGNl{;qf0t0NRRt4u-6*Kc z)LG$BTyxIKGIfJxC`WU&wGjvPye{&&R5;XQlNAx;KsAG`Oexc$CQ-T()6K<0K50Pw z-rIi2HN5&PGQy~c9$*UXJ}~fkM#pU5ZNAWqcLS#zH!exO#aJMPIgt-fd&{|aCDtKv zX$ZNqZ{KC^1aJ}4Y!VT5k`IjjBy&20d6aQM-?YAd-UHo|h<7)g{XxS$)GcI44+8`N~7H3!QrZy#dP0tAyuyZJk zJiaBVkW%h4(E%cz2{{~a<{a5iAcWX*%P1`0f{GmoYK25tr_#cmj>VNv|8Surqxaj_ z$-?SqbyW;f;=?sAZZ+?!I27Ko9Azn6^v^KWindOeuD3o0w&`#O3R+NI~rWuPm z_e(pKv_E$XXtl$y_E?W57yWi_uga+yY$?4II);ZQ=yy;>-V@_NF5K3%28rhCjPI2S zV7?%`4pf0FV1(^aQpda!84nx`vljwwWIqfnlfJJx4*bU?w$ z3yF&M^HI)I!*D1Ju~I;+dnv*B%xlwS{in>08q_*nS;)j-V) zcLm|fFNn_lx|ydCC##!ivqn-Qn;G9{MBFb15A3cA>g+9>y`azI$vk#i`v)gKhLJwG z6%UMR7Pfj8Xr$G+TA;mc`)43nr8T{Rc)rpwub)uNnHm@a%KfeYO07O{PmN0PTGLlH zp)lOz3egA-O#3`U43W?2pZ^@>Ku?e$|7^AR_qnBdOasm$qtb~cgLCX%32lfaHBpoV zF7SApV&P6>G-h}+gstnARob?re%PGOGVeqh+tVMuRr?qwP-W3U0%>IdQO*ojsQM}C z)@ti*p5(~RNJ;%B0wJmQ)aQ(k#&zlJ1GESAFXoEP+lKUOEfn)Th7TZweS+h4;x!zx7#=gm$@Wf??rXG zYD9e^7B{`nb$9@P)i-edw0=7wAye%m*iez@>}jmEs3f>|c0gH&F&bzX!YRzl>A}RfZH3_>8b=_Ed5L>4J z@0_-Zu$U@K0Za_H$pyiB&!2yMG%3)joutCpx1B;wTF+4|ed=?L*O?Pw4A z52}2g^W_jpnp}Sm@odS?R}t#cgR$dt9*bKbZck4?MA?ew3p4EoAW;+b%6ML0wbecg z5#uWR{Yg*je!sY@3jzKHb<%Ry4lKpjM5kvqaY)o%jCXFTHI9j95IY6zDX+lh@vlCY6Pc`b z%}&nB)4)p^7{JiZn{~+p&R!b`Ry3$>X(09~Y!co$na|${EOZLX*FB44YoX1JIPE7& zD-otS*fnESR9QHa+9G7-j5oLmHmyyTTh-x>-S~VcmS(oQvdaumHyQR9tDkQIEojF( zP=nkqB~JG^hb9^t%WCvWwB4Rn(GaCwSH$@JNozaQfG3dVgXFg4*V&byJs_pLg91rZ zY)e&W&BIdf&%(Zi3mIccdIN+4m$q8cx%F zgWs+u!(;d|L-CsYM8Uf?2wf^%!MRoyA7IB?%APyJ)>1Nz+r2zDA)g|Txz?kvrP6+! zf^Ikn{B-JGX`s}&#+A zSOUPrtrdg1U>0t%9`hfqUzt{Gd?AHO^Zo|S7eTbg=ts8>S{6`3KD@4!0Ub&`@8eaK zQav~Ij5GhJ(qfC74zii2f!<{)NTF!dyyM#xv3fpr70vivoKbjdw!}6+pC4QqGw(>E zfApXQQP+*mUQ2*yg~8?+;*X`GS(P9ACPsZM=&>5py)N2A0skIyb^4V`=T&X7n6K}7 znN5rqz(u&sTh#L0VI8okw%q;rYI>(hs`IO z8%}|av!m>6pGqYw<-9%0B73eptk1dSPI{(3xQhUi zU4KO4utr&J_1_{aO;MUB2Pyqwg=XeMG?i@QS$`0043yF*(uekZccicQQss1AhChcz zcGhVb+qPV|mL^Gc%kb_A)KzWZzGM+lP%-wwD|LnJbJ&rr0)!yt49rR$FBU|}f5Lg9 zUl)LXSWGQ6*ZhP*?=v=q1?PS~)73o=0e$iI-;w$M>5S5+_YO)(_YxV7RXg2EuajPZ zoWUG*L}E-6@*@f0#$9|+twg`tTy0uE@rJ)7hxP!f29%Mn3aoZ zr#^3PQw-Sbt@X3enyq#v(BmWK5#Osj=Eab)TK6{+_J{;l8Zf`rT5WJiG_;t=9?6H3 z#iC5$)$~*O?S2shZhnt6CZanFwOxop{M7`Z1|3n_cD?CEIiSLQQI& zN(0ruKIhGw$a6pU5idyY81^`C8=gQ^6K(n+pt?!ogzc|`i_{CGINITFMMW22FxYFh zJ=aFd;Z)qHsrkip(D&1+GZT%;PCli}C{xh=c%RbWc!|4A75T8>XZ{CcDTpEOm4Fi# zamO-Z8eS;OEvwi^(R@x4N;RZk@eDu1|T{%R&bUeN5MY=M}i?2k_>N zUc1J4fJ=zN4WqLOWI&O`6NXzi*Pqai?m;eXa9VPE!2u z7dXn1=9|xdfAtSM{J-d;|I7N?p4V=LS8A6ekYNyW*zrwN_=j?0(}8hP?eDqDGWQ6a z9lv*Kc_@ONOOFy^q-L-`U)a14cakevBS!#Te^X`oLvztLGUNP*C#!3<^rz|y-7vSvNEU&!RU(ym zXZIVGyPtK~`?HxtP_HJ_N{q)r8ls|{p!Q#cA6gpEfQ?7Nrwh6xE%Q9BDxbryJ1-_q z=58)`sy2sgGs`IW5Z#MAE$DnW`->T}S8*(fjp{XJD8H4gd^3-;QekJKu-uWO-~72} zc0$Lt&t(gJY=kH8PRf3GpyU3BAzr2byCKp^!Gzy1ZqY`-Ar0I6`in;OHMq>aviav-cPyJuKXL+f992V-1ln zxMdj65u6+>rX!(1fg!|Q3}~LjKeczhQUIyFQu_3*K> zzRt0?65*ggQw+GC4hffrohQ{(DCmfL3Rmyg{8SPXwj4Yf_G`SYw7pGFDj@5Tp;Tn7 zxHDZO5~FU_DtT>rJ{jXtwG_d&OQZv%PkhG1&ce#>yvX2Oyuc0}3PXAGwfJ3Y6Y4Pb z`TDL0J-@pPGz+tiOnK8_I{PPQt^1-BP+xDp-+s;?B8o&j{Ncr8y?4m?P=}W9nO^`2 zA2p)2FaN&8rc`#OxV0I!aHhod(mXr;-F<27^tN%}Ezm010P$ID==wUHNF4G8%ux2_ z{9$-`QBB??&lW)iWgR21^?Tjbog(0{w>Z$_;O3U6F0$!mB)GTC|1S62-p`*UVJ}Wo z7kh6*#mWBU0*&7|b>^bZj*Z#XgTTFQ{#s2RXmoD9YqUJS3c{XufLugglkpbY|M0(B zwOY%H==p8c#KT(O49;Ig`*3zFPga)mY!csa-bLTQY^;9so-0U18b=s*f9DzmQcy@` zw(v^p6Hcv?`h54nW!LiOR*6cD4+O1TPrA-5)qUq!%VZ?^Xb(w>+i>0hvVEw!7bpq$ zBClsTF8o%;SO1v{6To2CGMG^ae;MH>^Po@`gL8`56a?hz{by($78A3e^ArqGwq~IRKAlB zhxa>?2s1r)4p9XKXSEN?Q;zNXm81-~xK?;7_E@O%i4r2AhDacMv?ZxAgga94pr+;U z?AJ|boOnsgalAH8?%%v~r^781G|*TxB;2?*oyAk2ad3DSg&d~1I?Y??{OXN47(Tpw1!cJv-q;|%FWfQP)wT{@$c;wf9zPlw3wgf7V@ zhY$hNgv^KJhSi@?4ck1h{u=*bv4GjQ2WfUHtPddDEEBnTtFSrF#Y1Z|AT%(_f-a1K zTLct)xFfG~f4?Ab%R98_UOy#Iv2A>++W-y*JI83$0p(hwiq%f9F3dNKo@|tw0!-Jk z@cE2}!F}dkTruAM5hdi)olTSn8n098`RNzW*_4n~u*c6(;Au z4T(vE9yLZ$W_{kLSWns2^^qgt}+ zETdcM12j3;iWVIYr^+|~D#q2dlFge~9H)ssnacb<%aoT!mMiwXUd$Yn3aYOGt1uPREPL^rv_N#vreF7c` z7k=Gfy%r_Q#2joC-W@>yV=4}T%W$kG@r<1M=-~4^pX1& zH6pM z5n!LVc3(eByP!q%*Ic{V8TdrqOfM+NUcL3XeoCAbauD>)=5lC(NCQeCMxHmmT9?!V9g@9N9J>6FIJG0q_1DoZh3fYy1_fSRWsL{=I@Rkjr?A{U zjKgVn;&+xaP$#4Qd85|P17_jJ3)9;Fny_?D{{^lIg}%KD`9i?7=dkB{mn=I~RjU|= zBOVvmk7woLK^_kJWICCZxYz6L49SM_^2d?_l&r_bs`R)VdZMUd2NXQAD$u3L5J;`N zjeS20aM(thf+kAYYyH!pe{8WNe%&aRt!?xwQF3qZG8n9DK9vheVy;P%os1cX!6%}W zSMui=`&T*&p!q+jqYHcg6CK5|4Qu}clRIZ-i=;D!#+g&}3(gJD4c-K-{BPi>YGYuO zdDkbE^KhEsQ!rSdt0d;KOH0BxT>NyC|KW!H%ej9bk8<<%gO%1rb`isEcL9@iJzX25 z*P&FDVX##&SV;^dGF9j<8GotF5Zuq3&)(uDB-FN;8z$@@SolnsAQrU)q59Vnx>bB{ z6A*2tB}{)JADt937;%-;pB|zcG+)iCwHx~{xYN$+=MWA))UVChd6yd%j}(7iE(>Q9 z%zORj`Z!BcsI_U3$&@LZW+-N9lTE~C*7*CXcqJW2ijXCi>0;jNQj4xmiTh3xM1!^x)7 zuz3oewyce&9s7$6>N}j4J5bM~wqaX`qJC8_dgm-08ABOc6Y8nl&jo^9_mv)$kob(x zii&j7Y3C~fW=AlyJ$}rScWbz@mN19Jw1gZQJT3<_gLBWGj31;+L%Rm!zo`pVVStnS z{FJ(JGNkc>Ya-ltA18&G6xZuc4ySa`2mJc3@c3?5=z07KK8dhAZ_H(m@lfp-TPWhp z?S*5FW9&OAJnz(IR)Zp%OpzLiwvXrYgjkr#Q3!`x*uQndLR@QR9sIsV39Rs`yvqXW*)z?)C9)REa8v0=)_I?S#X zl@fW&58Kj!Q;AqFDoYvZD=Ut2z`{8r%+tRym}3sHY;7Kj1oGlWhwBPV6_Y8svfFV36$Mu_Wo#j$=$O)kigxERt@F44jLUVky$bH#Za~Gt+D+ z`}t!UQ`Et%~aZVRaV0GqZ==k*&t^;hJv@0frT@` z+Hk`ici?rcb1)g-u}H;1s?&h{nzuIzF48**x|Hml{kziwS2kF7X_T6GL&Vu`d}SEF zt%s?amgV<7=b+C(G9XSOd@Ri(Sma64)Mf3mSA)>>bxURU`kbgvEc&c-R{L( zK7QokN!HxdK??Uf>E_b&0Lx-YP~Pn<1`UGp-7o|^0H)8dlXfTqR8nWy#GWE?@TldH z@A*eFe!jGh&H{WKbnvP@I~A@M-#Lt2zuUE?KDo@&KsL&j1;^XpL0*ul}MdEWHI(KQG>+6>MX# zj;o>l5>b~teX}Dt8f8R!f)EbXDB!&)t8~^lY)h}CNIq4m;OgEe6h2k`degY;piVXL zojHD5eUnu#Qms6!AUl_VgO8eK_o3jXevu+Yv85oLsyhF z$)aF|=7O<4))XM{g&8w{#fU}g;2JYH*wfd5h7uR|KO9zI=G0-t!Wc+|M9khO*3@#- z*ABo3ZhQ8bkAs;YY`3HJ!O>hLp7;P50$g z`dXY?^J2BWeZ$-b7Rs2l&=##A$YROuZUgJp-k?jkPU6p>_Mvx{jik7KX9Rgjj|_VO zVu()H3ao?Jsf3TRTT^kJSnEY$H_-LU-X3df@26=osWKx?<|T$>Shit+{^0OWGelui z7gba4*}%4 z^A`nX3bXHqxrF4m#o7y&U*`YqtvM8Y;<~fb_3``zX}eENP5Lr)C)63UCE{fwXo%db zK#b+a1Z{6o?S=03FNno-{%RIYzU~X!Xww)dr-T6jsZq`9a`7QG{b66~E{;39YspWZ zN~gHLi8aMG1)Jz$$LpdLvhpPV$IR*iGPHNqEQSeA$e(t3#P_xqQlvE8Gv<{p%`xaW zC0Y`JgtdrjebmoGZDf=i7)uGJ)9pv0 zmEIc8|QY3C0GcVYTDLIgTT(9Pe;405nv3&dsN@UMasJbK&! zwZ}_O(w-U323z4w}nZPcU$a9_{YDfNz>fUi_bVdE92H0s=s@vMIO?q%f2_d`J zlBeg^3`+g(AhKrqwk5na@bt{NjOwgCc2$IYr=)0()VdmMb>qN<3`XZa{^!xl=8ofH zzW}2tlovDg#MT6b{LoHnG+YDYfuaot{-S9Am+=uNCDyzgb*=}AJxr5Xhp&p1U3l5W zoWqRBw6X+$rixq&;@S}LbXW9>vbo!=b}ESzW&JV|OAhG>BiSUK|(``;P`?~qyfgAcyn{p`hs*xrab3edC z_0@iToJQfLHLc&@e6!SRk~crt&@!OhYTkKdcUo{TTHFMZ*1I?#%5#Q9L?lc$xgeNR zYkhOB11L8V1!DR8>(ihK5Ci>01O1RS5RkA{IGErc5Lp)KDCh(SZoh^AbxdfFl=VBr zbaGfytpj27&B<5m7^I;cYy)nEwktG5r@Tl@=K8ZO`8oS7l9@#Q(v3G$9X0G(02xb<=oJS|pt((~IVI5hw~~~U z*GW88C40@mxmdW?D&Lo`e*@jWT-T=ihtO%UL8NYTb5p#OiZFSPGkJx>bESwi@S1M2 z2RCc0){w;oQ_pb9EIwNaa9HW16z#jZx|O`e#*~xI4?2eS^Eo%n+&U~yFJ)O&C{>5G zoa}~Ev+BGSI|#x*g2}skixTq={~WRj8&C6J*1%K+sQqV=lY9Q4FNOv3p^r53iuGo}9*4a{m?E!<#Z&~o{nZ07SYduM+r~k{ z{jSb3FUj5=&uuKrY|mZna$jI-t~=8p++3U;ax_{eX|A>(q1EFo0%GUL)VU@b5D@eU z>MU2)Sx5;8^k=4%`b92(o~I%{nI4DkSo78+tW_IGuKs~UP>c@}fCa4z#7UJ?~Tm)96M4ZL(+ z^mbvBOm>=y`eyhtUiTlpYuhtgv10DI4%{i1WV_A4#6bgn>N!Q(gnU!qx+7~TgY%N& zB7VD^xqi%aLBfGKxuPD|IVEke_z*HTG2Y?DiXeGho$i*%ana@&SA$P8V+y9bjAgzj zGmYG1rDENG&L)&pTypa!%mP14)9u^IbF;#~!J+{07xR6y(R z=v&X33kj2b53lS*$y%CbvwgGr`_llIT!&9s61f68=;+htYrFJ}{@ zst#_dF9}2*W^aCT#f$TvW*zz%$aJ4cwjvxV;N9Nr=h)tPSdsCt-W^#3*z1jeV+S-GLz|7VLeA`P zQ~Zm4nKc)^0?PL@`Tt`6RIgF5y}UFQR@Pv2st~A5AydZc^QE&Eqlu7GZV-3CNR(71 zVYDT8jb_Yc0}KqoO_=i;Ec!*0ss{ zrIEf(7E5XN`=VWyj3cP~zqGyH%8qoTURz+9-n)-9mwU4fKDW^e>VtMb9GTCv4P2<( z#A$IYU!Od#tX*9_1lV@qEdSVdAXrxIVFU(5sV(SxI0N3)JU7YO+Ob_MkwKYgN)dfj zh1!r1?7-Gb>d!0KSPVS%2g?HejeY2%&^ue!Im**9aSM?F=B>q{; z$RG%*M=rDE$QgDt8R}}$V6LT+*maLT4QIFC+CK^O-&yon+m+$tO6x#)vtloa)y7L~ z>fOzG`Pn4;4JN?0t$K22MjDca6!_WiOq^1oQ=(s{p+~c`reC?QT+MY#G6uS}`oB6X z&ZqXtiCincs+(=JxH+ntV&RdA+bH5qyfWK++r-}_>;=!>X6*ia^g>w|0`GMlbv{ae<)LQ$Spw~iHBHU*wt)1C4!gkJ5J4Z=D zM{N|5#e6c~pvcZ`t^Q?VFT5XBZljNH^M+My>#^Y?(u*6mwUxSwE#{wM=|`om*Zn%G zH*0%ePBjFxzxm;U?_}|kgjmr0K|GZ44Do8adk-MB_}&l$^AJ=j zq3*gmI3vlVvuVh%&VTU-_-i2j$jd9==a*<`_bxmfm!%1As+yM5BhUGe_=&s5 z>(yz>{iH#YmB_gwzQIT|v^25@Tm^=5EI)))$qkW__D`V%Z^vF+O~cW^@@VaB`Tu}d zG?e_qW5GYZ`ahdU{cq&;vFoFDad!c8Xcm6?%}Dt6(|;4}^0G&-W6+N8qaxVRuJXPN z9gMz2Eo7rV{MTN9S>9)7Q2qS^0a~mHF8dG9{_%7Ee@dkON=Tk}^ zDUrWiZcn9<$QxX6=TI`}x0}%s?)iBpmXJS=3OT!Jno994loBVCIcyY}TTY(c=*Rea0ioOSEf6;x`*-F$A);B9i1&cfuOq|jnMcG#3~vU%)(Kv$7&N5!rpL-26h>Z zZI6C3jcYy%Prd+}$(EXw-TNz@KTQE(b}x@5d$t}UKTBWLo>JST8pQXWLOj40MF{@5 zzAF-u7<9@WSodOo)^>U(?2kj2c^cU`i$M9!G~dggm;8kEIfVKn$RVLo&|iFNF<-*lX2AZlqFJQ78$PEIR2 zXJ1npDtJ{JnxhV?w`qqw(P`{PCAX_ii%RNr3 z!RkJWC*en|WPD~Wu1?=VA3*=>q|B0%SUk*6DuGj?AT82M4>!R0_K#KSQQGYqZ2FXX zEnX?*wY8Vm;^4D9Zao0s-?7zddp>GY-Tcm$K*_iikO8U5zeQGAolD`+1Kq~!j=2ma zps?-fjG&o6=x}@^-h2&xQgK0m@AQ*I@BtB`_4wt(V`iX0QCVs;fjhOm#Vx#Q=U}|9vR*=dbB%_dXZwhQ+)UGFKf9M;oRFP@zt)4Gql?W9y|hhj zzb`EkmVQ4vu}AYW>rdLFg15eU4^1Ix0UC?Ko&tSt$7Q0BKCg%8;n4vN(j}j+Z`vig2;!2r)h{ zc1SM*0JL47L!OsXQPy$+dVRcU5FGcmFChM2YEUf5{dlKOgB%F}fV$Q?cph$evNx%{ z@@P|l>?vyK?!j33@+lxQ~P6oT?I6_SoN2g&cyKwon zq??Cu&KuEvft$;5l6y8bufl{4Kl&q27+%yl`cOd{MQWYcsJJBf}3O+(F#OH>?R$ipP_^YNv*;Bz9AD@~-*O32~!A9VdyIyz<7|avQ zP#1*q-krioneQpv+cs7*>PnQ&y_hxj@b0~n9!Q%qzEUVcANM(WVOZ06UqnLFX}&PK zNwMIxua-1X;C|`fDO4}N>W;NvL6iDLrnlXuO(vp{R^}&JTyKSv?s=?9TS=Ri(-(rQ zF{7(YuiA&BU_b`f2&2#eY4Yh1?xfl>uE7WNo_sqUfik}&g*dG&^BRX+9i-$qG=@1$ zep>gK{b|7}3*8i%ZZPS~Rlorwc zAje<;%_oni+^qaE|CvL^b=u1ecC3L;gG*Mnuyhj^*f5j;Bf(fiV zPlkVVt&kq|KGx^f_jc6Mi-gwEy<%tIK}%WlbyuIts>i}ZDU|C1`&8MjSTq|G0^zgt zNdgMPp98vX?aB%#>O8%;H+ETS+Q*vh{C>9MFc0WUPnB}N+`^v}GRxZ4B9{=QH3itY zPaSdxZ{cZujDn7L*7|5^V2DLA+uYaEF1D(^wo@h=;HTd9w;|>^#_xX zVlzZ!{&!io`sUkp2q(bEMew@t6+yYzeE*~G>n{Y;^HtT*Fzq%#t8QvaFxSOcrk2`Z zLYYARl_zLgo_Fz-kf(@Df6g2J9L_MZ+Nz~iSbZ1lgGf{QW!B}e<;JVbqsHLTe6Rljy4`N9nkZ8EONBuCN{Nik% zCf7r_FFe4U^s#FAT9(ynnYc=L!G%* zRwv$lzy3ms*Dhj|^`xM3J-{IDD9neRV76{|+PJ{VOCPI=M*se{cQtxNutjU^JB`EQaOTv{Jdg+h-l~$}8C%Cii0Z+bV4aOkQjObj{Vy+cmG0bD z_B^?a90M9Mzvpb>b0Q|2yvAD(DLzS`Ls`{6tdfFod|4`!O5|2q&GuY+X8`Uly z{vY>FOXZhWy&IBA>-#|G%@`lAjHB?6Np+&e2b**+&-wApEu~XAdh^gUH_)@HQn!h* zZ5_yVIrw0Gi?{D{pQ>(d6*uSuTx&70qR+}zWq^YpHJxVND)!P9rp`U|N5Y1+2YMP` z&HSSm0G6UG%0zTwYr~(DQ{kjMRT(NR&4Y$U$1-#NerHjWs}INu8!~oc7rzXM%Z*EE z(6ipR=dCJo(`fv1P{?ZgOuv& zXHRBU6!B)(fe(%TDqtOC-6AG)WEST8ZuO}@zKuOU9qmO#enemm)NqhaY#$S+WU1r} z)znM`)_ZlP!&BsRy>6voL`kp>{}&uTa5{=V;8A z*MU5pxsUj$=9Nti>TIDto^%CVOY{nDj$|T_OL)3Kv)!^03e%M>vQzt_l@<^EDhJRX zQIS7M7OSqOgy;r>a-}Y+$e(=)iy-C=N#{|Nqy$^P@!m+*C2RzY=J!l@MRn>ksYrMI z{xUsQXR#2)wFaFv(3Xt9uQ(&3V)mZfwNbaI;TNaNhu@5oV`)(H@}WPxI|v+K_XSj%74%_}q2&#mJ2y`C=csm!;^8(|6?(0`mqjIZY(s7)N}NRbmoZ#_BAN`ZDbvB#R3s;M-wAD%Mn+WUBWvr{cX@id{X()P?yl01A_hC|X8v8A*{96=4?4?R8D?(tpW! z@_*MQ?iWRLoY!Ee;NQ8ROre82P0=h5T9It%3)6n3+yc`$8*RNLn zI1yu8mPeP{@WxDCVCFkv#0+sdyZ6~)vEIg;o0df+c!mqjb7jjCaE*?hC-bJUdQ^hp zH%~kAMU4mZ7^a}b#+Tp2(cEXEKeQ_({q%tuu!%VUU7e$vj0xt>XlKo({12EGN1QC45m`C^ zbI$OM1KE@=R&Q&$#4wfYfa(+`01!D})MUL_&?dP@?sjv8h_7)JoB}wr&YUt1*y#-o zm3_io+a9G1NX<}dy17DN=%JKh$jx?xjqHpK{*rK#Hb};Q__x?sn=9YcP$Pleh?6yj z6Oi#DSdJY^W`o_eAL{r~r8fo3fe#fh$hZ*)l|6QTm*n$=+Eg}NYIeMYfF3ql)2(z- zNKw{Trp`8&KVTWTXcZ)|rk8e=z}Xs>iS<{xLtI!F&Z3ZW-CN6|b`!s8RaBshndkiD zBsNJLPs@?i7NkOtkD-gz&?Gz9r!aRm&{>4RE6rHkP>(ueBo@+R-%4uv0C;ojHQP}x z;*l!^HfoWj%k8ip9DenduO$HS383{2)r9Oskl&5v;l1}ytJ>&>Bj~KNUhq$LW-t2_ zqci)^!rTB&vm1|oRq$~0Q0d@r;s4fWdtY#Rb3dQ&VgJiGC46(=UP#&@4|c52YgBcXnL^SOLjZG?8tYgwcqSb5lB$M_>`5n6H4D#Sj5mj7t%hj;%$YZ zNgI4<(CQq&L2BirV3WC>7BA#$5u{>z~XS=A2X$8i^WNpM4O}+ z^pEAy55l4o=ck4wh}wvcBBTIHyrR(9m43Zd;}30zhi>knt#Om}lmO_@Xtp>GMTg~q z-*qfP!&Kz*B;oSHP|ITO2CHlwY_1()kIC64sc$#QdTxIY5-h)z9|l2Rp(e5H#QB8~ zE8qgufhq`m7GW*$tz~hrGVac^4=ktjbuIs0f0LDKIMXhr`)*VFt}!P zQ3S6Ee$n#n>9RzBZWu`$(iFMOA%dI_dQ;?(5`8y#KQhS_Xb^`wYf<0BKWgzgZovG$ zrEGnQPF#O!@f~)C>+R%?VQTT3igbi&s?pqRLZ&+%kh;}Af5fab!`r2}YWSh&zDS0L z%f?RWr5L$+J+gRhcl+CxF1yv@e6onKd3faf%zI2LsC!&F-wN;Y;GKs28d@FV=39+# zEHJJXUoK$$j%7d}>#NF960UNJ)5Ur~UbXYqJ8}j!fL0NW6KlYT)&aEtOc}4E=q?Wn zjsFI?GGuzERKonJdT&l?r_ZVH9)0rdgBm8mtS{%yUZ!}~i@8IhW4~aFuRV=3MN&I* zaM2$v1l24xj?=lG`zNSE+lx!E@SFO8uZAB~M&r5UeXWcY~Ulw%wC*aeba_X9rUL1rwMyy`Dl%)XlVsifwY zx~)YqP>i-~j$zn+A)l-TgRFD$;>1^N1D?S20x43GQUQ|Y02`_1>4p&qivKEW* zwk0S*=U%Wh>;~gymF>$~s(zpJS*llBANe|J;gl}%vdzb`9x~g_Ji^Fn)9sA2icSf( zov@mvzE3T%t=4Gy5}0{1dPT3@;z1iS*R>Xj0Qfu2!C|s35YFJ%GJ&O&2|B9MhRjA^ zrIaLme!5?hIwRu2CY0w!Xg(L1`i@I#2k?W3XlI(oWJ{L={#dY6Ab0*%e+oJSy zsSh{|%YL#&s*~92*!SAXzV*G1sb?mB?Hf>Txjh;42-@djNjIS=`;9VjpJfW&3ZXh8 zU1o+#O)pXHRa3ODjO+pD2rB3BZ?|)Khmy0{F+$68^Q+nBW_n`B&&_=?tv;F5fCB((1W7j@IZe-`>NU0@FJ5Ka!wl@aui9uwu~_I|o%TT6Bn7KNlY0|%O ziRv2y^U>t4Y*HTcUbt{d5N@vKQaR4@kjs2xN(>|O0N!R4!%Est%aFflwdW)I^8Hb& z#FD{E`o_g8p|dm$alu1XQH9PBqm>C+&JaWo!obF4^GCXpTAe`jq3vTzJH~e2&asU0 zaQIFywGS#u3#g?s-fW^PnR0|SxI(}2gCwl?*Zpct*PL`yeameci3N9hWM0uk1#C}p z1}!BuZ8jmGPD`xWrqwUegnn&E1!IF{X|93sa?mfw-@be4I zFeQv$b1Zc_U)A(1w|nJQ?3scl6x7sJJ+9qS%Kget`q46S-pHGIFC<$3Sc_jA_2Apw z!jd|WM=Xb5r#e`VX@99Zykuw_)q82X_5X80pm(CfJ&3iW4m`oj+3meTtj}rc=?jL2 z3#+5POT8`aaP4e1jupyS378C;z(Tlt(J$DJ>6OGfdPzmAdJ3XZ zuZy54oavIdGKs5uiG*eC_G1Rour&n9{n*%VfUjR*bG!qFncEL_C1SE-V}Dn|AD7vADMc+%B#fSqJUiy_eQ3y*S0MclII3w zst|LV%%{^cZm2#2(Et{eU7V^Nrw z%%d$b3x;-B(H~kUEyzcfr<8ibsh3wt^klD4yujVr` zfgSK2bM@|`^rj3^j}~JwhJhi6Ft;qMw#o^yAD2`__I{Q?N^JAnfs=>Y$Wi}J zvK*e3a(_y@P4YD$XF|JTV*I*s^^c8QH*Z_~%3+bmZb5scQz+3(XY}Ffo(3!GXv6rE z<=9t5AzN{a)0yLyLcQVnbhoit1C)pP2rpw5QHUPZ%s~+LSkbM>lL610r4_9MVav~T z@W8vN_tF885g<+G(va1TgDE7soi7c<9qLW)qhoAwYCv1r;}>C*$YT>vF56w1D-nez{k7-zDjm}p-p2+pm2_A|%QS~fd2b-pkw+?$mnQV9GEzB|qXWG+x zU2sJnA7Q#jE-Oy>bJ<{HD z75PL%S6P0$`@tH1h*Emc5A;_7uWY-gkQgjzc}ttHKE-AZtaZPhY+U?yz@~mO8dU{) zx?A(ClLY0qZT)$14LseDY=LXY;y@g zdsiK(^Xn+0H;`yz z2~4#5CdP@LeYUMlqic!FNK<|{nZO55TuaoxZ(DmmH?!2D^;VOK+AnSyw}zYkA&0>g zdcF(@fDgJNUc>6uB7|Cq+YS7wUp%Hu1)>B-=!S+HtE`qT44(u!2-9_-j@ZXpg-O#W zqq|e?bkn7U3Ug=nglk~eiuO#$LJ?8FICO)>^;lV`5q#~E(PY7X^ZGdgzx#VB{a)x1 z8(y=Atp(GnHq_aCJ*sFzv{t#!+M+RVpV*i1F6CKq`m93-^SwXY!yek=#(gllupYh# z9M%hEyPvp%;z?)1n|cJ|5>{7YARP+izv{|`?-8m0oBtX5h-5N<|IJKJq7udk6ZCH` zas|3BxPMdcq6^pkvyl9^&1b&-hj#w13M2fsu%i+mklA~~xx`0o$sKldsAhmP;R z-+4SqkVbE1g^i`vYYE1u9r~x?t=Z)*7Z!*Tm9I;jiov?=P_Ox9^O>K&^<>f>ch>%d z(u=jj>g|Fel+rtNiKhAf7Qc;DkgvGK9p0)f0h55wGTWfHj^Op42mHwhBdA7<(nXV7t^Y?62p0g?k9-g>(1l%L?Pk+0v(#`hh)))-!|9lfpaJws46 z6E62}>VywJb$s(m;5UY*eTt7JP>*JHoE*SYMbzD|DDUa;oi7C=>3BRj9&_6dZcI;t z--`@q>1lJDYm0YudWx5FUC7*mv=mHFQ<+$)ffshJp{4Ig15sC=omeNJ>sgPt`(=_` zo_o}NF2>)G6GO*{p}E)QqW2;*6i3gwYqOl!PcL@4*Tii_yvE(ze|KwM<<#2T@vBPQ z?jtoFWWiPTN(IM6>l|itSBERH$EV>rCw;52n_SOhZdu1QM&P>g+l+!hXmkm~8wy6+!8LFJjp0n4#mnuWHKCg<9gkqF=I&@h?%UVeS6}e{#D>+yqfVN1Bi!sQPIR-Y zg-jami7eYn=hsH4NaJy-X7$y3J0#)@AX`SD@#k{2i;we@^%jrTMZQ^s5gKiV{mv^i zG0L+ioS4*`x60D8$5S!Q`knhNi%JK`<#N9{`XL|Ri+F#4q|z$)ZNpM4AwK!Z-(r3m z@M^|Fs?N)x){Kguws$A@a0#t*qb4-qGaLQ!cXRQ0hz`Bo@;aQYFA`N`oS%SA`U4|PM?vAm4p-gX~BNq_K7oME^MpAp>O#Fa)`@kj@C zvv||XF1BCe!O7qq(DmG4A|isOisPOZtA7Dq<)pd}=-dnVMzm^OgIl8SA*Kj#X8Sh` zExCthFd{L7@Lq%zg$+{InB>O>U3C+G& z^VwYZtJ1OI{m(5hkF3|-GzHs8Kg!{3+1|5(2`~X<-5L7hDyM^v$h<-X7T{9%Yhuj# z$-gpU_$7U>e)jFO+XBpJy>t20N4hJ9TE79U{jJjBrA8vHRNl(%Nf3Lc>t@^32&|vK z!@H;=YIg^hCLzE;_MBpk*?oMYexnYVUF(X<1BN#2kkWMn|g8Zd#{Z5@9hzE zq^^}0Cg+qr%eT{m2leyo#sp?Q{UUmNGw|&N{SBW@>O0f+n(-8~Kb^|Jj{>>`2Dz-? z2U9O4Z=|vBZtw2yncggUvz4I1m*f8Id^BHUaS}%~^G4$IHz$@={;Z>@{s)!`9{`ZB z`&$&?7_VvxP%zP3#BaL2?EY9cKU1<6-+5xm)w{R*k$>F7dVDM4D2qeqLVS2HlWx4w&nJdgF`#>6``6POV+JWpq<>AXjR((}dM`eVG= zUqvnU*2dsPTOyqjk6D&+YuwFo9!Oy z7w+$fFjtqTjlR@>BrKMi&HV)c9}ngbmN>6Sf#{6(HQa7+`HMTY#I4fEWjWh^FAQ2p z!q?_lg$arDJCu}&z>Z&DPw^yLw?5|m*k7dLEcyvypNNkzBB!-1fu?eoTrX6O5YZnF zhx8Wj_gu=&5{$Q8qMM?q3>D)tL#k&oU-#*Gr}*9ii)K zOT*to(I85F1v4E>%$<2(U0cc-`7AaYN-7+;T_ce z`dd&j=MUh^;?P^q(Vx@_&AnRMlll4qI*!;dSEttuH!WTD8;|k&rt+-NW>!@B?(Gs^ zQs({&_0L7(unJsEpB{;dJDd(A#1pfHY=1pL1z&d!7`Qe|%G+4{Q;_Jq%W|26I@wxA z>Q*FTKvCCG$Mrl}G7g)L_7$Vb7NuG0{Sidj2+TKU`la#r8@hP-w+{G^Mv!&DisasD z`>ClxXC8lKBqv*VHh*d$9Hac)z95+PVLfq&XDd@O;I@prUd?m5D)xAy~> zPNTYs`jht4e;io~@7ScZJKP}3{WfDC6blcI>rhlJh?M-etWYw7%Hnm~kRW4>w za=ex~j`{e1KSY1$dP+)v*w)qEZ%Lzt259$}thAElyo&ufE*-dnfB5VTe%5!G{Y2R9 zb7Cd((%Z+ikVZTZ4;Xq*XH%2*A4|*;iY3Ax+7HtRp_Xf*cOrBcY<`tPPQ6$#Uuc>t zrr7Dqv^puMWrfmKO{c1&;P-&+pX4n>lq+dc-ygzyn-*xscJ@Hylhf5K9MTahWhPE_ z)>1epZS)Bs&Oyo*`)5&tVO!him7>REimGBBmg(B+m8^Vc+V?go$nrF=TS*b>9Y3K7Xp+6Ld4~F z4x72~m=gk0b_k68z=iur1wK@Iug%%MuwOadv5_16&Pzz&phV;fnN6 z!@_E0csE~KIDf0t>=YLzW1?t1&X)2hnWU?`51a2#3M9U6iip~1dCCrz-tHE{-MD$B z<{D7v_3e7MlSAW&a~Ps4F`C0yFelb+9j<$kOXP`FV24ZX{{D?#2jD+s=*-2_Q3Tvr zeI@%#<#1gRo2(XK?hXJ3WV`n?j+u}F64csg`8tX>GVl-q%U0JlA{4}pF@)2SBV;g$ z@BPS_R$qlLn&h^a-r?OWeQ}(&F=Lo`^07Whu_GO8eQMY#^2Fei*%oCM79TMM5u|ar0v=_eP`!ze%&zdl{$iChGft(+c*lJdx;EiVP2tzRSaHnoxArtPBzY|Jb(kB7s2 zt=yzC%^K)91ZD*OYcGJZ`FNl_ceBR*Jl;ICU6#IeL`My*^iD89cxc;o@re5z(Osmz zF;b}q;QJL357GB4lEOBQcJv2?dN#W~SO6rpP=YJvtx86 zQ5G!}yCnrdvsz*ues)iJxO4Mh8+NOfi zpX7;P{t}kcXL-^vl*It~pf#+T!s4W481tAY-BnTKFgZ+m{CPUE1?MmzHGnR3J&vErIF#q^vcB=tL(?J&?RETyC>jj`#2p(A^&Ofl#jN+Qy~2|F`ti~ zRyy=>DOFKJqgFeI)k{Yy^N#b~;dL!oJ4G0=6kj7Y+54&L(uP+}5ttZ8iYb*GP^=U> zzT?InYgQ3zsH>K@6aW2Zi5avFT-l&j~=AcoLJBj&7+gPk!aB@RpyC$2O4((N_-`ZM0+DhK`-gM$)^?@uC(Po|M#)Q z2Z5w5?~Tx5;6x`_XV2dX?rQg$TlaEOj%F!jKB)>&;W-=5R$Vbt3)nw3jr4|S@p(|o zhe|3-`<6AXc{)~&;-?B9x5ZcQ>Yh23{|T!oe>98aKHe`CvCMT_tKX4Pqb9UbKmV|4fgv=JMReMR z@%F0|GVvIQv{)$P-n6W#SAeg=J8CVi3k7u`6oKk22p(yE*P_b9F3l2E?vKYbvj`5_g>=#CKRL-GlH~mb5EHXLq!b*@f2&SEQ>4C70-MD;?xaMM>>LD0Q z3$rCp>=zU?b4J1!L4*(rF-haaLD#UjLwTKX_&eE`+3z^GchzQ7eSHMf%0>=7;e^kg4hhw(OxckFO!scTO0It4p3utt;_6$br`f>!-iEqN(`7 zic0-p`i^}&)SLe6L=DZi?WQg;qUTdbNdLqq53^d2cS0`0yw;I)Dfpt}6huK)FKryIUF zL?5Ck3K|Ic7dPGlqNqWt!6l-!ENQQ6Vfe{C^ZFBkC8(wqy+(+7R7th%tUoEvM2~JS zm@^#HOcU54pR}Eb%{Y(U=ruQ&m_QY}(n>q)GT-N#b3#|LGpFvDI!%(fDn(T$)A+NO zT$x>VwC=QTs1CozptHFQdDcp&()wyzUwX1=xCeNIz-d<(FEaI6_T5eWRGt~$uhgzV z6S_&&J((E480MYB;maXnwV!?Dp4R(sJ?B5pGZuNkHUum7jNPq%4X(gb>6F*I<`Xk^ z5ehaIHf!cx)56vji@Wd8rR)Eh(ca1P$J{muC6U`?R0&j!HcDE3w3FRw5KZ{hz);ZB z91+B%CGDURhZsu!IKuj4$;XnELfNrym}G0#lp*L16$<7#P;IhFRGyMjj0=fm@5(Z{ zn>`e4JfV!g2P9~H z3|TCfb_x)0w?tmE!xI_=R6dhhJf56@*C<;Wf7Z)PMiQryO}+m1{l&I~GW8fohR5p- zIK@INP=SnCJ(F50x-*aHwxEg5V(USy%5J(M;wLp9?olj;wrD-g@s~ua`RO?2p0@yco(zpeq;bbZg4@VFA-*;slkW}Hr zaw$8KNTxV^(`(Z+%*<1&TGh$NNq)}R;3P8BX@2<>kOF^!^-+!FA6=`C>Nke?tB>Ik z9`ID|5M!Uh0nDL472C$rhpJXRa9|5HX4zJhvZBl6qX(?r#0W>$((tEW+!b%5X1G#x4m(m2Hsy-y_teY$0Q8SO3V*wL zarMj?@&@E3CG+NM!OxCT&nZ@grw8!#(C{``VS5L-Y_JgAPO%xD@V4iv;P)klc~|M4 zwBe02!om+59Xi62D(+Xy&R%m9>{!D8wMou=BYl1^DPn}rE3@rw0 zspgWc#5rM`G;H|uN@`(q&3Faz??QcG9C=a$b_$cwv!n~#q_bhE#u-Gusar3 z!7ebFa%g@L=iay40>Pp2rdxMsk8|K4fl)EvrKF6w;Hq8P5AoWfFLR5DB_5qJJL@sFcSJzYcy(mk zgKSl(@RkQ9$yRdteTC~JENfD$an6EP1MLsFw1U{SjW}Kt6tVcc@>L3nH92z^(N2O+ zo@?r=#vPAieM-Ke$>7ssakD>~4OV`4dn=HqsEs#T20a_@oU-xE$>;(**{bf4Q@E-!zQ~>W2fV3lV}U($0dfdtM46s^~c<8TL%~jNzXGsUA?61i^aW1$7GqaRN35B z<6{dlm%M}_Z59+-^}C8nnVu(_OgaQCG@QGmLXH%~uWe-*ChP5EaN4~$DudyvmTu?6iVj&1%8h+pvxu}GBC&~E;h4@FI6qkYjuGZIz6v6Ul6N<>_v7d zg`GwncQziLE9^x?HR?PcEAcEsn4gAr5V#|>IHrvN-ruZ5*4ARZA~gxJ!-m&E-cICR z2}DF9QTtXvv@p6~-Z>~~3F%0TT6rjrK91{-+w4t*so8X6&{+`C9**+bqzrk1 zIKL|t-j~*m1?U?XbZ$KNJBwa?M-WELvH6lCB|yw}1ldBf54&Wxwp?;*o#!8yMcBjo z2dxfmgv9c@6SO$bZSXFQf%NpR*a*lmg!hkpq}efq5j}^ylZ_!EQ&$|;AJZzE(`m}* ze(hg&mi4;WBa!%z&vEe%vP9Xa+orD2{`8PGy7anZhXPJdS12}oRP;cI%KoGIQQmGx z;9uFIbhmv3dcCCod2Elk0`&g>6?Z;j|I=Log|z>2O^znCrSZRABJnrJ_`d$Hw?Vf8 zizY)Q(En}oqW{>=O#gsT2z}un+EO@PPxT0Oq?{P2De4EIG z^ON^{F3>j-qn^f}4&?~uU$AeNbakRs-@iMc|J|`D6DvAxwJ3sFb<9vuc*LM%DTsb!Ud|9DP?~1 zauu`JXDfLI3A57PvrO)@L!UJra-9XUdDSm_>Fwx`)4==sx zHs{KF$@rTG|8G9}{}&$(Siuk`{`Ljj)L!9#lhOZ4WVGe!e|a)$`s+VJMmMfj zVvaYoQIzpXin(Y%#|kDDW`!=+UTI5ZrdupFjwYY47Z~>S3tX3SGPw!L)Qn08E>^`4 zwgjT`bP`nCb_a7?x*j(3*PPwzRu@DWye7OGp4Ig@uND^HTm+2-`;30n5X5QYfDij( zQgZ*?|6@O^GmRQ?hCkE3rM^Hz3n2Arq(9fF9Y}FH{|rV;tLP^2wGDaW{fGYA5!4Sqc&Gd&NgzPSokmR$NaK zt#>V-%*5wp9C{lMkiXv3uZX|`DX_C75-fu4&FVDE3RYip0N&J|ZWK{#f2RWP3?y$f zB?tD0gc<- zx5ULGJ21!zZ% zsa7BPOtuyOjO|cQPix!M2Av|%$yW7fV&3a|UUv5lADOi^+kSNV+%Q=Z&>A7(m>&MICx zPe~V3>7>i9cW5-W&J_$HCyc*mFGC^|bU1;@F4rpM1WJQqha5;0p2vZmz8F^?7mViWbL-v}`!TpyL> z&rlJ52S{*#5l%Ij0^mc9-o;bksx)?wDqV2@Ihx5Aw-|y%N{q(RYJYR$AeD$B8**&Q z^mjZxS_jcTicwWzq(OB`nW3Rn6P5Wk5zVB~uNm7fRAiWV zNIkuV9k4<9eq@0B1%D?~z4F>8e)hck!`5R23bHi?wz41=zgs$TG}fJrluz^52ar#W zno-Q=Yh*~rkCNvg+GR!~8J2|&vkohnqR`$F+0I}*PZ|GLTy2=3aU6+xM4dkCkYBc( z+>e_9Glgi>KE|`0`uY;K-Vx7Z=$PxDxgUX&BPGIheBNN;aXv$BQx;6` zp%UH1%(dxrITSd+Df{C>P8{0u0O}!`Nh|=)Pbl__jaSaScHu=bOvIyAIQL;gZPNaW zV8iuDBZlef%hK=Z_F$s#m)~%6NOxHTB4s~*D5WRcUK zWGpbyx3k4zr^(CDG*Wf`yw}G#TdaRR=wzwDi_&HsQBDFBnOa4^xJ>5UcgZTxEq4lstv220^;OOp_ z(*TQMvV}>EFL_TK30?f#NVK%YsSE)^=x``-LeX%Pm&o)I0q>Sl8|`7dO$U|p(nq|s zbIxD@e=TR1t0N9taGDh@zo@(vAc%&ojJ%`MAUlUWx?)&~lbmtuGPlr@X0hLzcFTPZ z*X9KD?fT^(B;`|6B{++evt;LoP(%UNQPq+l?FmV<17PToZ1=*{wDbs|Za|#j>`#Cw z1yT~qF?Q^6OD{DR0AKz!>1;?Uj-fR3!8ClxbXS3g9{5cSR%-w>RTX1Nh@BJdX&B>w zX?z`9c4@6B#GbwxbS4EfqX{yeF`2BiASEqu3IPBr37aaD=yoN{lXkh)_WDj+AqQVx zhrd$r!=MlZXxw@66*}%CRGzlEN?(bm22qsjXmQ+z56esOwLQQ$L>A)L;R5$YXJqgi z9ex$qAJkSGI7fjCbtOHcP-8D;=cazjjip!jd>SOk8kFNs@H4%Q>=beKIETUWNQMr! zK3JjC=42uQD9x4w%b0w}r2FUQdJ}Nh#+H4_*JCf0nlz;yru+2n7ZAY_?yZ4lh%@Uo z2=xxFjc+m8Q>6!f>GuHZNNx?Z-Snq9V$tsDf?NrEbqO4P1_PeQT8j-!LGA_3WS2H} zLm#PbG_FTk{NM7u?X*WDUUn=H-oItXvoEzDyn^DNQiPMLu!n-|%i~#zsvaqbF_08& zTA4{4R$dE5bab*GKg%Svoe4ZA|1&d@!;o5qp~*qa+4>-uJ!Joh;m6sUlgGq9?^r#( zysiQ^X(_cmga5eeG^Qz_nG#9;JN;N7-sacDGIt%J9la*=On0My?FIP18IwIb>yh~_ z+e;Qr;7PBOs#Ol_;0Km3KY}9dz{X~|V zVWLD&e11Svd<4$rPWHEhhJkCH0{vYOJU*QAupsVCA>7(x1q&*-hq)+puQBL!J;{`mL zt}VMW)Py^~z}JAZ@J%KrlBlob>VdeXN)5WCA`iVu0n|ABDrvq{zu&r}LeQZVsz<}m z5U0{Lgp98DQu6>ZJ#B%a3h7TZAuA2(verhz?*^wGx?@HSJlWB|d1q7NHIUvD-(+Iz z1m5wo+OItNn{^#hh;GS^{3i|I4)dpeo?xXqWwSZ}3^#nj^6kW8)+?fHUUXK&grkRh z$-w}8Mo|$RJl$eAV-C=DbZp}jU6nM``V5`179Y3K0rOS+|fCi zuuUzH^qFWxHLP*I!06mSKc6t#go7c@??go1QZ9=s_;r8(6>s(F6bOq506*>t>A^d8 z_O5mB>8&@Q=v~`uN$buujxtw%reqG=VLe9#C?sqTaef!$wHglo+;o?fnBNQ8uiY7y zS1NE?Ui#U&!IV1e%XseRjw80K3-@U&Qqs`{Ygd_8zAh73%% z*uAT4?3UTSAC&TH;>&8z5EK+ONcw=E-XIVLdLC!z)AXjDk&Ff3&lxLmITl;ytatNq zd+mi=mcD)MW7$-noFMaQ+Al3LTkst>n#=;<)*DEy4Ffq76H>Wtp0_NOg3lIOzI7aA z{+Y9ioF1JS-f`TN&$Q{vJ$A;=M7xd!H+Hk^imDOtw9sXyJ=&yclP$~4E>2fsRx?Nu z5F&*-k~NDd_~MxBOhu#F40t`?DufyE&nyo9DMm?iwYHM%7LLp_k1|HU7u9J5JxuBy zc`Y|lL;X{@LaO9#zYcV!n9jz_f}EUx*7LP4Pg&YWE{C!OD>YvKX3Ac6rjSqnL;rX} z^(D7r#oq?63sLZ`5LX_{+3l`)zm=nmuVg?iR?v|4rZxvhZ43FsTP!|X!|=Xn0~Rr* z^v$-3rJ{-O>$PEN*U;z(4iZnQZ-vq0k0UX4OQ$qa!@ieHqJsv z%!$3}o+a*!$fqjx%&Z|>*&g0cO!CQb^%ov);M+E|p=iDtL~U{}3-n&5xDOKFrB$#z z&k#&=3$+JuQjwdusvG|}>7~V`rgdV?Z#?MeX{mlcdTtKbD|@WSWC90xRz2f&w0gPj zUd)I$1t2B$STL2^TWMv;8(}q}Xt25W1}@OBkLXreTL$y>SD!>ltKD8>A5e;y7}seU z>T#rMmL=9!n1*DdH6Ny%o*ERpIc&IsHiqFD%0-iAZ@jYI4L`POxtPyr$<2Y1D7{e7Q3)Vn#2#*|M_T5ZNVn&&$%lSLGB&K}qAknCK zcyQiO!0w?j*exvEmtTj$WB8GV-Gc4cA(UBj>}vCZc=*cvaca~fQ`88TYNvwe$u!iQ z5|i)mQ^;OjcaNoy{)S?ru1)v)BsrW&${Da#aiLWdUJ(+=tTk0$NIHL7$I=#jH~wa3 zves;LLO9%qI(D|w3mFy5lW}!mHiXzi#nLxd;QLh=)$eT_TTq%28-J*gF zjyQ>qCSx)!7$oJXE#gV1i4WoBM*S3Y7`cPe!SYEVbnQNAxE(~{ignRBLP;j+adGly zLJI8%5K?P2w5-P$zje8Y*5lsl`J+9j6W701oDM%dr))Nq5c{iV2V(YuMxKoKYn%x2 znxzs#IOGXd+~-vjC`btjcN(n&cBA$dcqSU zFAaCM;jRd2O2h1Bu{oh*u1`{Jzogx4ZEUF^5n%Bg{uH6598)}9UGUXu>EN?9&`47E z_Ep-o4;mf~`wc$wSS>z&u%m%N>R?jp@p5%%T8+@VO8G`A@%yXQ;pHx^uA7qw&QhKL zh$pWUnjY;OH-;gZK^O{6kA|u`aD4*G;%f4}B~%Q#w81OIo6QR}*xWbP%A(&|>9qH>!=Q@py?l-yWU$|8_`Bpzw3DnIK?rQ=cbH(pM1rlR6Qn+{y zPcy&dO5IRXVrRNqzv=LmbZr%b?94ZY)SvikG!ynPVr0WWe0Skm99If$KtgSQ0<)8^ zEuv{^ZPE2QI|_-&6_&FvYaFYe5uB}0%ylp4^zF<28J=}CyfkentmNddF_^SPLP&_i zSlTC5B~o7ZC-_vefqNYupB-s82V~<4t`?KFJ}F=B2TSM#V#Mo7PN9DheRgBvgo>f*mC@v&Q@s zjsKox8j=EJM*1?*wwdp!nBS3bHf}a@c-hZ>(%R}g5wp#kGd+_#*ptGS#)a!i8+^sv*K_VEjUufxr8uk;=8 zg>AcR{*(Zl9#QZ4Y1(r^+v3b5j>1Dtu?Ibc<4g5dQ<3@RfL!2Y>PpM4jCQ?Vc>Bpx zCt`Su>aH#~@m-Lw#gT%w10k|BYK&u(t}dh@1N3ej z58685B}e**@_fkYSh6RkzMn*VQ9vM!}XCOrRS?Kt6=h#R;`aFG4ygH(*Z}np}X~C*f zTs?`yjDaL&1fn>x4Md()IEZZ?1R^DP*|={*MKzhfFbJCD$4n#NHLofppX| zmYnX*$_+boWUoMb517F=0mawWpUJ1PitG{--rz(zcn)%k^<$~t!BU>jnmNqx=4lnUUS5Xz)-D8g= zj`~Ei<%85IIo2Q!gNa2ADMiqa_Qipn!nUbYjc@Pn9KWxBpdZb0EoKyxJ@K$`%6Jj6 zHAWKo0UH%7F5b!9ul3g2KYVihY+ATFt}n4L%+@SW_-CwPTSHu%v?DdWZG=or3L+vR_@!ol zu|hKWNWaw;m+3kSflMERi-$0@3Z6pkcm*7-DQOhfg3xA)D8SqHrZ)I9zqDukrGE@H z0#`s04+a;lVtwlG?@)RYzEOb;^PKc?5E!)M-#fsa<3`nOo|Y1O4H7p$h=^39Imr*1 z9=Xf4&l-2eMk?{Gy@u#woU_%^1>6& z^V)OW*Zsqu!e?pn!QLJ8M1{Zn08c9RpobyU{`Msa%&)tAGyPXPXR-;K;y?YKxu|TB z>HcY^+}4VLY4gvMXKp`V{!4N)eBdNx&tL!Zliya6LjTgXZ^DI#llK2 z&`3cT@z4Hrn0_9?@k|f%*$V&UH^gV~#r7$j__I^utF^OCg6Fn~LF`iqf4?C=Ctb4V z-?YvCzc|kS?f-@c8c=y6oH3UA#$=|_^!uqSS?kcEtDulpkL7%#JM~Kle{%8oW5m#F zcZw0d`0Vs+iYM5R5q|*bfL`$15@s{PInT4am)j|V^rVa=GZ*)jNetzM(LxA~oSwTn zrGCL8y})?4WBJ=({hk_E77O3|ezYSeoCNnzc+giDge@-Do)iSBDVNV~)q3r(dYt() z9J^z98c5nidLq%iB=tym1$^9t4TdHiyCEiSx;L{JT#Zri#*R(=HqYvK31mk#j^ob(`zsI;aF{~X^u5>bwkY%$x?5l z#OLu2?7p0`M;AEU;%MWx+RljJ?sR}%bL>`=C%t_L(;TCj-!}y}Jf2HC?#qd<;dwQ7z7>pnI^f0TD7R_qq*n6UdE1O>T+Chf>y|XduJ{Q!gz1L&C z0%c2*`6-)*%iFutyQ1=$H!^(Q{eGI)I_Zq>+$H5O=W&s=MptyaNUKaq7=7S&?umiP zcCcD7IiVlSE%ir@Lti=0JT_x|%R4^c3C5ecF@D!*C1enb#AqSe;M*Q#77($&KH~lO zLX4q$3p1L|3-X|goQ+~$Z6eqd?&rgBatGBy&P#9Pu-K1#MvEK&_N=??V9&h}T?D6k zc<&UDiK3%9sj8v9AM~dN$dy;u9~Dp;-+r%o0J`?mpFSxykIHUQ>Q|@G(%P`IGZeM54@V z7_(Bct)0|dU3X6k&8x(F8z||m<`}b&=P_FQ3^nP=zJmjrXiSIN{aTL(tnZ{=~?`dwd;J43>#aq>w%M91`PEU!8EU%-G}gffc2|q7P9@@zl<~=FPXi@ zmPikjSwGkvLe{pX{BS2yoMY@9AOdX(ux{Sm zrisN8-qbLrcg4S&vPWHo?KVFEt5Jf3#4j&4riCZXGD~uIRzHnDyG@PyfiNZrPB5%S zOyV|+c^lHuFY31wr^@{%NIc%d$bHV*_>|Hlr-h?2a({A3`Io1MS%#Kl{%b+CiVM?Y zktg#@_#VCNV?HzE&26vrD@W|!7pqZDVBj6^)Cu=@VH(bF4=e4>PBfUqC?p=6wcXkD zZ@;bIQI%Qa0zDn~0>Yi?P)haWYsg9u<}>vphX@0mnFm_11AI(}M}!X2?hOy86H${F zJSIces?@D}J=}ld#|6Vrw*#))y>F8%Ar<!JSZk&%JI?n zk*CL2D$fFOB1EL5=B8+Z-K`CK2Jkg_O%r(%Nb~}RM}IZaXB)kqef47AH;uVyiqqYe z_qa4sGQHmP{?5tg)Np3HBDHxpmWhg)*JuN2nQ+V)@0;>-svv`(KItujal^FMetcODm-A;9y#JuQhPmW?@QVk) z7n7DgW0`we5l&3x8giSRf>Ydb_DO7Gk%yAYSmP)3*R)WYKYFkb$^x@|AgXFty`dV@U8^Q=mJ5anX)rCpXnxPp9(m$2dP9m30FR z;J7WWr4eDa*wU(VtNTcsv<13=mDJU&TZ99G%o+Dn#1$*WmjO1y8j}{uDW0^u0QIItFL6yhN z{+%j1iKfBg*v_9U&^3&|i#OMbN*{DB(>o7<(9=2hci#;bbWeKoM-A9oTrS?-)VzS$ z-X}iog_2Dex8W)6eab4l8*36Or{Yr;Dw;N7C%Cy5gr^uU^?K*xE8W=Rv}(kt3ek*A z@}9eRt!l80idnX;{z}fV44Y5_?X!c@b6xH9=K8}N62#M5?}aynC_=RvHR0L7z+B4G zMn+YP!U@F$oxbSXxyZLp2PXMEwmj$Qz_}){hLnDU6#wbGBg#tx=y`T|Aa;!G@KO(Z zS%sv8rVCHe8LfXYpYfoja{Pj9_Mt>>VSi{uetyg4dy?>nxp>VYI<(jY?wJ`Q%T~(R zX5Q7|Wv8=wjuw%O@3mjB5z$9`tz^gG9Wxry1bBN1oJ~&v6?ww8PxBAvP!YyZpx13) z1u*LH=PM&nUDy2UiJ#bM)HUU$Q5p4zvd_~`uDOx;S~7@W*~Yq9BStHW-jvgIG5Lsc z@lbLd6GLi-)K_eLY=j8igJCZcf-sU3YwaB>;#=Gi;^=fo2fzb3nOa04j<I>uR^J zUANO$oyapkGkV*)J*_JR20Tyjn9yn&8TDt?o1{pDaX}JH8{6$WUjg(YM>L;y3QI$N z2?BDb-fD5%E3i!``NxW$ymL(Fb>F`ozz+mid#ibTZZpzR<2dlHWoSS_M?O8wsIZ5X zZ=ff6IH$^UDSRMT?#Ns%;knW{Yen2dn8U&tWE!XvesMoQvT;=q$jv~D#U!BMehXCr z?H32N#QY{#x!IG=n=8r!uUUIGqk9PH`j7c9LKs(Dtf4iQ*(p-^gFno+$7sCQ=Nra^ zrRSL|LN=m?JbVC@kX=EnLLZ#TIP42m;#?2TEXA5Iczkxzln1nkb; zIT?-djkR|3HN8{<)qh0#W7|B-Ak(EN{N(UhJm4;u z%H?S`KfX-2kytcdFdGylIug=aqp8G!N4x;8_(^v9%`euDJeHN?MEKxECt77dHEl1M z-egFkAtt+%-r+!iuyU$Qtk`4Kp`D$LIc=m+mrXqpTF+vJ=_dmveeuzQWmV=C(yHH; z(8cV~CH>AFuTo$m!$jo;P0-(i^EmC@ z%x+~sQ>w+h8xOSL6cu(Zdqt+nU*aK*F8!X-ZZi6;ANwkNqxuGFA{k)C(v13(O!Q%H zfU+AbHsP%2$q%%ulN%sj7Yqzr&j$-L$@&}?^iLAJmd3LS5|kglc;;^r&YN*yklbA% zR^4jR=Hve!v`e#NEnlTWbi^>!a+#-8 zU1E2OCL;=Iy?rb$w=~AUsP_`gucV(KyUURfY=tJ2O?RC)H>2W#j%X#Ct_RO&y*Uc5 zme)QrBNZ-8Gd|s)n`QGNHXs*zRJLKOW*8R)YgfNsIHK1mu?0rl7fK-_ke{Ako!vd) zrER_>?`1j!RB@1zM~FSKw0kL_)z=(e_3jg&vvNq>zS=hDwNKFUBtx%h@MP>QNvrIV zR$OE{yw%x%JXW;kGfw4=!^PN+e>}v%C^{r$^#~5;#wf2aF9HY?)Tn=I%3?_{_gH7i zm$*i2IT~V28CkpnQG0l~EFatpj}Y7}I|Wdn7VK7-%$}RLPMi4_r{Lp} zw%$3G14u(oo;g)fV!M=lJWi7)O_z|U$Z?E~M7C^nssJB^_ z4NIjEBZsz%%Htg;n?NL^KsFWAYJaK*UMC;PrS_*v{KB_0T?;KAEqkudoG(6DURe5z z>YZGf?Z=i+^+~5aW_>=SqTH|nqjCugqfo5bNK3mmsrqMk-(pV9b5h`psy+QEoblR; zL*roJ7Um<%h-0O?=h3u_2e$kuJ-jVYj2ODh{yo_>a`J)>=I2MGwOy8EtP)T%1Qy)L z2PlUbexp%kN?{o^n#sxAx!or5{FbvMv(bR=@X@X9=iJDukj=(5x0RVW8729~nLl4k zVK!px*;()k_f|;NYb2$XSfTY8iJr0Y27VgRSuUR-ii)b8=+q$V?Gpu;46;tF7N%H= zbC6*!-^B+Oj$ON$BppLVrby6Tig2oDoC*arcnK8kJih$X@X<{;=V6VLa(GZsbtI$; zuPo0m@i{PsS#Yb$;^Fb?(V! zBb`z3kY4l8$bP-GktD9SXXnf2oTmon2vQG3hGq_LS3RSI#_eUgZ(t-HSQ$1S@{2qG z%qL%rw68X-!t4FV^OQd-*D%{pQMyk09Jt77yGgLoD)hwH==wkfm;M~PUlRRTq0O)W z*82eiZ|pGN*`nlN`!y<`F+=nNuf2QN%V#hHRTXro-d#u$Uw-N;E5?arxcv&w*IIm@ zb|Vcezc}HNIQLW7=!_3v0;%tiHo}y&+clJq3@yDJ3{rb&yS`klmNEydf9f(@fknHk zfT_?Dg}M@0cAiUG;sj4)YD86DuU!)INyy-C0=;&svo>ew;km?_y1g|kU-~N+&$(+$ zdygBPhx^Yvm8 z3X4BY*Wsg2sW0{BRaG#X96oyr7bMW=W+Tfh#oEZaz#|$w76Xs7bd&{Ynpo2A?4^3I zxTWIaa%5iLsh08)ESfbw9W@I{G6k~A3Tx&i#EJ#Dgar3>jvt9k1gf?^SI4}jczIJrS;fF_4?k%Q|4b2;qHS^BNR6>c*mZ+d0Da7xkZXUQ)yuD7bL#7L zRA}VW&BzN6f;Sw`9y5&il$Z9G^Rq5j9KIEk>qxu^E}xnV(AErR{`eqeHMPW)jk~Pk z@ta{fHl5TDor16|S>|;Tg?a6=YFq;1HgApDL4vJE|IDu?&TJ&W z;X#QZ4F^ZVy@n@3D85Txjmpv$?>F^Q((=sV$kw~hx*KL>!V%A55_&mnR-)hZRHJF# zK|8$owilR+%tb>1HBLtg^FyiwyP&tFN+xU>`A1=NpUPM)H%{Lz^j9-c@TL^gI^D4b zQ0tZR3C6D{2f0$~rGKKa+=N~!9lMjpk-38~>Lg(^A42r0v7@9D)UNi3hy5zY6`8h> zhy9m`)HS~~mJUk{53jvFI(vZkp-yr1TJC^{u&a*ZO?R}Yd4=n&wDMiv9%O5hlhcVe z;dJuM)=Wd$RiYQ8lq*#&el_cL9$_E$1kBkpEW+v>4CSn&BkZbxDn(Q`j4G4WXbPxPVIDC{3>Gu_m_>!FqDm5(C=?6$Xfh*TudYK z2@aq7k#f&<0dVE`s56nP5yAdDz38aN2Z1JwOF)Jm4}MNGQ{Vd|~%^3eUWwHMGL`10qG_>m(!~(H>dR-d=Yj z0$cZ9amB4Ctpsv5ER`g&zfd;Bi+t9fFI@2AS#yTb4VZUkrQG6Z?O{dhVaY;xAWn>i z;)V}$#5dAAP%IXXy{XD@wzpbX@xcm`9P66Srup(p!C6<`cxR}J5Mu*W@9wZ~l{C7? zMoGfj&Zn>f^l+*6Aqe zKC`#?j><5lOOzGr?|1EFG{vQxTOL{uogv?THG~bntS_X>?aH-?nCt_;pEnbt5cS%3 zGh!MoX~B+#l2d?w9MZ3U5=O}$9{grN*JT910}Y6*A(f*Pw09riwPj+Q?-v2GBP;9h zK{^@|{1H+}U1^)!W?BNiA<=KHDqFD=hTWhAw12ls#r z`;^So72h=;JG`w0l~XdiKjrf2qpPYukA5L`bC@~&KX72nH~fR%IMI%#Dx!<(uUHcb~CHW@yd92 z!JZ0l;^)`hF?esnzSLFoJLkg=hE zzdm5*6jd4Q`_*G+)D8RLFDQHp_SwQ)P*CMDW^O15LbngmqrDs zJkH9!%|7)braf5mJr4Gf0z-ul^srK6Ry#3nP$Y}%RIo!wc9Ak+?;w}W?MTljiq*-(SD?qQl3?Hz|f(hOvlz9i68!} zt79_f^rurw}eE7dgN$+sDFF%PeV93_nrRCyR>>HK> zX;mjJ<7zsMryxp;)^msSn3n;syc|<`)a-+9wPiQhIMaXjFXv5R)}lG#Xo~Z^OCr<1 zJt4WFctjBveoNjGY=};JU4CiXN?(sATa)?R$cRW!N^ZFzgI`jR3}Cj-h#3@i|8nuk zi_Gx2ow8a@37lsi#F(gyA_U|PVWPC=zf)j(=LY2pP~rH@2KT0o`DGMLHy}ev^61=^;NCk=z{e8zZ*G z+B&?wQ9ID0hS)2)NUhRfF=p%<6B1x4%D&K;@Yv{YGen4@9#u(ZvD`6EXF7bM!-RRPTId}S4Lo4n{r zN~W_5%JLVSN|ZWHzre#0sYIAiGm6BF8(DLsV3egq$_u0^_gZd-d<)AAk{wI%+{e$_?#FHYP>V0=x^XZM= zGT-aDS5@eX$PwLj z-?Q#kx#h&+>KiM7Ylkzo2nxn27)iGOl!eihy4yx_wTuD2I2P$sq{$@4*jD?2jSb$i zMU<4voITliG#w*)VM?X6@k1+{XP44;5aXn0H&=QF9`oO{_RU%UreyScK^6txvQR%< zCQSt3r?Jj*m8G=nO=hSNXQJg^yuBF^qTDhFsKL<|a9=DC-2_oVTO0l6 z=qnP#l2Ua<7`C4ln@y>*OKEGbuf2ty{dl4**83Zd-nxX`hvq-JiB5c+S`F#lbi6y? zy_zl(5@Q;~m1qV*kWVMo?v%l9YxCa6tM8tz2_n`0?x@`mgfrSbTgs65rvlNxvUsBd z^ihBOdf7%BtRereq!ktpU(WxB!H!YD>p}kcx_X1e_0RXu_`kkD`{&8$$p3>)Tnb+N zkaOZOq{_myxs?wu_&Xl>Z(MwKk`zJr>{e+iucqLq_v!D~aDx7KJTS>WJo#Gb-^K&~ zUmWNE_D5T@9|Of$StPNSi&dt}AhkE;*rP?Phrgw>2;gJ79(B7(W*ecmTGVrd4R z8PJq@U`Ma?$zqoA_K<<#8IFj;!!Pi8q~+R$kldDaukTKG0@5Q+FP)SL7M3&W(X93I z;fk;-x5W+EZ}J|^3V*Irv9!!`wvxk;P6x8@efW zt;kcp2YfQueAv+&qCmdOSTQ*XbiCh83VB-s|9eL(dR_j-8>G3z&0IBdlWpmnBfFd8 zk7Dk$-JyL!VEPZ)r5@|%Ym(2`rdF8YphC-ol(LJx`CSdIsO8D zdm16pUBW4KcmQ~+8>sF58C$ScEg7qNuR$8dG!dKYImop1i7^ewpr49d`QZeLAZA$) zx_(0{fT^ zLZEenC45Dp5O*hH`FyE%@kvD*ZzHkAb!j78A~)g5XViZ*=np2%c{m(uUgES2WnI7X ze;@mUdK*d_lIo0fWbXXNiEX__vPL+qF*chK(C6XtfWxHrm`F=<|@vk=eJNDbYOSbK}Po$jow&xeF&Q;kFf&%X2 z{EJ?#BZs?IU1a1S^;WOTwr~m|)xg(KX={s+BC=;sJvX0C#tIsqAFKusej)e)8dA@4Gkc(=l=8nF9OPxU6H;+|Vd zIN=mK-{U_;SN%sr|E$mvv^BOGiZf4wE zopJGfWCHq2jY24=H2X6F8u)SdO0LLI-SUf3L~@RAmQ}_0Z|-env=`(GR2lOmH;ePU zNCbt}!zcQ8?9SLmq!|mws>}QB=n2IhK=<*d1-{f3XXmx#>T8`5B5dmY@s_o&lh{bs zF0^WQG`#VCnfI;EHl@qK!fbx8S8pxP_d>oZ)p;QuE&8kI5I5*im8#7wvtZun}yG zsNQ97S91Uy?D!@hM=9YMQwLEoDo!o0j&*|H?~obIJ%rUavYI*BSbm6NoP$?*hUdKnpn4Twnclg&sX>a( zEPALEwixD;6lnXd&cmk~xeYc=mK)zfI-pyFg>)_T>a(IwUYHDx_a18g14Bhlxpw}U zZEqNkP-p?F?Vy;FZu8}P7i^=iC|WO?>+MS^>3}Jrh)!v{L#8DI^;d^aH?6!WvV5QH zGk6T-2P@YDb*?easS@=;O!Bdx8R(8aaD5GqibFI@2r4RiK=)vZ+ZNAHBb?;!@EF+$ zhP@K*GSVhkwGCen>)ssnls45Y0u3*7mhe7Mma0$*kx%64gi5m)POd5!yJ>D3|3Rk@x788Rcoelfs7ums zdEcjJ`jhc1n^hU20>}ZU>*+FsHJahBw7bc3vqwDN9nj^F^3|qDe@*-~IfpHOHT_yX z?BvvOCy%Y;w3z;S4SUN9aNYg+T#TSpXK^k>s4I_@=yo!54JyS8>>FT>(szs>9XKJ; zsyI9A-a`TVn?J5R22?p?wD7jCw?2_j$Re>7iZ*2%E<`ap8b7~!9?AsYkB%mrl4Ldl z1Uf~B<~(mbjs`svD9THfmgnP&;^F$8-Gqp33Yl(^m}5kC&A`l?!+62*Buve^+Ew-Y&;95%AciuLSpA-miqX3YZ}bNHa<; z;^h`9O`0n4OI>btmBr%kGeq-vLxQ^kMRA&ebrB$%YDo!gXN?kah1`6G)+mbea#HN0 zsc~7y4TlZ>>M~c2+JqjWM7Nd7L%cWa7n@@G==79n$@Sr;-woqq>dqWg))n8XyN)C* zg6OK#L}z0q6=QLU<>Wkf16ck&IEp>;{}PT$2eAIna5Q8^A`XOkQ@jMSz*y+cR0{+b z5k>fWN2#7bEom^{PF>*ULr7&EP4K?=(=2ZvK4^Yi?CNX!swlS&-W+`!6jW8gYfSQ{ z=%p&_Lti2HY=_|7H?Lz0X%Y;K6cR_)nX%IPO0`)GTpj?3DvX;TEt`rQa6M7T6}k|f zy1I{r7=a23JxvW|8mjHzl+(&LlOdKQcq*UARSCaaWejEN?CiL|%Rfr>E`cP)<(mD zj>eg^SHoH?`@2@#Ct%H*W7}W|LnUn>4m@#Vv|@A2Rriq)6u}TN^zA8)8MyFS))+ca z=eH?(6peV5T5oi)MFMcf9scFW=XZbeGQSH_rqm&QP~)l&c*uue>6DkfKwD#@w4lTe zD9KQEJq^anmt*8@&=7gB;~=;uz?kS$Kc7tq1-;4^YE(}6w8d{`qabUbkk^wYuJwIR zccu)m%_VFBJg<=U?UA+6nvW_7sWiPP$Z*n3pYdAS8u*Q z=)U*EaR7L`J4Cx%%(dxT!>k2OdW$1wLKwL9DLGt9yA6LD8GUsGMkq)HtlqWLlUybR zvD^|WE+xZ(WWpvUK2MOcB3kZKy!lMmk4^P{rnc!J<*s+Zd#(o|n$-b8Z{t|g=SG+3 zgof%g-sEg6?`Z1b1&m{Mej*Bp+sDs0h|wM{E(!_~w_sKg;7Nu3ZmS}|z$lf~x~B$M z3+&m|h}fJ$7!K#IwwPEDcWle?vLXkFWs>^{pB>l%#J+bK$6cT^hQr(qiA(oCG z8BO6q3W60}rl0u}MH(Z-28&d%43?YTnakI=1_sSB5Q}MU%@oIc@0bs)_o_x=Eg=c3 zwO}o1`ms_!M`KgC&8&up&sh3RE62lazoDVS#CiNY=E7RFMsu@@2%?zy_LcC;=KlIM zlm*VcboBIOg^Q0CK&a6HmN+P-CSe@-I{eVIX?}06G$Eyy#kiZ1hl*E({|i`oI)Z#X zILUiTSd;!>Z*7f`C7CW7T$@kw^b>(QZJVCtR3JfV^M3yEF$|xaS*&vFbv8uU5Ku_t z%GN%9olf_SYA9!JAlGWH?UM9_;TgPIRtFL!yP;M)`=g;bXjrb<`Jihg$j|8>j=1-UzUoJg^s?^83YD?Dizjc@dodUaa&*sto1ZiUtg-R&IyuZGOq8C)% zjQRAoXf&T&_+f0XN)`tHRdES3``(2yFV5YCksMN0UkEkJsuhz4#xrR2v&F4poaC^< zsZf7wu;*~-4#rpeU@T@fl^)g}azQy+O$C*$LEpN#rN({&FcwC%yqWqem(}piUaTxN z_MjAqmq5Z**xKv0NJ?T-2ADyua&x6938`r6&BWV`&4-cKwrrc1YINy8HZRhK>O~-L z&VNCClZM5W%cy5=L`>OA0$XEWo5otxvt!!lt0L=$<%${|9N{7YnaMB$%`Mx=$o24^ z2kPq$Y&O#oG4>?>9%;cZcj>u%EnR)XmhVuY+fTNE1kH(~ou6HgDVn-K^w8=ROS3bW zAw^81c5=d%KhIcQ|5ItpSIO#NX!VF8SY4>c5Xw_;Hy@@BsHEoL*xj(%1H0E2sE@j? zw_J`YuWai@h#;8{+i`rj+r!TH=X_w$mp=77xKcynhsgM;}= zlbe1*uPqr(`<00T4DqM77LK*W)y7DF{JQi!forC#@o38`d>Xx5S{ROF$jbvS8a_;X<{3pJP=AfDo>N{syxr2heow*>a}t|#R! zeqzfDu1?@;in*iniPAZ+2_g$HtadSI$2H?WUO?NcGM)&DckY!4A&tZI{@LCKPBw&< zAIfcGH;Hs>&+Lea^_Zr9B@%vmywk%&6Dr<0&hRjt25#7J3tQmpll-1CidJJRN95K( z7ievdopN9)**4au=do&z2@1j>aI!cgj$>#f&al6CTWTrl@m%1VqH{8@g99`?AeuQX zBgDv~>y1B3|7xU(8#8Lpn|jg^UB;qZ2q&CRSl1r@KVgv8|Bm^p&{&4P4!o^(&}@M{ zY*rmaLvEGx>j}KL>`=4_RkQ1R)A=C$#Ku_cGc8>dj1J~IMQL6vD%8+J&2)%7r_hdT zuuMeqG>*A2Ufx_d=D!-VEG1Xr|I`9Bg0W_lyq48m6|OEl2vy-#0UEgQ2e0nbbouKn zLq7h>aAWqkK&Rx*BLd+R>RpM}So^oZ3kQea)~gbopK`Ua4X58$*9^tt!lw+7 zQEb6=$v6F`bk>ngGg?@_n0X4C4)<``3jgwOYn^e{upC+cMoIq_Sd#hLc?22(PHr$H z(y8E}h76nd4mUKI&r8UJpTJaB+w@+g7lN@lA7zzh4#g}?E;maB%il#PoO%2jjW`c

    +

    + + + {% if dict_domain['languages'] %} + Languages: + {% for language in dict_domain['languages'] %} + {{ language }} + {% endfor %} + {% endif %} + + +

    -
    - {% for tag in dict_domain['tags'] %} - - {{ tag }} - - {% endfor %} -
    +
    + {% for tag in dict_domain['tags'] %} + + {{ tag }} + + {% endfor %} +
    {% with obj_type='domain', obj_id=dict_domain["id"], obj_lvl=0%} {% include 'import_export/block_add_user_object_to_export_small.html' %} {% endwith %} - - {% if loop.index0 % 4 == 3 %} - {% endif %} + + {% if loop.index0 % 4 == 3 %} + + {% endif %} {% endfor %} diff --git a/var/www/templates/domains/domains_result_list.html b/var/www/templates/domains/domains_result_list.html index 3ef5b18f..10fa7ca3 100644 --- a/var/www/templates/domains/domains_result_list.html +++ b/var/www/templates/domains/domains_result_list.html @@ -172,14 +172,12 @@ function img_error(canevas_id) { blocks.addEventListener('change', pixelate_all, false); {% for dict_domain in l_dict_domains['list_elem'] %} - {% if 'screenshot' in dict_domain %} {% if dict_domain['is_tags_safe'] %} - var screenshot_url = "{{ url_for('objects_item.screenshot', filename="") }}{{dict_domain['screenshot']}}"; - {% else %} - var screenshot_url = "{{ url_for('static', filename='image/AIL.png') }}"; + {% if 'screenshot' in dict_domain %} + var screenshot_url = "{{ url_for('objects_item.screenshot', filename="") }}{{dict_domain['screenshot']}}"; + init_canevas_blurr_img("canvas_{{loop.index0}}", screenshot_url); + {% endif %} {% endif %} - init_canevas_blurr_img("canvas_{{loop.index0}}", screenshot_url); - {% endif %} {% endfor %} From 7c7799564ffbbfbe878fa1f1a3d46bf40d05ba72 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 2 Jun 2023 11:03:32 +0200 Subject: [PATCH 017/238] chg: [importers] improve abstract class and logs --- bin/DB_KVROCKS_MIGRATION.py | 2 +- bin/importer/FileImporter.py | 32 ++++++-------- bin/importer/PystemonImporter.py | 14 ++----- bin/importer/abstract_importer.py | 70 ++++++++++++++++++++++++++++--- bin/lib/ail_files.py | 14 ------- bin/modules/abstract_module.py | 5 +-- 6 files changed, 83 insertions(+), 54 deletions(-) diff --git a/bin/DB_KVROCKS_MIGRATION.py b/bin/DB_KVROCKS_MIGRATION.py index d39fd108..9178c896 100755 --- a/bin/DB_KVROCKS_MIGRATION.py +++ b/bin/DB_KVROCKS_MIGRATION.py @@ -642,7 +642,7 @@ def domain_migration(): domain = Domains.Domain(dom) domain.update_daterange(first_seen) domain.update_daterange(last_check) - domain._set_ports(ports) # TODO ############################################################################ + # domain._set_ports(ports) if last_origin: domain.set_last_origin(last_origin) for language in languages: diff --git a/bin/importer/FileImporter.py b/bin/importer/FileImporter.py index ba5e54b9..410cc65d 100755 --- a/bin/importer/FileImporter.py +++ b/bin/importer/FileImporter.py @@ -24,37 +24,29 @@ from lib import ail_files # TODO RENAME ME logging.config.dictConfig(ail_logger.get_config(name='modules')) -# TODO Clean queue one object destruct - class FileImporter(AbstractImporter): def __init__(self, feeder='file_import'): - super().__init__() + super().__init__(queue=True) self.logger = logging.getLogger(f'{self.__class__.__name__}') self.feeder_name = feeder # TODO sanityze feeder name - # Setup the I/O queues - self.queue = AILQueue('FileImporter', 'manual') - def importer(self, path): if os.path.isfile(path): with open(path, 'rb') as f: content = f.read() - mimetype = ail_files.get_mimetype(content) - if ail_files.is_text(mimetype): + if content: + mimetype = ail_files.get_mimetype(content) item_id = ail_files.create_item_id(self.feeder_name, path) - content = ail_files.create_gzipped_b64(content) - if content: - message = f'dir_import {item_id} {content}' - self.logger.info(message) - self.queue.send_message(message) - elif mimetype == 'application/gzip': - item_id = ail_files.create_item_id(self.feeder_name, path) - content = ail_files.create_b64(content) - if content: - message = f'dir_import {item_id} {content}' - self.logger.info(message) - self.queue.send_message(message) + gzipped = False + if mimetype == 'application/gzip': + gzipped = True + elif not ail_files.is_text(mimetype): + return None + + message = self.create_message(item_id, content, gzipped=gzipped, source='dir_import') + if message: + self.add_message_to_queue(message) class DirImporter(AbstractImporter): def __init__(self): diff --git a/bin/importer/PystemonImporter.py b/bin/importer/PystemonImporter.py index d17dfc8a..536801ba 100755 --- a/bin/importer/PystemonImporter.py +++ b/bin/importer/PystemonImporter.py @@ -10,9 +10,7 @@ # https://github.com/cvandeplas/pystemon/blob/master/pystemon.yaml#L52 # -import base64 import os -import gzip import sys import redis @@ -32,10 +30,6 @@ class PystemonImporter(AbstractImporter): self.r_pystemon = redis.StrictRedis(host=host, port=port, db=db, decode_responses=True) self.dir_pystemon = pystemon_dir - # # TODO: add exception - def encode_and_compress_data(self, content): - return base64.b64encode(gzip.compress(content)).decode() - def importer(self): item_id = self.r_pystemon.lpop("pastes") print(item_id) @@ -53,9 +47,8 @@ class PystemonImporter(AbstractImporter): if not content: return None - b64_gzipped_content = self.encode_and_compress_data(content) - print(item_id, b64_gzipped_content) - return f'{item_id} {b64_gzipped_content}' + return self.create_message(item_id, content, source='pystemon') + except IOError as e: print(f'Error: {full_item_path}, IOError') return None @@ -81,8 +74,7 @@ class PystemonModuleImporter(AbstractModule): return self.importer.importer() def compute(self, message): - relay_message = f'pystemon {message}' - self.add_message_to_queue(relay_message) + self.add_message_to_queue(message) if __name__ == '__main__': diff --git a/bin/importer/abstract_importer.py b/bin/importer/abstract_importer.py index 2e344bc4..e5155775 100755 --- a/bin/importer/abstract_importer.py +++ b/bin/importer/abstract_importer.py @@ -7,26 +7,41 @@ Importer Class Import Content """ +import base64 +import gzip +import logging +import logging.config import os import sys from abc import ABC, abstractmethod -# sys.path.append(os.environ['AIL_BIN']) +sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## # from ConfigLoader import ConfigLoader +from lib import ail_logger +from lib.ail_queues import AILQueue -class AbstractImporter(ABC): - def __init__(self): +logging.config.dictConfig(ail_logger.get_config(name='modules')) + +# TODO Clean queue one object destruct + +class AbstractImporter(ABC): # TODO ail queues + def __init__(self, queue=False): """ - Init Module - importer_name: str; set the importer name if different from the instance ClassName + AIL Importer + :param queue: Allow to push messages to other modules """ # Module name if provided else instance className self.name = self._name() + self.logger = logging.getLogger(f'{self.__class__.__name__}') + + # Setup the I/O queues for one shot importers + if queue: + self.queue = AILQueue(self.name, 'importer_manual') @abstractmethod def importer(self, *args, **kwargs): @@ -39,4 +54,49 @@ class AbstractImporter(ABC): """ return self.__class__.__name__ + def add_message_to_queue(self, message, queue_name=None): + """ + Add message to queue + :param message: message to send in queue + :param queue_name: queue or module name + + ex: add_message_to_queue(item_id, 'Mail') + """ + if message: + self.queue.send_message(message, queue_name) + + @staticmethod + def b64(content): + if isinstance(content, str): + content = content.encode() + return base64.b64encode(content).decode() + + @staticmethod + def create_gzip(content): + if isinstance(content, str): + content = content.encode() + return gzip.compress(content) + + def b64_gzip(self, content): + try: + gziped = self.create_gzip(content) + return self.b64(gziped) + except Exception as e: + self.logger.warning(e) + return '' + + def create_message(self, obj_id, content, b64=False, gzipped=False, source=None): + if not gzipped: + content = self.b64_gzip(content) + elif not b64: + content = self.b64(gzipped) + if not content: + return None + if isinstance(content, bytes): + content = content.decode() + if not source: + source = self.name + self.logger.info(f'{source} {obj_id}') + # self.logger.debug(f'{source} {obj_id} {content}') + return f'{source} {obj_id} {content}' diff --git a/bin/lib/ail_files.py b/bin/lib/ail_files.py index 26929873..31a27669 100755 --- a/bin/lib/ail_files.py +++ b/bin/lib/ail_files.py @@ -1,9 +1,7 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* -import base64 import datetime -import gzip import logging.config import magic import os @@ -181,15 +179,3 @@ def create_item_id(feeder_name, path): item_id = os.path.join(feeder_name, date, basename) # TODO check if already exists return item_id - -def create_b64(b_content): - return base64.standard_b64encode(b_content).decode() - -def create_gzipped_b64(b_content): - try: - gzipencoded = gzip.compress(b_content) - gzip64encoded = create_b64(gzipencoded) - return gzip64encoded - except Exception as e: - logger.warning(e) - return '' diff --git a/bin/modules/abstract_module.py b/bin/modules/abstract_module.py index 38989bd9..0a1a12cd 100644 --- a/bin/modules/abstract_module.py +++ b/bin/modules/abstract_module.py @@ -33,10 +33,9 @@ class AbstractModule(ABC): def __init__(self, module_name=None, queue=True): """ - Init Module + AIL Module, module_name: str; set the module name if different from the instance ClassName - queue_name: str; set the queue name if different from the instance ClassName - logger_channel: str; set the logger channel name, 'Script' by default + :param queue: Allow to push messages to other modules """ self.logger = logging.getLogger(f'{self.__class__.__name__}') From 9efc3485067f00f3312d498ce4036f0f15422796 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 2 Jun 2023 11:23:52 +0200 Subject: [PATCH 018/238] chg: [correlation] filter blank screenshots --- bin/crawlers/Crawler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index e8822561..73c17472 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -61,7 +61,8 @@ class Crawler(AbstractModule): self.domain = None # TODO Replace with warning list ??? - self.placeholder_screenshots = {'27e14ace10b0f96acd2bd919aaa98a964597532c35b6409dff6cc8eec8214748', + self.placeholder_screenshots = {'07244254f73e822bd4a95d916d8b27f2246b02c428adc29082d09550c6ed6e1a' # blank + '27e14ace10b0f96acd2bd919aaa98a964597532c35b6409dff6cc8eec8214748', # not found '3e66bf4cc250a68c10f8a30643d73e50e68bf1d4a38d4adc5bfc4659ca2974c0'} # 404 # Send module state to logs From e7eceab2b32c05102989d59f05e7a7f167d37214 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 2 Jun 2023 11:50:21 +0200 Subject: [PATCH 019/238] fix: [correlation] fix tagging nb nodes --- var/www/blueprints/correlation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index b6b9776d..38a2b73e 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -247,8 +247,8 @@ def correlation_tags_add(): if tags: ail_objects.obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=filter_types, lvl=level + 1, nb_max=nb_max) - return redirect(url_for('correlation.show_correlation', type=obj_type, subtype=subtype, id=obj_id, level=level, + max_nodes=nb_max, filter=",".join(filter_types))) From 6ed76b2e3bc01f0709cb763ff98bbbcfbc9de3b5 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 2 Jun 2023 11:57:05 +0200 Subject: [PATCH 020/238] fix: [domains explorer] fix empty screenshots --- var/www/templates/domains/domains_result_list.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/var/www/templates/domains/domains_result_list.html b/var/www/templates/domains/domains_result_list.html index 10fa7ca3..3c5a6077 100644 --- a/var/www/templates/domains/domains_result_list.html +++ b/var/www/templates/domains/domains_result_list.html @@ -175,9 +175,12 @@ blocks.addEventListener('change', pixelate_all, false); {% if dict_domain['is_tags_safe'] %} {% if 'screenshot' in dict_domain %} var screenshot_url = "{{ url_for('objects_item.screenshot', filename="") }}{{dict_domain['screenshot']}}"; - init_canevas_blurr_img("canvas_{{loop.index0}}", screenshot_url); + {% else %} + var screenshot_url = "{{ url_for('static', filename='image/AIL.png') }}"; {% endif %} + init_canevas_blurr_img("canvas_{{loop.index0}}", screenshot_url); {% endif %} + {% endfor %} From 3e9dd8a702edec7a74e9bff237459a4cb97bedf0 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 2 Jun 2023 13:24:40 +0200 Subject: [PATCH 021/238] fix: [domains explorer] domain screeenshot --- .../crawler/crawler_splash/domain_explorer.html | 15 ++++++++------- .../templates/domains/domains_result_list.html | 1 - 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/var/www/templates/crawler/crawler_splash/domain_explorer.html b/var/www/templates/crawler/crawler_splash/domain_explorer.html index 63606205..12472b4f 100644 --- a/var/www/templates/crawler/crawler_splash/domain_explorer.html +++ b/var/www/templates/crawler/crawler_splash/domain_explorer.html @@ -211,15 +211,16 @@ function img_error(canevas_id) { blocks.addEventListener('change', pixelate_all, false); -{% for dict_domain in dict_data['list_elem'] %} - {% if 'screenshot' in dict_domain %} +{% for dict_domain in dict_data['list_elem'] %} {% if dict_domain['is_tags_safe'] %} - var screenshot_url = "{{ url_for('objects_item.screenshot', filename="") }}{{dict_domain['screenshot']}}"; - {% else %} - var screenshot_url = "{{ url_for('static', filename='image/AIL.png') }}"; + {% if 'screenshot' in dict_domain %} + var screenshot_url = "{{ url_for('objects_item.screenshot', filename="") }}{{dict_domain['screenshot']}}"; + {% else %} + var screenshot_url = "{{ url_for('static', filename='image/AIL.png') }}"; + {% endif %} + init_canevas_blurr_img("canvas_{{loop.index0}}", screenshot_url); {% endif %} - init_canevas_blurr_img("canvas_{{loop.index0}}", screenshot_url); - {% endif %} + {% endfor %} diff --git a/var/www/templates/domains/domains_result_list.html b/var/www/templates/domains/domains_result_list.html index 3c5a6077..8ca378d4 100644 --- a/var/www/templates/domains/domains_result_list.html +++ b/var/www/templates/domains/domains_result_list.html @@ -104,7 +104,6 @@ function toggle_sidebar(){ } - + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + + {% include 'title/block_titles_search.html' %} + + + + + + + + + + + + + + + {% for obj_id in dict_objects %} + + + + + + + + {% endfor %} + +
    First SeenLast SeenTotalLast days
    + + {% if type_to_search == 'content' %} + {{ dict_objects[obj_id]['content'][:search_result[obj_id]['hl-start']] }}{{dict_objects[obj_id]['content'][search_result[obj_id]['hl-start']:search_result[obj_id]['hl-end']]}}{{ dict_objects[obj_id]['content'][search_result[obj_id]['hl-end']:] }} + {% else %} + {{ dict_objects[obj_id]['content'] }} + {% endif %} + + {{ dict_objects[obj_id]['first_seen'] }}{{ dict_objects[obj_id]['last_seen'] }}{{ dict_objects[obj_id]['nb_seen'] }}
    + +
    +
    +
    + + + + + + + + From f3946cc4f3755b4c56a602cbf8ef27e30f434d0b Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 14 Jun 2023 10:38:33 +0200 Subject: [PATCH 037/238] chg: [title search] add pagination --- var/www/blueprints/objects_title.py | 37 ++++++++++++++++--- .../objects/title/block_titles_search.html | 3 +- .../objects/title/search_title_result.html | 10 ++++- 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/var/www/blueprints/objects_title.py b/var/www/blueprints/objects_title.py index c20f8626..ab6ffd25 100644 --- a/var/www/blueprints/objects_title.py +++ b/var/www/blueprints/objects_title.py @@ -19,6 +19,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## +from lib.ail_core import paginate_iterator from lib.objects import Titles from packages import Date @@ -72,14 +73,37 @@ def objects_title_range_json(): date_to = date['date_to'] return jsonify(Titles.Titles().api_get_chart_nb_by_daterange(date_from, date_to)) -@objects_title.route("/objects/title/search", methods=['POST']) +@objects_title.route("/objects/title/search_post", methods=['POST']) @login_required -@login_read_only -def objects_title_search(): +@login_analyst +def objects_title_search_post(): to_search = request.form.get('to_search') - type_to_search = request.form.get('search_type', 'id') + search_type = request.form.get('search_type', 'id') case_sensitive = request.form.get('case_sensitive') case_sensitive = bool(case_sensitive) + page = request.form.get('page', 1) + try: + page = int(page) + except (TypeError, ValueError): + page = 1 + return redirect( + url_for('objects_title.objects_title_search', search=to_search, page=page, + search_type=search_type, case_sensitive=case_sensitive)) + +@objects_title.route("/objects/title/search", methods=['GET']) +@login_required +@login_analyst +def objects_title_search(): + to_search = request.args.get('search') + type_to_search = request.args.get('search_type', 'id') + case_sensitive = request.args.get('case_sensitive') + case_sensitive = bool(case_sensitive) + page = request.args.get('page', 1) + try: + page = int(page) + except (TypeError, ValueError): + page = 1 + titles = Titles.Titles() if type_to_search == 'id': @@ -97,9 +121,12 @@ def objects_title_search(): return create_json_response({'error': 'Unknown search type'}, 400) if search_result: - dict_objects = titles.get_metas(search_result.keys(), options={'sparkline'}) + ids = sorted(search_result.keys()) + dict_page = paginate_iterator(ids, nb_obj=500, page=page) + dict_objects = titles.get_metas(dict_page['list_elem'], options={'sparkline'}) else: dict_objects = {} return render_template("search_title_result.html", dict_objects=dict_objects, search_result=search_result, + dict_page=dict_page, to_search=to_search, case_sensitive=case_sensitive, type_to_search=type_to_search) diff --git a/var/www/templates/objects/title/block_titles_search.html b/var/www/templates/objects/title/block_titles_search.html index 4f49c287..d90afaf7 100644 --- a/var/www/templates/objects/title/block_titles_search.html +++ b/var/www/templates/objects/title/block_titles_search.html @@ -1,8 +1,9 @@
    Titles Search:
    -
    +
    + +
    + + +
    diff --git a/var/www/templates/objects/cookie-name/CookieNameDaterange.html b/var/www/templates/objects/cookie-name/CookieNameDaterange.html new file mode 100644 index 00000000..2a17b742 --- /dev/null +++ b/var/www/templates/objects/cookie-name/CookieNameDaterange.html @@ -0,0 +1,602 @@ + + + + + Cookies Names - AIL + + + + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + +
    +
    +
    + +{# {% include 'cookie-name/block_cookie_name_search.html' %}#} + +
    + + +
    + +
    +
    +
    Select a date range :
    + +
    +
    + +
    +
    +
    + +
    +
    + + +
    + + +
    +
    + +
    +
    +
    +
    +
    +
    + + {% if dict_objects %} + {% if date_from|string == date_to|string %} +

    {{ date_from }} Cookie Name:

    + {% else %} +

    {{ date_from }} to {{ date_to }} Cookie Name:

    + {% endif %} + + + + + + + + + + + + {% for obj_id in dict_objects %} + + + + + + + + {% endfor %} + +
    First SeenLast SeenTotalLast days
    {{ dict_objects[obj_id]['content'] }}{{ dict_objects[obj_id]['first_seen'] }}{{ dict_objects[obj_id]['last_seen'] }}{{ dict_objects[obj_id]['nb_seen'] }}
    + + + {% else %} + {% if show_objects %} + {% if date_from|string == date_to|string %} +

    {{ date_from }}, No Cookie Name

    + {% else %} +

    {{ date_from }} to {{ date_to }}, No Cookie Name

    + {% endif %} + {% endif %} + {% endif %} +
    + +
    +
    + + + + + + + + + + + + + + + + + diff --git a/var/www/templates/objects/cookie-name/block_cookie_name_search.html b/var/www/templates/objects/cookie-name/block_cookie_name_search.html new file mode 100644 index 00000000..d90afaf7 --- /dev/null +++ b/var/www/templates/objects/cookie-name/block_cookie_name_search.html @@ -0,0 +1,20 @@ +
    +
    +
    Titles Search:
    +
    +
    + + + + +
    +
    + + +
    +
    +
    +
    \ No newline at end of file diff --git a/var/www/templates/objects/cookie-name/search_cookie_name_result.html b/var/www/templates/objects/cookie-name/search_cookie_name_result.html new file mode 100644 index 00000000..34a4524a --- /dev/null +++ b/var/www/templates/objects/cookie-name/search_cookie_name_result.html @@ -0,0 +1,119 @@ + + + + + Titles - AIL + + + + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + + {% with page=dict_page['page'] %} + {% include 'title/block_titles_search.html' %} + {% endwith %} + + + + + + + + + + + + + + {% for obj_id in dict_objects %} + + + + + + + + {% endfor %} + +
    First SeenLast SeenTotalLast days
    + + {% if type_to_search == 'content' %} + {{ dict_objects[obj_id]['content'][:search_result[obj_id]['hl-start']] }}{{dict_objects[obj_id]['content'][search_result[obj_id]['hl-start']:search_result[obj_id]['hl-end']]}}{{ dict_objects[obj_id]['content'][search_result[obj_id]['hl-end']:] }} + {% else %} + {{ dict_objects[obj_id]['content'] }} + {% endif %} + + {{ dict_objects[obj_id]['first_seen'] }}{{ dict_objects[obj_id]['last_seen'] }}{{ dict_objects[obj_id]['nb_seen'] }}
    + + {% with page=dict_page['page'], nb_page_max=dict_page['nb_pages'], nb_first_elem=dict_page['nb_first_elem'], nb_last_elem=dict_page['nb_last_elem'], nb_all_elem=dict_page['nb_all_elem'] %} + {% set target_url=url_for('objects_title.objects_title_search') + "?search=" + to_search + "&search_type=" + type_to_search + "&case_sensitive=" + case_sensitive|string %} + {% include 'pagination.html' %} + {% endwith %} + +
    +
    +
    + + + + + + + + diff --git a/var/www/templates/sidebars/sidebar_objects.html b/var/www/templates/sidebars/sidebar_objects.html index 4f09e138..89239701 100644 --- a/var/www/templates/sidebars/sidebar_objects.html +++ b/var/www/templates/sidebars/sidebar_objects.html @@ -34,6 +34,12 @@ CVE
    • +
  • + + + Cookie Name + +
  • From e9539e640ba3f01189e11b7536e5380ed90724df Mon Sep 17 00:00:00 2001 From: Terrtia Date: Sun, 18 Jun 2023 15:09:09 +0200 Subject: [PATCH 041/238] fix: [crawler] fix incomplete response --- bin/crawlers/Crawler.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index a2966d9a..5f075038 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -186,7 +186,7 @@ class Crawler(AbstractModule): parent_id = task.get_parent() entries = self.lacus.get_capture(capture.uuid) - print(entries['status']) + print(entries.get('status')) self.har = task.get_har() self.screenshot = task.get_screenshot() # DEBUG @@ -218,12 +218,12 @@ class Crawler(AbstractModule): if 'error' in entries: # TODO IMPROVE ERROR MESSAGE self.logger.warning(str(entries['error'])) - print(entries['error']) + print(entries.get('error')) if entries.get('html'): print('retrieved content') # print(entries.get('html')) - if 'last_redirected_url' in entries and entries['last_redirected_url']: + if 'last_redirected_url' in entries and entries.get('last_redirected_url'): last_url = entries['last_redirected_url'] unpacked_last_url = crawlers.unpack_url(last_url) current_domain = unpacked_last_url['domain'] @@ -238,7 +238,7 @@ class Crawler(AbstractModule): else: last_url = f'http://{self.domain.id}' - if 'html' in entries and entries['html']: + if 'html' in entries and entries.get('html'): item_id = crawlers.create_item_id(self.items_dir, self.domain.id) print(item_id) gzip64encoded = crawlers.get_gzipped_b64_item(item_id, entries['html']) @@ -264,7 +264,7 @@ class Crawler(AbstractModule): # SCREENSHOT if self.screenshot: - if 'png' in entries and entries['png']: + if 'png' in entries and entries.get('png'): screenshot = Screenshots.create_screenshot(entries['png'], b64=False) if screenshot: if not screenshot.is_tags_safe(): @@ -278,7 +278,7 @@ class Crawler(AbstractModule): screenshot.add_correlation('domain', '', self.domain.id) # HAR if self.har: - if 'har' in entries and entries['har']: + if 'har' in entries and entries.get('har'): har_id = crawlers.create_har_id(self.date, item_id) crawlers.save_har(har_id, entries['har']) for cookie_name in crawlers.extract_cookies_names_from_har(entries['har']): From 84bc2c06694a1475140a4f19a1aff72229b0fc81 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Sun, 18 Jun 2023 16:09:57 +0200 Subject: [PATCH 042/238] chg: [correlation] add an option to remove max number of nodes if max_node == 0 --- bin/lib/correlations_engine.py | 2 +- var/www/blueprints/correlation.py | 2 +- var/www/templates/correlation/show_correlation.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index d1d7e3b6..bf234c9c 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -193,7 +193,7 @@ def _get_correlations_graph_node(links, nodes, obj_type, subtype, obj_id, level, if obj2_str_id == previous_str_obj: continue - if len(nodes) > max_nodes: + if len(nodes) > max_nodes != 0: break nodes.add(obj2_str_id) links.add((obj_str_id, obj2_str_id)) diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index b43cfe1e..bcf61729 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -46,7 +46,7 @@ def sanitise_graph_mode(graph_mode): def sanitise_nb_max_nodes(nb_max_nodes): try: nb_max_nodes = int(nb_max_nodes) - if nb_max_nodes < 2: + if nb_max_nodes < 2 and nb_max_nodes != 0: nb_max_nodes = 300 except (TypeError, ValueError): nb_max_nodes = 300 diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 667d0cb8..93ca5c2b 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -256,7 +256,7 @@
    - +
    From b4d3c6256d4f8cb4b55afce4ab76de55259a7c42 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Sun, 18 Jun 2023 16:31:26 +0200 Subject: [PATCH 043/238] fix: [title] fix title search empty result --- var/www/blueprints/objects_title.py | 1 + .../templates/objects/title/search_title_result.html | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/var/www/blueprints/objects_title.py b/var/www/blueprints/objects_title.py index ab6ffd25..3cac6b31 100644 --- a/var/www/blueprints/objects_title.py +++ b/var/www/blueprints/objects_title.py @@ -126,6 +126,7 @@ def objects_title_search(): dict_objects = titles.get_metas(dict_page['list_elem'], options={'sparkline'}) else: dict_objects = {} + dict_page = {} return render_template("search_title_result.html", dict_objects=dict_objects, search_result=search_result, dict_page=dict_page, diff --git a/var/www/templates/objects/title/search_title_result.html b/var/www/templates/objects/title/search_title_result.html index 34a4524a..d62f9daf 100644 --- a/var/www/templates/objects/title/search_title_result.html +++ b/var/www/templates/objects/title/search_title_result.html @@ -71,10 +71,12 @@ - {% with page=dict_page['page'], nb_page_max=dict_page['nb_pages'], nb_first_elem=dict_page['nb_first_elem'], nb_last_elem=dict_page['nb_last_elem'], nb_all_elem=dict_page['nb_all_elem'] %} - {% set target_url=url_for('objects_title.objects_title_search') + "?search=" + to_search + "&search_type=" + type_to_search + "&case_sensitive=" + case_sensitive|string %} - {% include 'pagination.html' %} - {% endwith %} + {% if dict_page %} + {% with page=dict_page['page'], nb_page_max=dict_page['nb_pages'], nb_first_elem=dict_page['nb_first_elem'], nb_last_elem=dict_page['nb_last_elem'], nb_all_elem=dict_page['nb_all_elem'] %} + {% set target_url=url_for('objects_title.objects_title_search') + "?search=" + to_search + "&search_type=" + type_to_search + "&case_sensitive=" + case_sensitive|string %} + {% include 'pagination.html' %} + {% endwith %} + {% endif %}
    From 6c21a8f8cfa545021a2159f82cb5284edbc1bb26 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 19 Jun 2023 08:55:56 +0200 Subject: [PATCH 044/238] fix: [daterange object] fix objects by date --- bin/lib/objects/abstract_daterange_object.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/bin/lib/objects/abstract_daterange_object.py b/bin/lib/objects/abstract_daterange_object.py index 1465c371..b96c5ec4 100755 --- a/bin/lib/objects/abstract_daterange_object.py +++ b/bin/lib/objects/abstract_daterange_object.py @@ -135,17 +135,21 @@ class AbstractDaterangeObject(AbstractObject, ABC): self.update_daterange(date) update_obj_date(date, self.type) - # Correlations - self.add_correlation(obj_type, subtype, obj_id) - if obj_type == 'item': # NB Object seen by day TODO if not self.is_correlated(obj_type, subtype, obj_id): # nb seen by day r_object.zincrby(f'{self.type}:date:{date}', 1, self.id) + + # Correlations + self.add_correlation(obj_type, subtype, obj_id) + if is_crawled(obj_id): # Domain domain = get_item_domain(obj_id) self.add_correlation('domain', '', domain) else: + # Correlations + self.add_correlation(obj_type, subtype, obj_id) + # TODO Don't increase on reprocess r_object.zincrby(f'{self.type}:date:{date}', 1, self.id) # r_object.zincrby(f'{self.type}:obj:{obj_type}', 1, self.id) From b64c70e0cd54ea2c08edf2d13a6f6f73b0556e1a Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 19 Jun 2023 10:53:15 +0200 Subject: [PATCH 045/238] fix: [retro hunt] fix object tag queue + decoded content --- bin/lib/objects/Decodeds.py | 16 ++++++++++++++-- bin/trackers/Retro_Hunt.py | 12 +++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/bin/lib/objects/Decodeds.py b/bin/lib/objects/Decodeds.py index a936a1a9..001f7dfd 100755 --- a/bin/lib/objects/Decodeds.py +++ b/bin/lib/objects/Decodeds.py @@ -111,13 +111,25 @@ class Decoded(AbstractDaterangeObject): def get_rel_path(self, mimetype=None): if not mimetype: mimetype = self.get_mimetype() + if not mimetype: + self.logger.warning(f'Decoded {self.id}: Empty mimetype') + return None return os.path.join(HASH_DIR, mimetype, self.id[0:2], self.id) def get_filepath(self, mimetype=None): - return os.path.join(os.environ['AIL_HOME'], self.get_rel_path(mimetype=mimetype)) + rel_path = self.get_rel_path(mimetype=mimetype) + if not rel_path: + return None + else: + return os.path.join(os.environ['AIL_HOME'], rel_path) def get_content(self, mimetype=None, r_type='str'): filepath = self.get_filepath(mimetype=mimetype) + if not filepath: + if r_type == 'str': + return '' + else: + return b'' if r_type == 'str': with open(filepath, 'r') as f: content = f.read() @@ -442,7 +454,7 @@ def get_all_decodeds_objects(filters={}): if i >= len(files): files = [] for file in files: - yield Decoded(file).id + yield Decoded(file) ############################################################################ diff --git a/bin/trackers/Retro_Hunt.py b/bin/trackers/Retro_Hunt.py index a280e3a1..2c63abd0 100755 --- a/bin/trackers/Retro_Hunt.py +++ b/bin/trackers/Retro_Hunt.py @@ -128,10 +128,16 @@ class Retro_Hunt_Module(AbstractModule): self.retro_hunt.add(self.obj.get_type(), self.obj.get_subtype(), obj_id) # TODO FILTER Tags + + # TODO refactor Tags module for all object type # Tags - for tag in self.tags: - msg = f'{tag};{id}' - self.add_message_to_queue(msg, 'Tags') + if self.obj.get_type() == 'item': + for tag in self.tags: + msg = f'{tag};{obj_id}' + self.add_message_to_queue(msg, 'Tags') + else: + for tag in self.tags: + self.obj.add_tag(tag) # # Mails # EXPORTER MAILS From c02b9f2a815e9ca42a68cfe9b54e8d2519e235dd Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 19 Jun 2023 15:17:41 +0200 Subject: [PATCH 046/238] fix: [IPAdress module] empty queue if no IP ranges provided --- bin/modules/IPAddress.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/bin/modules/IPAddress.py b/bin/modules/IPAddress.py index 5f86bf90..1a5d53c7 100755 --- a/bin/modules/IPAddress.py +++ b/bin/modules/IPAddress.py @@ -43,14 +43,15 @@ class IPAddress(AbstractModule): networks = config_loader.get_config_str("IP", "networks") if not networks: print('No IP ranges provided') - sys.exit(0) - try: - for network in networks.split(","): - self.ip_networks.add(IPv4Network(network)) - print(f'IP Range To Search: {network}') - except: - print('Please provide a list of valid IP addresses') - sys.exit(0) + # sys.exit(0) + else: + try: + for network in networks.split(","): + self.ip_networks.add(IPv4Network(network)) + print(f'IP Range To Search: {network}') + except: + print('Please provide a list of valid IP addresses') + sys.exit(0) self.re_ipv4 = r'(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)' re.compile(self.re_ipv4) @@ -62,6 +63,9 @@ class IPAddress(AbstractModule): self.logger.info(f"Module {self.module_name} initialized") def compute(self, message, r_result=False): + if not self.ip_networks: + return None + item = Item(message) content = item.get_content() From 501d10bbbd4ab603c9572e1fea36dbdfad6178c0 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 20 Jun 2023 08:11:44 +0200 Subject: [PATCH 047/238] chg: [crawler] auto tag crawled domains --- bin/crawlers/Crawler.py | 5 +- bin/lib/crawlers.py | 58 +++++++++++++++---- var/www/blueprints/crawler_splash.py | 35 ++++++++++- .../crawler_splash/crawler_manual.html | 23 +++++++- .../crawler_splash/crawler_schedule_uuid.html | 8 +++ .../crawler_scheduler_dashboard.html | 9 ++- 6 files changed, 121 insertions(+), 17 deletions(-) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index 5f075038..be615993 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -200,10 +200,13 @@ class Crawler(AbstractModule): self.save_capture_response(parent_id, entries) self.domain.update_daterange(self.date.replace('/', '')) - # Origin + History + # Origin + History + tags if self.root_item: self.domain.set_last_origin(parent_id) self.domain.add_history(epoch, root_item=self.root_item) + # Tags + for tag in task.get_tags(): + self.domain.add_tag(tag) elif self.domain.was_up(): self.domain.add_history(epoch, root_item=epoch) diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 1883490e..300edb66 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -967,6 +967,7 @@ class CrawlerScheduler: task_uuid = create_task(meta['url'], depth=meta['depth'], har=meta['har'], screenshot=meta['screenshot'], header=meta['header'], cookiejar=meta['cookiejar'], proxy=meta['proxy'], + tags=meta['tags'], user_agent=meta['user_agent'], parent='scheduler', priority=40) if task_uuid: schedule.set_task(task_uuid) @@ -1069,6 +1070,14 @@ class CrawlerSchedule: def _set_field(self, field, value): return r_crawler.hset(f'schedule:{self.uuid}', field, value) + def get_tags(self): + return r_crawler.smembers(f'schedule:tags:{self.uuid}') + + def set_tags(self, tags=[]): + for tag in tags: + r_crawler.sadd(f'schedule:tags:{self.uuid}', tag) + # Tag.create_custom_tag(tag) + def get_meta(self, ui=False): meta = { 'uuid': self.uuid, @@ -1083,6 +1092,7 @@ class CrawlerSchedule: 'cookiejar': self.get_cookiejar(), 'header': self.get_header(), 'proxy': self.get_proxy(), + 'tags': self.get_tags(), } status = self.get_status() if ui: @@ -1098,6 +1108,7 @@ class CrawlerSchedule: meta = {'uuid': self.uuid, 'url': self.get_url(), 'user': self.get_user(), + 'tags': self.get_tags(), 'next_run': self.get_next_run(r_str=True)} status = self.get_status() if isinstance(status, ScheduleStatus): @@ -1106,7 +1117,7 @@ class CrawlerSchedule: return meta def create(self, frequency, user, url, - depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None, user_agent=None): + depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None, user_agent=None, tags=[]): if self.exists(): raise Exception('Error: Monitor already exists') @@ -1135,6 +1146,9 @@ class CrawlerSchedule: if user_agent: self._set_field('user_agent', user_agent) + if tags: + self.set_tags(tags) + r_crawler.sadd('scheduler:schedules', self.uuid) def delete(self): @@ -1148,12 +1162,13 @@ class CrawlerSchedule: # delete meta r_crawler.delete(f'schedule:{self.uuid}') + r_crawler.delete(f'schedule:tags:{self.uuid}') r_crawler.srem('scheduler:schedules', self.uuid) -def create_schedule(frequency, user, url, depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None, user_agent=None): +def create_schedule(frequency, user, url, depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None, user_agent=None, tags=[]): schedule_uuid = gen_uuid() schedule = CrawlerSchedule(schedule_uuid) - schedule.create(frequency, user, url, depth=depth, har=har, screenshot=screenshot, header=header, cookiejar=cookiejar, proxy=proxy, user_agent=user_agent) + schedule.create(frequency, user, url, depth=depth, har=har, screenshot=screenshot, header=header, cookiejar=cookiejar, proxy=proxy, user_agent=user_agent, tags=tags) return schedule_uuid # TODO sanityze UUID @@ -1289,6 +1304,11 @@ def get_captures_status(): status.append(meta) return status +def delete_captures(): + for capture_uuid in get_crawler_captures(): + capture = CrawlerCapture(capture_uuid) + capture.delete() + ##-- CRAWLER STATE --## @@ -1371,6 +1391,14 @@ class CrawlerTask: def _set_field(self, field, value): return r_crawler.hset(f'crawler:task:{self.uuid}', field, value) + def get_tags(self): + return r_crawler.smembers(f'crawler:task:tags:{self.uuid}') + + def set_tags(self, tags): + for tag in tags: + r_crawler.sadd(f'crawler:task:tags:{self.uuid}', tag) + # Tag.create_custom_tag(tag) + def get_meta(self): meta = { 'uuid': self.uuid, @@ -1385,6 +1413,7 @@ class CrawlerTask: 'header': self.get_header(), 'proxy': self.get_proxy(), 'parent': self.get_parent(), + 'tags': self.get_tags(), } return meta @@ -1392,7 +1421,7 @@ class CrawlerTask: # TODO SANITIZE PRIORITY # PRIORITY: discovery = 0/10, feeder = 10, manual = 50, auto = 40, test = 100 def create(self, url, depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None, - user_agent=None, parent='manual', priority=0): + user_agent=None, tags=[], parent='manual', priority=0): if self.exists(): raise Exception('Error: Task already exists') @@ -1423,7 +1452,7 @@ class CrawlerTask: # TODO SANITIZE COOKIEJAR -> UUID # Check if already in queue - hash_query = get_task_hash(url, domain, depth, har, screenshot, priority, proxy, cookiejar, user_agent, header) + hash_query = get_task_hash(url, domain, depth, har, screenshot, priority, proxy, cookiejar, user_agent, header, tags) if r_crawler.hexists(f'crawler:queue:hash', hash_query): self.uuid = r_crawler.hget(f'crawler:queue:hash', hash_query) return self.uuid @@ -1444,6 +1473,9 @@ class CrawlerTask: if user_agent: self._set_field('user_agent', user_agent) + if tags: + self.set_tags(tags) + r_crawler.hset('crawler:queue:hash', hash_query, self.uuid) self._set_field('hash', hash_query) r_crawler.zadd('crawler:queue', {self.uuid: priority}) @@ -1483,10 +1515,10 @@ class CrawlerTask: # TODO move to class ??? -def get_task_hash(url, domain, depth, har, screenshot, priority, proxy, cookiejar, user_agent, header): +def get_task_hash(url, domain, depth, har, screenshot, priority, proxy, cookiejar, user_agent, header, tags): to_enqueue = {'domain': domain, 'depth': depth, 'har': har, 'screenshot': screenshot, 'priority': priority, 'proxy': proxy, 'cookiejar': cookiejar, 'user_agent': user_agent, - 'header': header} + 'header': header, 'tags': tags} if priority != 0: to_enqueue['url'] = url return hashlib.sha512(pickle.dumps(to_enqueue)).hexdigest() @@ -1502,7 +1534,7 @@ def add_task_to_lacus_queue(): # PRIORITY: discovery = 0/10, feeder = 10, manual = 50, auto = 40, test = 100 def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None, - user_agent=None, parent='manual', priority=0, task_uuid=None): + user_agent=None, tags=[], parent='manual', priority=0, task_uuid=None): if task_uuid: if CrawlerTask(task_uuid).exists(): task_uuid = gen_uuid() @@ -1510,7 +1542,7 @@ def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar= task_uuid = gen_uuid() task = CrawlerTask(task_uuid) task_uuid = task.create(url, depth=depth, har=har, screenshot=screenshot, header=header, cookiejar=cookiejar, - proxy=proxy, user_agent=user_agent, parent=parent, priority=priority) + proxy=proxy, user_agent=user_agent, tags=tags, parent=parent, priority=priority) return task_uuid @@ -1586,15 +1618,17 @@ def api_add_crawler_task(data, user_id=None): if verify[1] != 200: return verify + tags = data.get('tags', []) + if frequency: # TODO verify user return create_schedule(frequency, user_id, url, depth=depth_limit, har=har, screenshot=screenshot, header=None, - cookiejar=cookiejar_uuid, proxy=proxy, user_agent=None), 200 + cookiejar=cookiejar_uuid, proxy=proxy, user_agent=None, tags=tags), 200 else: # TODO HEADERS # TODO USER AGENT return create_task(url, depth=depth_limit, har=har, screenshot=screenshot, header=None, - cookiejar=cookiejar_uuid, proxy=proxy, user_agent=None, + cookiejar=cookiejar_uuid, proxy=proxy, user_agent=None, tags=tags, parent='manual', priority=90), 200 @@ -1870,6 +1904,8 @@ def test_ail_crawlers(): load_blacklist() # if __name__ == '__main__': +# delete_captures() + # item_id = 'crawled/2023/02/20/data.gz' # item = Item(item_id) # content = item.get_content() diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index 52ac5be9..39d84971 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -96,7 +96,8 @@ def manual(): is_manager_connected=crawlers.get_lacus_connection_metadata(), crawlers_types=crawlers_types, proxies=proxies, - l_cookiejar=l_cookiejar) + l_cookiejar=l_cookiejar, + tags_selector_data=Tag.get_tags_selector_data()) @crawler_splash.route("/crawlers/send_to_spider", methods=['POST']) @@ -113,6 +114,34 @@ def send_to_spider(): depth_limit = request.form.get('depth_limit') cookiejar_uuid = request.form.get('cookiejar') + # TAGS + tags = request.form.get("tags", []) + taxonomies_tags = request.form.get('taxonomies_tags') + if taxonomies_tags: + try: + taxonomies_tags = json.loads(taxonomies_tags) + except: + taxonomies_tags = [] + else: + taxonomies_tags = [] + galaxies_tags = request.form.get('galaxies_tags') + if galaxies_tags: + try: + galaxies_tags = json.loads(galaxies_tags) + except: + galaxies_tags = [] + else: + galaxies_tags = [] + # custom tags + if tags: + tags = tags.split() + else: + tags = [] + escaped = [] + for tag in tags: + escaped.append(tag) + tags = escaped + taxonomies_tags + galaxies_tags + # Frequency if request.form.get('crawler_scheduler'): frequency = request.form.get('frequency') @@ -147,6 +176,8 @@ def send_to_spider(): data['proxy'] = proxy if cookiejar_uuid: data['cookiejar'] = cookiejar_uuid + if tags: + data['tags'] = tags # print(data) res = crawlers.api_add_crawler_task(data, user_id=user_id) @@ -163,6 +194,7 @@ def scheduler_dashboard(): # print(schedulers) # TODO list currently queued ? return render_template("crawler_scheduler_dashboard.html", + bootstrap_label=bootstrap_label, schedulers=schedulers, is_manager_connected=crawlers.get_lacus_connection_metadata()) @@ -176,6 +208,7 @@ def schedule_show(): abort(404) meta = schedule.get_meta(ui=True) return render_template("crawler_schedule_uuid.html", + bootstrap_label=bootstrap_label, meta=meta) @crawler_splash.route("/crawlers/schedule/delete", methods=['GET']) diff --git a/var/www/templates/crawler/crawler_splash/crawler_manual.html b/var/www/templates/crawler/crawler_splash/crawler_manual.html index 94e376cf..72a5e47b 100644 --- a/var/www/templates/crawler/crawler_splash/crawler_manual.html +++ b/var/www/templates/crawler/crawler_splash/crawler_manual.html @@ -8,14 +8,16 @@ + - - - + + + + @@ -119,6 +121,21 @@ +
    +
    + Tags +
    +
    +
    +
    +
    +
    + +
    + {% include 'tags/block_tags_selector.html' %} +
    +
    + diff --git a/var/www/templates/crawler/crawler_splash/crawler_schedule_uuid.html b/var/www/templates/crawler/crawler_splash/crawler_schedule_uuid.html index e925ff3e..86325c5b 100644 --- a/var/www/templates/crawler/crawler_splash/crawler_schedule_uuid.html +++ b/var/www/templates/crawler/crawler_splash/crawler_schedule_uuid.html @@ -72,6 +72,14 @@ {% endif %} + + Tags + + {%for tag in meta['tags']%} + {{ tag }} + {%endfor%} + + diff --git a/var/www/templates/crawler/crawler_splash/crawler_scheduler_dashboard.html b/var/www/templates/crawler/crawler_splash/crawler_scheduler_dashboard.html index 160365d0..eace714b 100644 --- a/var/www/templates/crawler/crawler_splash/crawler_scheduler_dashboard.html +++ b/var/www/templates/crawler/crawler_splash/crawler_scheduler_dashboard.html @@ -45,7 +45,14 @@ {% for meta in schedulers %} -     {{ meta['url'] }} + + {{ meta['url'] }} +
    + {% for tag in meta['tags'] %} + {{ tag }} + {%endfor%} +
    + {{ meta['status'] }} {% if not meta['next_run'] %} From 4567c9d40081b6cb2abf2f0f9daa017a7ec0b176 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 20 Jun 2023 11:23:58 +0200 Subject: [PATCH 048/238] chg: [correlation graph] show message if max_nodes reached + fix cookie-name sparkline --- bin/lib/correlations_engine.py | 15 ++++++++++----- bin/lib/objects/ail_objects.py | 18 ++++++++++-------- var/www/blueprints/correlation.py | 2 +- .../correlation/show_correlation.html | 10 ++++++++++ .../investigations/investigations.html | 4 ++-- 5 files changed, 33 insertions(+), 16 deletions(-) diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index bf234c9c..8e29837d 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -167,20 +167,22 @@ def delete_obj_correlations(obj_type, subtype, obj_id): def get_obj_str_id(obj_type, subtype, obj_id): if subtype is None: subtype = '' - return f'{obj_type};{subtype};{obj_id}' + return f'{obj_type}:{subtype}:{obj_id}' def get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, flask_context=False): links = set() nodes = set() + meta = {'complete': True, 'objs': set()} obj_str_id = get_obj_str_id(obj_type, subtype, obj_id) - _get_correlations_graph_node(links, nodes, obj_type, subtype, obj_id, level, max_nodes, filter_types=filter_types, previous_str_obj='') - return obj_str_id, nodes, links + _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=filter_types, previous_str_obj='') + return obj_str_id, nodes, links, meta -def _get_correlations_graph_node(links, nodes, obj_type, subtype, obj_id, level, max_nodes, filter_types=[], previous_str_obj=''): +def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=[], previous_str_obj=''): obj_str_id = get_obj_str_id(obj_type, subtype, obj_id) + meta['objs'].add(obj_str_id) nodes.add(obj_str_id) obj_correlations = get_correlations(obj_type, subtype, obj_id, filter_types=filter_types) @@ -189,15 +191,18 @@ def _get_correlations_graph_node(links, nodes, obj_type, subtype, obj_id, level, for str_obj in obj_correlations[correl_type]: subtype2, obj2_id = str_obj.split(':', 1) obj2_str_id = get_obj_str_id(correl_type, subtype2, obj2_id) + meta['objs'].add(obj2_str_id) if obj2_str_id == previous_str_obj: continue if len(nodes) > max_nodes != 0: + meta['complete'] = False break nodes.add(obj2_str_id) links.add((obj_str_id, obj2_str_id)) if level > 0: next_level = level - 1 - _get_correlations_graph_node(links, nodes, correl_type, subtype2, obj2_id, next_level, max_nodes, filter_types=filter_types, previous_str_obj=obj_str_id) + _get_correlations_graph_node(links, nodes, meta, correl_type, subtype2, obj2_id, next_level, max_nodes, filter_types=filter_types, previous_str_obj=obj_str_id) + diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index b17ce3e6..df598a70 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* - import os import sys @@ -169,7 +168,7 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False): obj = get_object(obj_type, subtype, id) meta = obj.get_meta() meta['icon'] = obj.get_svg_icon() - if subtype or obj_type == 'cve' or obj_type == 'title' or obj_type == 'favicon': + if subtype or obj_type == 'cookie-name' or obj_type == 'cve' or obj_type == 'title' or obj_type == 'favicon': meta['sparkline'] = obj.get_sparkline() if obj_type == 'cve': meta['cve_search'] = obj.get_cve_search() @@ -396,7 +395,7 @@ def create_correlation_graph_links(links_set): def create_correlation_graph_nodes(nodes_set, obj_str_id, flask_context=True): graph_nodes_list = [] for node_id in nodes_set: - obj_type, subtype, obj_id = node_id.split(';', 2) + obj_type, subtype, obj_id = node_id.split(':', 2) dict_node = {'id': node_id} dict_node['style'] = get_object_svg(obj_type, subtype, obj_id) @@ -418,12 +417,15 @@ def create_correlation_graph_nodes(nodes_set, obj_str_id, flask_context=True): def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, flask_context=False): - obj_str_id, nodes, links = correlations_engine.get_correlations_graph_nodes_links(obj_type, subtype, obj_id, - filter_types=filter_types, - max_nodes=max_nodes, level=level, - flask_context=flask_context) + obj_str_id, nodes, links, meta = correlations_engine.get_correlations_graph_nodes_links(obj_type, subtype, obj_id, + filter_types=filter_types, + max_nodes=max_nodes, level=level, + flask_context=flask_context) + # print(meta) + meta['objs'] = list(meta['objs']) return {"nodes": create_correlation_graph_nodes(nodes, obj_str_id, flask_context=flask_context), - "links": create_correlation_graph_links(links)} + "links": create_correlation_graph_links(links), + "meta": meta} # --- CORRELATION --- # diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index bcf61729..f6e7feda 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -156,7 +156,7 @@ def show_correlation(): @login_read_only def get_description(): object_id = request.args.get('object_id') - object_id = object_id.split(';') + object_id = object_id.split(':') # unpack object_id # # TODO: put me in lib if len(object_id) == 3: object_type = object_id[0] diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 93ca5c2b..95aa922c 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -162,6 +162,9 @@  Resize Graph +
    +  Graph Incomplete, Max Nodes Reached. +
    @@ -350,6 +353,7 @@ var all_graph = {}; $(document).ready(function(){ + $("#incomplete_graph").hide(); $("#page-Decoded").addClass("active"); all_graph.node_graph = create_graph("{{ url_for('correlation.graph_node_json') }}?id={{ dict_object["correlation_id"] }}&type={{ dict_object["object_type"] }}&mode={{ dict_object["mode"] }}&level={{ dict_object["level"] }}&filter={{ dict_object["filter_str"] }}&max_nodes={{dict_object["max_nodes"]}}{% if 'type_id' in dict_object["metadata"] %}&subtype={{ dict_object["metadata"]["type_id"] }}{% endif %}"); @@ -526,6 +530,12 @@ d3.json(url) // Loading ... $("#graph_loading").remove(); + if (!data.meta.complete){ + $("#incomplete_graph").show(); + } + + + }) .catch(function(error) { $("#graph_loading").remove() diff --git a/var/www/templates/investigations/investigations.html b/var/www/templates/investigations/investigations.html index fc23891f..a15eea15 100644 --- a/var/www/templates/investigations/investigations.html +++ b/var/www/templates/investigations/investigations.html @@ -11,8 +11,8 @@ - - + + From b459498db23f3b908bfc1b30793a298d786fdd0e Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 22 Jun 2023 15:38:04 +0200 Subject: [PATCH 049/238] chg: [queues] track object + check if object processed --- bin/core/Sync_module.py | 44 +++---- bin/importer/FeederImporter.py | 2 +- bin/importer/FileImporter.py | 2 +- bin/importer/PystemonImporter.py | 2 +- bin/importer/ZMQImporter.py | 2 +- bin/lib/ail_queues.py | 166 ++++++++++++++++--------- bin/lib/objects/ail_objects.py | 24 ++-- bin/modules/ApiKey.py | 16 +-- bin/modules/Categ.py | 6 +- bin/modules/Credential.py | 9 +- bin/modules/CreditCards.py | 8 +- bin/modules/Cryptocurrencies.py | 10 +- bin/modules/CveModule.py | 9 +- bin/modules/Decoder.py | 7 +- bin/modules/DomClassifier.py | 6 +- bin/modules/Duplicates.py | 2 +- bin/modules/Global.py | 4 +- bin/modules/Hosts.py | 6 +- bin/modules/IPAddress.py | 6 +- bin/modules/Iban.py | 6 +- bin/modules/Indexer.py | 6 +- bin/modules/Keys.py | 60 ++++----- bin/modules/Languages.py | 2 +- bin/modules/LibInjection.py | 8 +- bin/modules/MISP_Thehive_Auto_Push.py | 5 +- bin/modules/Mail.py | 8 +- bin/modules/Mixer.py | 2 +- bin/modules/Onion.py | 8 +- bin/modules/PgpDump.py | 6 +- bin/modules/Phone.py | 6 +- bin/modules/SQLInjectionDetection.py | 9 +- bin/modules/SubmitPaste.py | 5 +- bin/modules/Tags.py | 27 ++-- bin/modules/Telegram.py | 6 +- bin/modules/Tools.py | 6 +- bin/modules/Urls.py | 12 +- bin/modules/Zerobins.py | 5 +- bin/modules/abstract_module.py | 41 +++++- bin/trackers/Tracker_Regex.py | 7 +- bin/trackers/Tracker_Term.py | 7 +- bin/trackers/Tracker_Typo_Squatting.py | 8 +- bin/trackers/Tracker_Yara.py | 7 +- 42 files changed, 311 insertions(+), 277 deletions(-) diff --git a/bin/core/Sync_module.py b/bin/core/Sync_module.py index e37d48d6..22f814f4 100755 --- a/bin/core/Sync_module.py +++ b/bin/core/Sync_module.py @@ -53,37 +53,23 @@ class Sync_module(AbstractModule): print('sync queues refreshed') print(self.dict_sync_queues) - # Extract object from message - # # TODO: USE JSON DICT ???? - mess_split = message.split(';') - if len(mess_split) == 3: - obj_type = mess_split[0] - obj_subtype = mess_split[1] - obj_id = mess_split[2] + obj = self.get_obj() - # OBJECT => Item - # if obj_type == 'item': - obj = Item(obj_id) + tags = obj.get_tags() - tags = obj.get_tags() - - # check filter + tags - # print(message) - for queue_uuid in self.dict_sync_queues: - filter_tags = self.dict_sync_queues[queue_uuid]['filter'] - if filter_tags and tags: - # print('tags: {tags} filter: {filter_tags}') - if filter_tags.issubset(tags): - obj_dict = obj.get_default_meta() - # send to queue push and/or pull - for dict_ail in self.dict_sync_queues[queue_uuid]['ail_instances']: - print(f'ail_uuid: {dict_ail["ail_uuid"]} obj: {message}') - ail_2_ail.add_object_to_sync_queue(queue_uuid, dict_ail['ail_uuid'], obj_dict, - push=dict_ail['push'], pull=dict_ail['pull']) - - else: - # Malformed message - raise Exception(f'too many values to unpack (expected 3) given {len(mess_split)} with message {message}') + # check filter + tags + # print(message) + for queue_uuid in self.dict_sync_queues: + filter_tags = self.dict_sync_queues[queue_uuid]['filter'] + if filter_tags and tags: + # print('tags: {tags} filter: {filter_tags}') + if filter_tags.issubset(tags): + obj_dict = obj.get_default_meta() + # send to queue push and/or pull + for dict_ail in self.dict_sync_queues[queue_uuid]['ail_instances']: + print(f'ail_uuid: {dict_ail["ail_uuid"]} obj: {message}') + ail_2_ail.add_object_to_sync_queue(queue_uuid, dict_ail['ail_uuid'], obj_dict, + push=dict_ail['push'], pull=dict_ail['pull']) if __name__ == '__main__': diff --git a/bin/importer/FeederImporter.py b/bin/importer/FeederImporter.py index e7a06132..7acd6ae9 100755 --- a/bin/importer/FeederImporter.py +++ b/bin/importer/FeederImporter.py @@ -113,7 +113,7 @@ class FeederModuleImporter(AbstractModule): # TODO HANDLE Invalid JSON json_data = json.loads(message) relay_message = self.importer.importer(json_data) - self.add_message_to_queue(relay_message) + self.add_message_to_queue(message=relay_message) # Launch Importer diff --git a/bin/importer/FileImporter.py b/bin/importer/FileImporter.py index 410cc65d..4a926a41 100755 --- a/bin/importer/FileImporter.py +++ b/bin/importer/FileImporter.py @@ -46,7 +46,7 @@ class FileImporter(AbstractImporter): message = self.create_message(item_id, content, gzipped=gzipped, source='dir_import') if message: - self.add_message_to_queue(message) + self.add_message_to_queue(message=message) class DirImporter(AbstractImporter): def __init__(self): diff --git a/bin/importer/PystemonImporter.py b/bin/importer/PystemonImporter.py index 536801ba..df7d8d3a 100755 --- a/bin/importer/PystemonImporter.py +++ b/bin/importer/PystemonImporter.py @@ -74,7 +74,7 @@ class PystemonModuleImporter(AbstractModule): return self.importer.importer() def compute(self, message): - self.add_message_to_queue(message) + self.add_message_to_queue(message=message) if __name__ == '__main__': diff --git a/bin/importer/ZMQImporter.py b/bin/importer/ZMQImporter.py index dd9986d2..91728cf9 100755 --- a/bin/importer/ZMQImporter.py +++ b/bin/importer/ZMQImporter.py @@ -73,7 +73,7 @@ class ZMQModuleImporter(AbstractModule): for message in messages: message = message.decode() print(message.split(' ', 1)[0]) - self.add_message_to_queue(message) + self.add_message_to_queue(message=message) if __name__ == '__main__': diff --git a/bin/lib/ail_queues.py b/bin/lib/ail_queues.py index 81014024..3ab68708 100755 --- a/bin/lib/ail_queues.py +++ b/bin/lib/ail_queues.py @@ -6,6 +6,8 @@ import sys import datetime import time +from hashlib import sha256 + sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages @@ -15,10 +17,16 @@ from lib.ConfigLoader import ConfigLoader config_loader = ConfigLoader() r_queues = config_loader.get_redis_conn("Redis_Queues") +r_obj_process = config_loader.get_redis_conn("Redis_Process") config_loader = None MODULES_FILE = os.path.join(os.environ['AIL_HOME'], 'configs', 'modules.cfg') +# # # # # # # # +# # +# AIL QUEUE # +# # +# # # # # # # # class AILQueue: @@ -60,16 +68,26 @@ class AILQueue: # Update queues stats r_queues.hset('queues', self.name, self.get_nb_messages()) r_queues.hset(f'modules', f'{self.pid}:{self.name}', int(time.time())) + # Get Message message = r_queues.lpop(f'queue:{self.name}:in') if not message: return None else: - # TODO SAVE CURRENT ITEMS (OLD Module information) + row_mess = message.split(';', 1) + if len(row_mess) != 2: + return None, None, message + # raise Exception(f'Error: queue {self.name}, no AIL object provided') + else: + obj_global_id, mess = row_mess + sha256_mess = sha256(message.encode()).hexdigest() + add_processed_obj(obj_global_id, sha256_mess, module=self.name) + return obj_global_id, sha256_mess, mess - return message + def end_message(self, obj_global_id, sha256_mess): + end_processed_obj(obj_global_id, sha256_mess, module=self.name) - def send_message(self, message, queue_name=None): + def send_message(self, obj_global_id, message='', queue_name=None): if not self.subscribers_modules: raise ModuleQueueError('This Module don\'t have any subscriber') if queue_name: @@ -80,8 +98,17 @@ class AILQueue: raise ModuleQueueError('Queue name required. This module push to multiple queues') queue_name = list(self.subscribers_modules)[0] + message = f'{obj_global_id};{message}' + if obj_global_id != '::': + sha256_mess = sha256(message.encode()).hexdigest() + else: + sha256_mess = None + # Add message to all modules for module_name in self.subscribers_modules[queue_name]: + if sha256_mess: + add_processed_obj(obj_global_id, sha256_mess, queue=module_name) + r_queues.rpush(f'queue:{module_name}:in', message) # stats nb_mess = r_queues.llen(f'queue:{module_name}:in') @@ -98,6 +125,7 @@ class AILQueue: def error(self): r_queues.hdel(f'modules', f'{self.pid}:{self.name}') + def get_queues_modules(): return r_queues.hkeys('queues') @@ -132,6 +160,74 @@ def get_modules_queues_stats(): def clear_modules_queues_stats(): r_queues.delete('modules') + +# # # # # # # # # +# # +# OBJ QUEUES # PROCESS ?? +# # +# # # # # # # # # + + +def get_processed_objs(): + return r_obj_process.smembers(f'objs:process') + +def get_processed_objs_by_type(obj_type): + return r_obj_process.zrange(f'objs:process:{obj_type}', 0, -1) + +def is_processed_obj_queued(obj_global_id): + return r_obj_process.exists(f'obj:queues:{obj_global_id}') + +def is_processed_obj_moduled(obj_global_id): + return r_obj_process.exists(f'obj:modules:{obj_global_id}') + +def is_processed_obj(obj_global_id): + return is_processed_obj_queued(obj_global_id) and is_processed_obj_moduled(obj_global_id) + +def get_processed_obj_modules(obj_global_id): + return r_obj_process.zrange(f'obj:modules:{obj_global_id}', 0, -1) + +def get_processed_obj_queues(obj_global_id): + return r_obj_process.zrange(f'obj:queues:{obj_global_id}', 0, -1) + +def get_processed_obj(obj_global_id): + return {'modules': get_processed_obj_modules(obj_global_id), 'queues': get_processed_obj_queues(obj_global_id)} + +def add_processed_obj(obj_global_id, sha256_mess, module=None, queue=None): + obj_type = obj_global_id.split(':', 1)[0] + new_obj = r_obj_process.sadd(f'objs:process', obj_global_id) + # first process: + if new_obj: + r_obj_process.zadd(f'objs:process:{obj_type}', {obj_global_id: int(time.time())}) + if queue: + r_obj_process.zadd(f'obj:queues:{obj_global_id}', {f'{queue}:{sha256_mess}': int(time.time())}) + if module: + r_obj_process.zadd(f'obj:modules:{obj_global_id}', {f'{module}:{sha256_mess}': int(time.time())}) + r_obj_process.zrem(f'obj:queues:{obj_global_id}', f'{module}:{sha256_mess}') + +def end_processed_obj(obj_global_id, sha256_mess, module=None, queue=None): + if queue: + r_obj_process.zrem(f'obj:queues:{obj_global_id}', f'{queue}:{sha256_mess}') + if module: + r_obj_process.zrem(f'obj:modules:{obj_global_id}', f'{module}:{sha256_mess}') + + # TODO HANDLE QUEUE DELETE + # process completed + if not is_processed_obj(obj_global_id): + obj_type = obj_global_id.split(':', 1)[0] + r_obj_process.zrem(f'objs:process:{obj_type}', obj_global_id) + r_obj_process.srem(f'objs:process', obj_global_id) + + r_obj_process.sadd(f'objs:processed', obj_global_id) # TODO use list ?????? + +################################################################################### + + +# # # # # # # # +# # +# GRAPH # +# # +# # # # # # # # + def get_queue_digraph(): queues_ail = {} modules = {} @@ -223,64 +319,10 @@ def save_queue_digraph(): sys.exit(1) -########################################################################################### -########################################################################################### -########################################################################################### -########################################################################################### -########################################################################################### - -# def get_all_queues_name(): -# return r_queues.hkeys('queues') -# -# def get_all_queues_dict_with_nb_elem(): -# return r_queues.hgetall('queues') -# -# def get_all_queues_with_sorted_nb_elem(): -# res = r_queues.hgetall('queues') -# res = sorted(res.items()) -# return res -# -# def get_module_pid_by_queue_name(queue_name): -# return r_queues.smembers('MODULE_TYPE_{}'.format(queue_name)) -# -# # # TODO: remove last msg part -# def get_module_last_process_start_time(queue_name, module_pid): -# res = r_queues.get('MODULE_{}_{}'.format(queue_name, module_pid)) -# if res: -# return res.split(',')[0] -# return None -# -# def get_module_last_msg(queue_name, module_pid): -# return r_queues.get('MODULE_{}_{}_PATH'.format(queue_name, module_pid)) -# -# def get_all_modules_queues_stats(): -# all_modules_queues_stats = [] -# for queue_name, nb_elem_queue in get_all_queues_with_sorted_nb_elem(): -# l_module_pid = get_module_pid_by_queue_name(queue_name) -# for module_pid in l_module_pid: -# last_process_start_time = get_module_last_process_start_time(queue_name, module_pid) -# if last_process_start_time: -# last_process_start_time = datetime.datetime.fromtimestamp(int(last_process_start_time)) -# seconds = int((datetime.datetime.now() - last_process_start_time).total_seconds()) -# else: -# seconds = 0 -# all_modules_queues_stats.append((queue_name, nb_elem_queue, seconds, module_pid)) -# return all_modules_queues_stats -# -# -# def _get_all_messages_from_queue(queue_name): -# #self.r_temp.hset('queues', self.subscriber_name, int(self.r_temp.scard(in_set))) -# return r_queues.smembers(f'queue:{queue_name}:in') -# -# # def is_message_in queue(queue_name): -# # pass -# -# def remove_message_from_queue(queue_name, message): -# queue_key = f'queue:{queue_name}:in' -# r_queues.srem(queue_key, message) -# r_queues.hset('queues', queue_name, int(r_queues.scard(queue_key))) - - if __name__ == '__main__': # clear_modules_queues_stats() - save_queue_digraph() + # save_queue_digraph() + oobj_global_id = 'item::submitted/2023/06/22/submitted_f656119e-f2ea-49d7-9beb-fb97077f8fe5.gz' + while True: + print(get_processed_obj(oobj_global_id)) + time.sleep(0.5) diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index df598a70..01990996 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -46,29 +46,29 @@ def sanitize_objs_types(objs): return l_types -def get_object(obj_type, subtype, id): +def get_object(obj_type, subtype, obj_id): if obj_type == 'item': - return Item(id) + return Item(obj_id) elif obj_type == 'domain': - return Domain(id) + return Domain(obj_id) elif obj_type == 'decoded': - return Decoded(id) + return Decoded(obj_id) elif obj_type == 'cookie-name': - return CookiesNames.CookieName(id) + return CookiesNames.CookieName(obj_id) elif obj_type == 'cve': - return Cve(id) + return Cve(obj_id) elif obj_type == 'favicon': - return Favicon(id) + return Favicon(obj_id) elif obj_type == 'screenshot': - return Screenshot(id) + return Screenshot(obj_id) elif obj_type == 'cryptocurrency': - return CryptoCurrencies.CryptoCurrency(id, subtype) + return CryptoCurrencies.CryptoCurrency(obj_id, subtype) elif obj_type == 'pgp': - return Pgps.Pgp(id, subtype) + return Pgps.Pgp(obj_id, subtype) elif obj_type == 'title': - return Titles.Title(id) + return Titles.Title(obj_id) elif obj_type == 'username': - return Usernames.Username(id, subtype) + return Usernames.Username(obj_id, subtype) def get_objects(objects): objs = set() diff --git a/bin/modules/ApiKey.py b/bin/modules/ApiKey.py index b9324e7b..bf54c095 100755 --- a/bin/modules/ApiKey.py +++ b/bin/modules/ApiKey.py @@ -47,8 +47,8 @@ class ApiKey(AbstractModule): self.logger.info(f"Module {self.module_name} initialized") def compute(self, message, r_result=False): - item_id, score = message.split() - item = Item(item_id) + score = message + item = self.get_obj() item_content = item.get_content() google_api_key = self.regex_findall(self.re_google_api_key, item.get_id(), item_content, r_set=True) @@ -63,8 +63,8 @@ class ApiKey(AbstractModule): print(f'found google api key: {to_print}') self.redis_logger.warning(f'{to_print}Checked {len(google_api_key)} found Google API Key;{item.get_id()}') - msg = f'infoleak:automatic-detection="google-api-key";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = 'infoleak:automatic-detection="google-api-key"' + self.add_message_to_queue(message=tag, queue='Tags') # # TODO: # FIXME: AWS regex/validate/sanitize KEY + SECRET KEY if aws_access_key: @@ -74,12 +74,12 @@ class ApiKey(AbstractModule): print(f'found AWS secret key') self.redis_logger.warning(f'{to_print}Checked {len(aws_secret_key)} found AWS secret Key;{item.get_id()}') - msg = f'infoleak:automatic-detection="aws-key";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = 'infoleak:automatic-detection="aws-key"' + self.add_message_to_queue(message=tag, queue='Tags') # Tags - msg = f'infoleak:automatic-detection="api-key";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = 'infoleak:automatic-detection="api-key"' + self.add_message_to_queue(message=tag, queue='Tags') if r_result: return google_api_key, aws_access_key, aws_secret_key diff --git a/bin/modules/Categ.py b/bin/modules/Categ.py index 60517d64..124f92bc 100755 --- a/bin/modules/Categ.py +++ b/bin/modules/Categ.py @@ -82,7 +82,7 @@ class Categ(AbstractModule): def compute(self, message, r_result=False): # Create Item Object - item = Item(message) + item = self.get_obj() # Get item content content = item.get_content() categ_found = [] @@ -94,11 +94,11 @@ class Categ(AbstractModule): lenfound = len(found) if lenfound >= self.matchingThreshold: categ_found.append(categ) - msg = f'{item.get_id()} {lenfound}' + msg = str(lenfound) # Export message to categ queue print(msg, categ) - self.add_message_to_queue(msg, categ) + self.add_message_to_queue(message=msg, queue=categ) self.redis_logger.debug( f'Categ;{item.get_source()};{item.get_date()};{item.get_basename()};Detected {lenfound} as {categ};{item.get_id()}') diff --git a/bin/modules/Credential.py b/bin/modules/Credential.py index 7fcbafb0..f20f2ba9 100755 --- a/bin/modules/Credential.py +++ b/bin/modules/Credential.py @@ -29,7 +29,6 @@ Redis organization: import os import sys import time -import re from datetime import datetime from pyfaup.faup import Faup @@ -85,8 +84,8 @@ class Credential(AbstractModule): def compute(self, message): - item_id, count = message.split() - item = Item(item_id) + count = message + item = self.get_obj() item_content = item.get_content() @@ -111,8 +110,8 @@ class Credential(AbstractModule): print(f"========> Found more than 10 credentials in this file : {item.get_id()}") self.redis_logger.warning(to_print) - msg = f'infoleak:automatic-detection="credential";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = 'infoleak:automatic-detection="credential"' + self.add_message_to_queue(message=tag, queue='Tags') site_occurrence = self.regex_findall(self.regex_site_for_stats, item.get_id(), item_content) diff --git a/bin/modules/CreditCards.py b/bin/modules/CreditCards.py index e29e42a1..1d8411bb 100755 --- a/bin/modules/CreditCards.py +++ b/bin/modules/CreditCards.py @@ -68,8 +68,8 @@ class CreditCards(AbstractModule): return extracted def compute(self, message, r_result=False): - item_id, score = message.split() - item = Item(item_id) + score = message + item = self.get_obj() content = item.get_content() all_cards = self.regex_findall(self.regex, item.id, content) @@ -90,8 +90,8 @@ class CreditCards(AbstractModule): print(mess) self.redis_logger.warning(mess) - msg = f'infoleak:automatic-detection="credit-card";{item.id}' - self.add_message_to_queue(msg, 'Tags') + tag = 'infoleak:automatic-detection="credit-card"' + self.add_message_to_queue(message=tag, queue='Tags') if r_result: return creditcard_set diff --git a/bin/modules/Cryptocurrencies.py b/bin/modules/Cryptocurrencies.py index 6197f8a1..fd5c5402 100755 --- a/bin/modules/Cryptocurrencies.py +++ b/bin/modules/Cryptocurrencies.py @@ -114,7 +114,7 @@ class Cryptocurrencies(AbstractModule, ABC): self.logger.info(f'Module {self.module_name} initialized') def compute(self, message): - item = Item(message) + item = self.get_obj() item_id = item.get_id() date = item.get_date() content = item.get_content() @@ -134,14 +134,14 @@ class Cryptocurrencies(AbstractModule, ABC): # Check private key if is_valid_address: - msg = f'{currency["tag"]};{item_id}' - self.add_message_to_queue(msg, 'Tags') + msg = f'{currency["tag"]}' + self.add_message_to_queue(message=msg, queue='Tags') if currency.get('private_key'): private_keys = self.regex_findall(currency['private_key']['regex'], item_id, content) if private_keys: - msg = f'{currency["private_key"]["tag"]};{item_id}' - self.add_message_to_queue(msg, 'Tags') + msg = f'{currency["private_key"]["tag"]}' + self.add_message_to_queue(message=msg, queue='Tags') # debug print(private_keys) diff --git a/bin/modules/CveModule.py b/bin/modules/CveModule.py index 8df75c6a..55fa0c91 100755 --- a/bin/modules/CveModule.py +++ b/bin/modules/CveModule.py @@ -44,9 +44,8 @@ class CveModule(AbstractModule): self.logger.info(f'Module {self.module_name} initialized') def compute(self, message): - - item_id, count = message.split() - item = Item(item_id) + count = message + item = self.get_obj() item_id = item.get_id() cves = self.regex_findall(self.reg_cve, item_id, item.get_content()) @@ -61,9 +60,9 @@ class CveModule(AbstractModule): print(warning) self.redis_logger.warning(warning) - msg = f'infoleak:automatic-detection="cve";{item_id}' + tag = 'infoleak:automatic-detection="cve"' # Send to Tags Queue - self.add_message_to_queue(msg, 'Tags') + self.add_message_to_queue(message=tag, queue='Tags') if __name__ == '__main__': diff --git a/bin/modules/Decoder.py b/bin/modules/Decoder.py index c7627480..a8ba07af 100755 --- a/bin/modules/Decoder.py +++ b/bin/modules/Decoder.py @@ -87,8 +87,7 @@ class Decoder(AbstractModule): self.logger.info(f'Module {self.module_name} initialized') def compute(self, message): - - item = Item(message) + item = self.get_obj() content = item.get_content() date = item.get_date() new_decodeds = [] @@ -129,8 +128,8 @@ class Decoder(AbstractModule): self.logger.info(f'{item.id} - {dname}') # Send to Tags - msg = f'infoleak:automatic-detection="{dname}";{item.id}' - self.add_message_to_queue(msg, 'Tags') + tag = f'infoleak:automatic-detection="{dname}"' + self.add_message_to_queue(message=tag, queue='Tags') #################### # TRACKERS DECODED diff --git a/bin/modules/DomClassifier.py b/bin/modules/DomClassifier.py index fd16939d..e7b77d26 100755 --- a/bin/modules/DomClassifier.py +++ b/bin/modules/DomClassifier.py @@ -51,9 +51,9 @@ class DomClassifier(AbstractModule): self.logger.info(f"Module: {self.module_name} Launched") def compute(self, message, r_result=False): - host, item_id = message.split() + host = message - item = Item(item_id) + item = self.get_obj() item_basename = item.get_basename() item_date = item.get_date() item_source = item.get_source() @@ -69,7 +69,7 @@ class DomClassifier(AbstractModule): if self.c.vdomain and d4.is_passive_dns_enabled(): for dns_record in self.c.vdomain: - self.add_message_to_queue(dns_record) + self.add_message_to_queue(obj=None, message=dns_record) localizeddomains = self.c.include(expression=self.cc_tld) if localizeddomains: diff --git a/bin/modules/Duplicates.py b/bin/modules/Duplicates.py index d0a39c0d..57641979 100755 --- a/bin/modules/Duplicates.py +++ b/bin/modules/Duplicates.py @@ -52,7 +52,7 @@ class Duplicates(AbstractModule): def compute(self, message): # IOError: "CRC Checksum Failed on : {id}" - item = Item(message) + item = self.get_obj() # Check file size if item.get_size() < self.min_item_size: diff --git a/bin/modules/Global.py b/bin/modules/Global.py index f24a8603..1c05fcf3 100755 --- a/bin/modules/Global.py +++ b/bin/modules/Global.py @@ -130,14 +130,14 @@ class Global(AbstractModule): update_obj_date(item.get_date(), 'item') - self.add_message_to_queue(item_id, 'Item') + self.add_message_to_queue(obj=item, queue='Item') self.processed_item += 1 # DIRTY FIX AIL SYNC - SEND TO SYNC MODULE # # FIXME: DIRTY FIX message = f'{item.get_type()};{item.get_subtype(r_str=True)};{item.get_id()}' print(message) - self.add_message_to_queue(message, 'Sync') + self.add_message_to_queue(obj=item, queue='Sync') print(item_id) if r_result: diff --git a/bin/modules/Hosts.py b/bin/modules/Hosts.py index 37f278a5..fce5595e 100755 --- a/bin/modules/Hosts.py +++ b/bin/modules/Hosts.py @@ -49,7 +49,7 @@ class Hosts(AbstractModule): self.logger.info(f"Module: {self.module_name} Launched") def compute(self, message): - item = Item(message) + item = self.get_obj() # mimetype = item_basic.get_item_mimetype(item.get_id()) # if mimetype.split('/')[0] == "text": @@ -60,9 +60,7 @@ class Hosts(AbstractModule): print(f'{len(hosts)} host {item.get_id()}') for host in hosts: # print(host) - - msg = f'{host} {item.get_id()}' - self.add_message_to_queue(msg, 'Host') + self.add_message_to_queue(message=str(host), queue='Host') if __name__ == '__main__': diff --git a/bin/modules/IPAddress.py b/bin/modules/IPAddress.py index 1a5d53c7..31a0ff68 100755 --- a/bin/modules/IPAddress.py +++ b/bin/modules/IPAddress.py @@ -66,7 +66,7 @@ class IPAddress(AbstractModule): if not self.ip_networks: return None - item = Item(message) + item = self.get_obj() content = item.get_content() # list of the regex results in the Item @@ -86,8 +86,8 @@ class IPAddress(AbstractModule): self.redis_logger.warning(f'{item.get_id()} contains {item.get_id()} IPs') # Tag message with IP - msg = f'infoleak:automatic-detection="ip";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = 'infoleak:automatic-detection="ip"' + self.add_message_to_queue(message=tag, queue='Tags') if __name__ == "__main__": diff --git a/bin/modules/Iban.py b/bin/modules/Iban.py index 091a85a5..aa1aa5d6 100755 --- a/bin/modules/Iban.py +++ b/bin/modules/Iban.py @@ -73,7 +73,7 @@ class Iban(AbstractModule): return extracted def compute(self, message): - item = Item(message) + item = self.get_obj() item_id = item.get_id() ibans = self.regex_findall(self.iban_regex, item_id, item.get_content()) @@ -97,8 +97,8 @@ class Iban(AbstractModule): to_print = f'Iban;{item.get_source()};{item.get_date()};{item.get_basename()};' self.redis_logger.warning(f'{to_print}Checked found {len(valid_ibans)} IBAN;{item_id}') # Tags - msg = f'infoleak:automatic-detection="iban";{item_id}' - self.add_message_to_queue(msg, 'Tags') + tag = 'infoleak:automatic-detection="iban"' + self.add_message_to_queue(message=tag, queue='Tags') if __name__ == '__main__': diff --git a/bin/modules/Indexer.py b/bin/modules/Indexer.py index 03fededa..48378bdd 100755 --- a/bin/modules/Indexer.py +++ b/bin/modules/Indexer.py @@ -93,12 +93,12 @@ class Indexer(AbstractModule): self.last_refresh = time_now def compute(self, message): - docpath = message.split(" ", -1)[-1] - - item = Item(message) + item = self.get_obj() item_id = item.get_id() item_content = item.get_content() + docpath = item_id + self.logger.debug(f"Indexing - {self.indexname}: {docpath}") print(f"Indexing - {self.indexname}: {docpath}") diff --git a/bin/modules/Keys.py b/bin/modules/Keys.py index f86b6223..e14523cf 100755 --- a/bin/modules/Keys.py +++ b/bin/modules/Keys.py @@ -56,7 +56,7 @@ class Keys(AbstractModule): self.pending_seconds = 1 def compute(self, message): - item = Item(message) + item = self.get_obj() content = item.get_content() # find = False @@ -65,107 +65,107 @@ class Keys(AbstractModule): if KeyEnum.PGP_MESSAGE.value in content: self.redis_logger.warning(f'{item.get_basename()} has a PGP enc message') - msg = f'infoleak:automatic-detection="pgp-message";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = 'infoleak:automatic-detection="pgp-message"' + self.add_message_to_queue(message=tag, queue='Tags') get_pgp_content = True # find = True if KeyEnum.PGP_PUBLIC_KEY_BLOCK.value in content: - msg = f'infoleak:automatic-detection="pgp-public-key-block";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = f'infoleak:automatic-detection="pgp-public-key-block";{item.get_id()}' + self.add_message_to_queue(message=tag, queue='Tags') get_pgp_content = True if KeyEnum.PGP_SIGNATURE.value in content: - msg = f'infoleak:automatic-detection="pgp-signature";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = f'infoleak:automatic-detection="pgp-signature";{item.get_id()}' + self.add_message_to_queue(message=tag, queue='Tags') get_pgp_content = True if KeyEnum.PGP_PRIVATE_KEY_BLOCK.value in content: self.redis_logger.warning(f'{item.get_basename()} has a pgp private key block message') - msg = f'infoleak:automatic-detection="pgp-private-key";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = f'infoleak:automatic-detection="pgp-private-key";{item.get_id()}' + self.add_message_to_queue(message=tag, queue='Tags') get_pgp_content = True if KeyEnum.CERTIFICATE.value in content: self.redis_logger.warning(f'{item.get_basename()} has a certificate message') - msg = f'infoleak:automatic-detection="certificate";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = f'infoleak:automatic-detection="certificate";{item.get_id()}' + self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.RSA_PRIVATE_KEY.value in content: self.redis_logger.warning(f'{item.get_basename()} has a RSA private key message') print('rsa private key message found') - msg = f'infoleak:automatic-detection="rsa-private-key";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = f'infoleak:automatic-detection="rsa-private-key";{item.get_id()}' + self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.PRIVATE_KEY.value in content: self.redis_logger.warning(f'{item.get_basename()} has a private key message') print('private key message found') - msg = f'infoleak:automatic-detection="private-key";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = f'infoleak:automatic-detection="private-key";{item.get_id()}' + self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.ENCRYPTED_PRIVATE_KEY.value in content: self.redis_logger.warning(f'{item.get_basename()} has an encrypted private key message') print('encrypted private key message found') - msg = f'infoleak:automatic-detection="encrypted-private-key";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = f'infoleak:automatic-detection="encrypted-private-key";{item.get_id()}' + self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.OPENSSH_PRIVATE_KEY.value in content: self.redis_logger.warning(f'{item.get_basename()} has an openssh private key message') print('openssh private key message found') - msg = f'infoleak:automatic-detection="private-ssh-key";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = f'infoleak:automatic-detection="private-ssh-key";{item.get_id()}' + self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.SSH2_ENCRYPTED_PRIVATE_KEY.value in content: self.redis_logger.warning(f'{item.get_basename()} has an ssh2 private key message') print('SSH2 private key message found') - msg = f'infoleak:automatic-detection="private-ssh-key";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = f'infoleak:automatic-detection="private-ssh-key";{item.get_id()}' + self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.OPENVPN_STATIC_KEY_V1.value in content: self.redis_logger.warning(f'{item.get_basename()} has an openssh private key message') print('OpenVPN Static key message found') - msg = f'infoleak:automatic-detection="vpn-static-key";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = f'infoleak:automatic-detection="vpn-static-key";{item.get_id()}' + self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.DSA_PRIVATE_KEY.value in content: self.redis_logger.warning(f'{item.get_basename()} has a dsa private key message') - msg = f'infoleak:automatic-detection="dsa-private-key";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = f'infoleak:automatic-detection="dsa-private-key";{item.get_id()}' + self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.EC_PRIVATE_KEY.value in content: self.redis_logger.warning(f'{item.get_basename()} has an ec private key message') - msg = f'infoleak:automatic-detection="ec-private-key";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = f'infoleak:automatic-detection="ec-private-key";{item.get_id()}' + self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.PUBLIC_KEY.value in content: self.redis_logger.warning(f'{item.get_basename()} has a public key message') - msg = f'infoleak:automatic-detection="public-key";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = f'infoleak:automatic-detection="public-key";{item.get_id()}' + self.add_message_to_queue(message=tag, queue='Tags') # find = True # pgp content if get_pgp_content: - self.add_message_to_queue(item.get_id(), 'PgpDump') + self.add_message_to_queue(queue='PgpDump') # if find : # # Send to duplicate diff --git a/bin/modules/Languages.py b/bin/modules/Languages.py index 4924b379..b0b87230 100755 --- a/bin/modules/Languages.py +++ b/bin/modules/Languages.py @@ -25,7 +25,7 @@ class Languages(AbstractModule): self.logger.info(f'Module {self.module_name} initialized') def compute(self, message): - item = Item(message) + item = self.get_obj() if item.is_crawled(): domain = Domain(item.get_domain()) for lang in item.get_languages(min_probability=0.8): diff --git a/bin/modules/LibInjection.py b/bin/modules/LibInjection.py index 9f357441..eb1174f6 100755 --- a/bin/modules/LibInjection.py +++ b/bin/modules/LibInjection.py @@ -40,7 +40,8 @@ class LibInjection(AbstractModule): self.redis_logger.info(f"Module: {self.module_name} Launched") def compute(self, message): - url, item_id = message.split() + item = self.get_obj() + url = message self.faup.decode(url) url_parsed = self.faup.get() @@ -68,7 +69,6 @@ class LibInjection(AbstractModule): # print(f'query is sqli : {result_query}') if result_path['sqli'] is True or result_query['sqli'] is True: - item = Item(item_id) item_id = item.get_id() print(f"Detected (libinjection) SQL in URL: {item_id}") print(unquote(url)) @@ -77,8 +77,8 @@ class LibInjection(AbstractModule): self.redis_logger.warning(to_print) # Add tag - msg = f'infoleak:automatic-detection="sql-injection";{item_id}' - self.add_message_to_queue(msg, 'Tags') + tag = 'infoleak:automatic-detection="sql-injection"' + self.add_message_to_queue(message=tag, queue='Tags') # statistics # # # TODO: # FIXME: remove me diff --git a/bin/modules/MISP_Thehive_Auto_Push.py b/bin/modules/MISP_Thehive_Auto_Push.py index a9b27db9..9747a69b 100755 --- a/bin/modules/MISP_Thehive_Auto_Push.py +++ b/bin/modules/MISP_Thehive_Auto_Push.py @@ -45,8 +45,9 @@ class MISP_Thehive_Auto_Push(AbstractModule): self.last_refresh = time.time() self.redis_logger.info('Tags Auto Push refreshed') - item_id, tag = message.split(';', 1) - item = Item(item_id) + tag = message + item = self.get_obj() + item_id = item.get_id() # enabled if 'misp' in self.tags: diff --git a/bin/modules/Mail.py b/bin/modules/Mail.py index 7eb25084..bbdbcfce 100755 --- a/bin/modules/Mail.py +++ b/bin/modules/Mail.py @@ -135,8 +135,8 @@ class Mail(AbstractModule): # # TODO: sanitize mails def compute(self, message): - item_id, score = message.split() - item = Item(item_id) + score = message + item = self.get_obj() item_date = item.get_date() mails = self.regex_findall(self.email_regex, item_id, item.get_content()) @@ -177,8 +177,8 @@ class Mail(AbstractModule): print(f'{item_id} Checked {num_valid_email} e-mail(s)') self.redis_logger.warning(msg) # Tags - msg = f'infoleak:automatic-detection="mail";{item_id}' - self.add_message_to_queue(msg, 'Tags') + tag = 'infoleak:automatic-detection="mail"' + self.add_message_to_queue(message=tag, queue='Tags') elif num_valid_email > 0: self.redis_logger.info(msg) diff --git a/bin/modules/Mixer.py b/bin/modules/Mixer.py index 49cd3046..b8f2bedf 100755 --- a/bin/modules/Mixer.py +++ b/bin/modules/Mixer.py @@ -173,7 +173,7 @@ class Mixer(AbstractModule): self.r_cache.expire(digest, self.ttl_key) self.increase_stat_processed(feeder_name) - self.add_message_to_queue(relay_message) + self.add_message_to_queue(message=relay_message) # Need To Be Fixed, Currently doesn't check the source (-> same as operation 1) # # Keep duplicate coming from different sources diff --git a/bin/modules/Onion.py b/bin/modules/Onion.py index 5e76d0fa..2066e9a3 100755 --- a/bin/modules/Onion.py +++ b/bin/modules/Onion.py @@ -69,8 +69,8 @@ class Onion(AbstractModule): onion_urls = [] domains = [] - item_id, score = message.split() - item = Item(item_id) + score = message + item = self.get_obj() item_content = item.get_content() # max execution time on regex @@ -100,8 +100,8 @@ class Onion(AbstractModule): self.redis_logger.warning(f'{to_print}Detected {len(domains)} .onion(s);{item.get_id()}') # TAG Item - msg = f'infoleak:automatic-detection="onion";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = 'infoleak:automatic-detection="onion"' + self.add_message_to_queue(message=tag, queue='Tags') if __name__ == "__main__": diff --git a/bin/modules/PgpDump.py b/bin/modules/PgpDump.py index 0647e897..1e8a27a7 100755 --- a/bin/modules/PgpDump.py +++ b/bin/modules/PgpDump.py @@ -171,7 +171,7 @@ class PgpDump(AbstractModule): print('symmetrically encrypted') def compute(self, message): - item = Item(message) + item = self.get_obj() self.item_id = item.get_id() content = item.get_content() @@ -234,8 +234,8 @@ class PgpDump(AbstractModule): print(f' private key: {key}') if self.symmetrically_encrypted: - msg = f'infoleak:automatic-detection="pgp-symmetric";{self.item_id}' - self.add_message_to_queue(msg, 'Tags') + tag = 'infoleak:automatic-detection="pgp-symmetric"' + self.add_message_to_queue(message=tag, queue='Tags') if __name__ == '__main__': diff --git a/bin/modules/Phone.py b/bin/modules/Phone.py index 16af8303..0c01a40f 100755 --- a/bin/modules/Phone.py +++ b/bin/modules/Phone.py @@ -49,7 +49,7 @@ class Phone(AbstractModule): return extracted def compute(self, message): - item = Item(message) + item = self.get_obj() content = item.get_content() # TODO use language detection to choose the country code ? @@ -59,8 +59,8 @@ class Phone(AbstractModule): if results: # TAGS - msg = f'infoleak:automatic-detection="phone-number";{item.get_id()}' - self.add_message_to_queue(msg, 'Tags') + tag = 'infoleak:automatic-detection="phone-number"' + self.add_message_to_queue(message=tag, queue='Tags') self.redis_logger.warning(f'{item.get_id()} contains {len(phone)} Phone numbers') diff --git a/bin/modules/SQLInjectionDetection.py b/bin/modules/SQLInjectionDetection.py index ed6dc89d..dad6a15c 100755 --- a/bin/modules/SQLInjectionDetection.py +++ b/bin/modules/SQLInjectionDetection.py @@ -44,22 +44,21 @@ class SQLInjectionDetection(AbstractModule): self.logger.info(f"Module: {self.module_name} Launched") def compute(self, message): - url, item_id = message.split() + url = message + item = self.get_obj() if self.is_sql_injection(url): self.faup.decode(url) url_parsed = self.faup.get() - item = Item(item_id) - item_id = item.get_id() print(f"Detected SQL in URL: {item_id}") print(urllib.request.unquote(url)) to_print = f'SQLInjection;{item.get_source()};{item.get_date()};{item.get_basename()};Detected SQL in URL;{item_id}' self.redis_logger.warning(to_print) # Tag - msg = f'infoleak:automatic-detection="sql-injection";{item_id}' - self.add_message_to_queue(msg, 'Tags') + tag = f'infoleak:automatic-detection="sql-injection";{item_id}' + self.add_message_to_queue(message=tag, queue='Tags') # statistics # tld = url_parsed['tld'] diff --git a/bin/modules/SubmitPaste.py b/bin/modules/SubmitPaste.py index 782323ef..740090ea 100755 --- a/bin/modules/SubmitPaste.py +++ b/bin/modules/SubmitPaste.py @@ -16,8 +16,6 @@ import gzip import base64 import datetime import time -# from sflock.main import unpack -# import sflock sys.path.append(os.environ['AIL_BIN']) ################################## @@ -48,7 +46,6 @@ class SubmitPaste(AbstractModule): """ super(SubmitPaste, self).__init__() - # TODO KVROCKS self.r_serv_db = ConfigLoader.ConfigLoader().get_db_conn("Kvrocks_DB") self.r_serv_log_submit = ConfigLoader.ConfigLoader().get_redis_conn("Redis_Log_submit") @@ -281,7 +278,7 @@ class SubmitPaste(AbstractModule): # send paste to Global module relay_message = f"submitted {rel_item_path} {gzip64encoded}" - self.add_message_to_queue(relay_message) + self.add_message_to_queue(message=relay_message) # add tags for tag in ltags: diff --git a/bin/modules/Tags.py b/bin/modules/Tags.py index 6c2fb0fb..760cb138 100755 --- a/bin/modules/Tags.py +++ b/bin/modules/Tags.py @@ -20,9 +20,6 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from modules.abstract_module import AbstractModule -from lib.objects.Items import Item -from lib import Tag - class Tags(AbstractModule): """ @@ -39,25 +36,17 @@ class Tags(AbstractModule): self.logger.info(f'Module {self.module_name} initialized') def compute(self, message): - # Extract item ID and tag from message - mess_split = message.split(';') - if len(mess_split) == 2: - tag = mess_split[0] - item = Item(mess_split[1]) + item = self.obj + tag = message - # Create a new tag - Tag.add_object_tag(tag, 'item', item.get_id()) - print(f'{item.get_id()}: Tagged {tag}') + # Create a new tag + item.add_tag(tag) + print(f'{item.get_id()}: Tagged {tag}') - # Forward message to channel - self.add_message_to_queue(message, 'Tag_feed') + # Forward message to channel + self.add_message_to_queue(message=tag, queue='Tag_feed') - message = f'{item.get_type()};{item.get_subtype(r_str=True)};{item.get_id()}' - self.add_message_to_queue(message, 'Sync') - - else: - # Malformed message - raise Exception(f'too many values to unpack (expected 2) given {len(mess_split)} with message {message}') + self.add_message_to_queue(queue='Sync') if __name__ == '__main__': diff --git a/bin/modules/Telegram.py b/bin/modules/Telegram.py index 31d90878..273d20b9 100755 --- a/bin/modules/Telegram.py +++ b/bin/modules/Telegram.py @@ -41,7 +41,7 @@ class Telegram(AbstractModule): self.logger.info(f"Module {self.module_name} initialized") def compute(self, message, r_result=False): - item = Item(message) + item = self.get_obj() item_content = item.get_content() item_date = item.get_date() @@ -86,8 +86,8 @@ class Telegram(AbstractModule): # CREATE TAG if invite_code_found: # tags - msg = f'infoleak:automatic-detection="telegram-invite-hash";{item.id}' - self.add_message_to_queue(msg, 'Tags') + tag = 'infoleak:automatic-detection="telegram-invite-hash"' + self.add_message_to_queue(message=tag, queue='Tags') if __name__ == "__main__": diff --git a/bin/modules/Tools.py b/bin/modules/Tools.py index 06b2d53a..1a9025c8 100755 --- a/bin/modules/Tools.py +++ b/bin/modules/Tools.py @@ -416,7 +416,7 @@ class Tools(AbstractModule): return extracted def compute(self, message): - item = Item(message) + item = self.get_obj() content = item.get_content() for tool_name in TOOLS: @@ -425,8 +425,8 @@ class Tools(AbstractModule): if match: print(f'{item.id} found: {tool_name}') # Tag Item - msg = f"{tool['tag']};{item.id}" - self.add_message_to_queue(msg, 'Tags') + tag = tool['tag'] + self.add_message_to_queue(message=tag, queue='Tags') # TODO ADD LOGS diff --git a/bin/modules/Urls.py b/bin/modules/Urls.py index 2dc97226..c5a3dfe4 100755 --- a/bin/modules/Urls.py +++ b/bin/modules/Urls.py @@ -62,10 +62,9 @@ class Urls(AbstractModule): """ Search for Web links from given message """ - # Extract item - item_id, score = message.split() + score = message - item = Item(item_id) + item = self.get_obj() item_content = item.get_content() # TODO Handle invalid URL @@ -79,10 +78,9 @@ class Urls(AbstractModule): except AttributeError: url = url_decoded['url'] - to_send = f"{url} {item.get_id()}" - print(to_send) - self.add_message_to_queue(to_send, 'Url') - self.logger.debug(f"url_parsed: {to_send}") + print(url, item.get_id()) + self.add_message_to_queue(message=str(url), queue='Url') + self.logger.debug(f"url_parsed: {url}") if len(l_urls) > 0: to_print = f'Urls;{item.get_source()};{item.get_date()};{item.get_basename()};' diff --git a/bin/modules/Zerobins.py b/bin/modules/Zerobins.py index f3fcea5a..f81bf9f8 100755 --- a/bin/modules/Zerobins.py +++ b/bin/modules/Zerobins.py @@ -51,10 +51,11 @@ class Zerobins(AbstractModule): """ Compute a message in queue """ - url, item_id = message.split() + url = message + item = self.get_obj() # Extract zerobins addresses - matching_binz = self.regex_findall(self.regex, item_id, url) + matching_binz = self.regex_findall(self.regex, item.get_id(), url) if len(matching_binz) > 0: for bin_url in matching_binz: diff --git a/bin/modules/abstract_module.py b/bin/modules/abstract_module.py index 0a1a12cd..05e253d5 100644 --- a/bin/modules/abstract_module.py +++ b/bin/modules/abstract_module.py @@ -23,6 +23,7 @@ from lib import ail_logger from lib.ail_queues import AILQueue from lib import regex_helper from lib.exceptions import ModuleQueueError +from lib.objects.ail_objects import get_obj_from_global_id logging.config.dictConfig(ail_logger.get_config(name='modules')) @@ -47,6 +48,8 @@ class AbstractModule(ABC): # Setup the I/O queues if queue: self.queue = AILQueue(self.module_name, self.pid) + self.obj = None + self.sha256_mess = None # Init Redis Logger self.redis_logger = publisher @@ -70,24 +73,45 @@ class AbstractModule(ABC): # Debug Mode self.debug = False + def get_obj(self): + return self.obj + def get_message(self): """ Get message from the Redis Queue (QueueIn) Input message can change between modules ex: '' """ - return self.queue.get_message() + message = self.queue.get_message() + if message: + obj_global_id, sha256_mess, mess = message + if obj_global_id: + self.sha256_mess = sha256_mess + self.obj = get_obj_from_global_id(obj_global_id) + else: + self.sha256_mess = None + self.obj = None + return mess + self.sha256_mess = None + self.obj = None + return None - def add_message_to_queue(self, message, queue_name=None): + def add_message_to_queue(self, message='', obj=None, queue=None): """ Add message to queue + :param obj: AILObject :param message: message to send in queue - :param queue_name: queue or module name + :param queue: queue name or module name ex: add_message_to_queue(item_id, 'Mail') """ - self.queue.send_message(message, queue_name) - # add to new set_module + if obj: + obj_global_id = obj.get_global_id() + elif self.obj: + obj_global_id = self.obj.get_global_id() + else: + obj_global_id = '::' + self.queue.send_message(obj_global_id, message, queue) def get_available_queues(self): return self.queue.get_out_queues() @@ -130,7 +154,7 @@ class AbstractModule(ABC): # Get one message (ex:item id) from the Redis Queue (QueueIn) message = self.get_message() - if message: + if message or self.obj: try: # Module processing with the message from the queue self.compute(message) @@ -152,6 +176,11 @@ class AbstractModule(ABC): # remove from set_module ## check if item process == completed + if self.obj: + self.queue.end_message(self.obj.get_global_id(), self.sha256_mess) + self.obj = None + self.sha256_mess = None + else: self.computeNone() # Wait before next process diff --git a/bin/trackers/Tracker_Regex.py b/bin/trackers/Tracker_Regex.py index 5cc06410..bf756f49 100755 --- a/bin/trackers/Tracker_Regex.py +++ b/bin/trackers/Tracker_Regex.py @@ -47,7 +47,7 @@ class Tracker_Regex(AbstractModule): self.redis_logger.info(f"Module: {self.module_name} Launched") - def compute(self, obj_id, obj_type='item', subtype=''): + def compute(self, message): # refresh Tracked regex if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'): self.tracked_regexs = Tracker.get_tracked_regexs() @@ -55,7 +55,7 @@ class Tracker_Regex(AbstractModule): self.redis_logger.debug('Tracked regex refreshed') print('Tracked regex refreshed') - obj = ail_objects.get_object(obj_type, subtype, obj_id) + obj = self.get_obj() obj_id = obj.get_id() obj_type = obj.get_type() @@ -87,8 +87,7 @@ class Tracker_Regex(AbstractModule): for tag in tracker.get_tags(): if obj.get_type() == 'item': - msg = f'{tag};{obj_id}' - self.add_message_to_queue(msg, 'Tags') + self.add_message_to_queue(message=tag, queue='Tags') else: obj.add_tag(tag) diff --git a/bin/trackers/Tracker_Term.py b/bin/trackers/Tracker_Term.py index 3e0ba43b..a3067674 100755 --- a/bin/trackers/Tracker_Term.py +++ b/bin/trackers/Tracker_Term.py @@ -64,7 +64,7 @@ class Tracker_Term(AbstractModule): self.redis_logger.info(f"Module: {self.module_name} Launched") - def compute(self, obj_id, obj_type='item', subtype=''): + def compute(self, message): # refresh Tracked term if self.last_refresh_word < Tracker.get_tracker_last_updated_by_type('word'): self.tracked_words = Tracker.get_tracked_words() @@ -78,7 +78,7 @@ class Tracker_Term(AbstractModule): self.redis_logger.debug('Tracked set refreshed') print('Tracked set refreshed') - obj = ail_objects.get_object(obj_type, subtype, obj_id) + obj = self.get_obj() obj_type = obj.get_type() # Object Filter @@ -132,8 +132,7 @@ class Tracker_Term(AbstractModule): # Tags for tag in tracker.get_tags(): if obj.get_type() == 'item': - msg = f'{tag};{obj_id}' - self.add_message_to_queue(msg, 'Tags') + self.add_message_to_queue(message=tag, queue='Tags') else: obj.add_tag(tag) diff --git a/bin/trackers/Tracker_Typo_Squatting.py b/bin/trackers/Tracker_Typo_Squatting.py index 99cacee3..9e093b3e 100755 --- a/bin/trackers/Tracker_Typo_Squatting.py +++ b/bin/trackers/Tracker_Typo_Squatting.py @@ -45,7 +45,7 @@ class Tracker_Typo_Squatting(AbstractModule): self.redis_logger.info(f"Module: {self.module_name} Launched") - def compute(self, message, obj_type='item', subtype=''): + def compute(self, message): # refresh Tracked typo if self.last_refresh_typosquatting < Tracker.get_tracker_last_updated_by_type('typosquatting'): self.tracked_typosquattings = Tracker.get_tracked_typosquatting() @@ -53,8 +53,8 @@ class Tracker_Typo_Squatting(AbstractModule): self.redis_logger.debug('Tracked typosquatting refreshed') print('Tracked typosquatting refreshed') - host, obj_id = message.split() - obj = ail_objects.get_object(obj_type, subtype, obj_id) + host = message + obj = self.get_obj() obj_type = obj.get_type() # Object Filter @@ -84,7 +84,7 @@ class Tracker_Typo_Squatting(AbstractModule): for tag in tracker.get_tags(): if obj.get_type() == 'item': msg = f'{tag};{obj_id}' - self.add_message_to_queue(msg, 'Tags') + self.add_message_to_queue(message=tag, queue='Tags') else: obj.add_tag(tag) diff --git a/bin/trackers/Tracker_Yara.py b/bin/trackers/Tracker_Yara.py index fab397d1..7bf13dfd 100755 --- a/bin/trackers/Tracker_Yara.py +++ b/bin/trackers/Tracker_Yara.py @@ -46,7 +46,7 @@ class Tracker_Yara(AbstractModule): self.redis_logger.info(f"Module: {self.module_name} Launched") - def compute(self, obj_id, obj_type='item', subtype=''): + def compute(self, message): # refresh YARA list if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'): self.rules = Tracker.get_tracked_yara_rules() @@ -54,7 +54,7 @@ class Tracker_Yara(AbstractModule): self.redis_logger.debug('Tracked set refreshed') print('Tracked set refreshed') - self.obj = ail_objects.get_object(obj_type, subtype, obj_id) + self.obj = self.get_obj() obj_type = self.obj.get_type() # Object Filter @@ -89,8 +89,7 @@ class Tracker_Yara(AbstractModule): # Tags for tag in tracker.get_tags(): if self.obj.get_type() == 'item': - msg = f'{tag};{obj_id}' - self.add_message_to_queue(msg, 'Tags') + self.add_message_to_queue(message=tag, queue='Tags') else: self.obj.add_tag(tag) From 47e13431873483604642bcfb412f20f53bf120ad Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 22 Jun 2023 16:09:18 +0200 Subject: [PATCH 050/238] fix: [crawler] same capture uuid if a domain is already crawled --- bin/lib/crawlers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 300edb66..6f6bf5b7 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -1239,7 +1239,8 @@ class CrawlerCapture: def create(self, task_uuid): if self.exists(): - raise Exception(f'Error: Capture {self.uuid} already exists') + print(f'Capture {self.uuid} already exists') # TODO LOGS + return None launch_time = int(time.time()) r_crawler.hset(f'crawler:task:{task_uuid}', 'capture', self.uuid) r_crawler.hset('crawler:captures:tasks', self.uuid, task_uuid) From f1f33d6c1b4d1b1a5171b8d0be1efed1a28bde17 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 22 Jun 2023 16:34:14 +0200 Subject: [PATCH 051/238] chg: [pystemon importer] fix gzipped pastes --- bin/importer/PystemonImporter.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bin/importer/PystemonImporter.py b/bin/importer/PystemonImporter.py index 536801ba..d7a260f5 100755 --- a/bin/importer/PystemonImporter.py +++ b/bin/importer/PystemonImporter.py @@ -35,7 +35,7 @@ class PystemonImporter(AbstractImporter): print(item_id) if item_id: print(item_id) - full_item_path = os.path.join(self.dir_pystemon, item_id) # TODO SANITIZE PATH + full_item_path = os.path.join(self.dir_pystemon, item_id) # TODO SANITIZE PATH # Check if pystemon file exists if not os.path.isfile(full_item_path): print(f'Error: {full_item_path}, file not found') @@ -47,7 +47,12 @@ class PystemonImporter(AbstractImporter): if not content: return None - return self.create_message(item_id, content, source='pystemon') + if full_item_path[-3:] == '.gz': + gzipped = True + else: + gzipped = False + + return self.create_message(item_id, content, gzipped=gzipped, source='pystemon') except IOError as e: print(f'Error: {full_item_path}, IOError') From a0686eefcf005a4db17b19f498d0cfa9bcdbc18d Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 26 Jun 2023 11:24:04 +0200 Subject: [PATCH 052/238] fix: [pystemon importer] fix base64 encoding --- bin/importer/abstract_importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/importer/abstract_importer.py b/bin/importer/abstract_importer.py index e5155775..1c4b458d 100755 --- a/bin/importer/abstract_importer.py +++ b/bin/importer/abstract_importer.py @@ -89,7 +89,7 @@ class AbstractImporter(ABC): # TODO ail queues if not gzipped: content = self.b64_gzip(content) elif not b64: - content = self.b64(gzipped) + content = self.b64(content) if not content: return None if isinstance(content, bytes): From b4d536f174b4f8976e056709a220880f050aa4ec Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 26 Jun 2023 16:28:31 +0200 Subject: [PATCH 053/238] chg: [investigation] add object comment --- bin/lib/Investigations.py | 21 +++++++++++++++---- bin/lib/objects/Items.py | 7 +++---- bin/lib/objects/Screenshots.py | 2 +- bin/lib/objects/abstract_daterange_object.py | 7 ++++--- bin/lib/objects/abstract_object.py | 2 +- var/www/blueprints/investigations_b.py | 15 ++++++++++--- .../investigations/view_investigation.html | 18 ++++++++++------ .../modals/investigations_register_obj.html | 16 ++++++++++---- 8 files changed, 62 insertions(+), 26 deletions(-) diff --git a/bin/lib/Investigations.py b/bin/lib/Investigations.py index 1944d00f..9c6def0f 100755 --- a/bin/lib/Investigations.py +++ b/bin/lib/Investigations.py @@ -235,18 +235,27 @@ class Investigation(object): objs.append(dict_obj) return objs + def get_objects_comment(self, obj_global_id): + return r_tracking.hget(f'investigations:objs:comment:{self.uuid}', obj_global_id) + + def set_objects_comment(self, obj_global_id, comment): + if comment: + r_tracking.hset(f'investigations:objs:comment:{self.uuid}', obj_global_id, comment) + # # TODO: def register_object(self, Object): in OBJECT CLASS - def register_object(self, obj_id, obj_type, subtype): + def register_object(self, obj_id, obj_type, subtype, comment=''): r_tracking.sadd(f'investigations:objs:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}') r_tracking.sadd(f'obj:investigations:{obj_type}:{subtype}:{obj_id}', self.uuid) + if comment: + self.set_objects_comment(f'{obj_type}:{subtype}:{obj_id}', comment) timestamp = int(time.time()) self.set_last_change(timestamp) - def unregister_object(self, obj_id, obj_type, subtype): r_tracking.srem(f'investigations:objs:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}') r_tracking.srem(f'obj:investigations:{obj_type}:{subtype}:{obj_id}', self.uuid) + r_tracking.hdel(f'investigations:objs:comment:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}') timestamp = int(time.time()) self.set_last_change(timestamp) @@ -351,7 +360,7 @@ def get_investigations_selector(): for investigation_uuid in get_all_investigations(): investigation = Investigation(investigation_uuid) name = investigation.get_info() - l_investigations.append({"id":investigation_uuid, "name": name}) + l_investigations.append({"id": investigation_uuid, "name": name}) return l_investigations #{id:'8dc4b81aeff94a9799bd70ba556fa345',name:"Paris"} @@ -453,7 +462,11 @@ def api_register_object(json_dict): if subtype == 'None': subtype = '' obj_id = json_dict.get('id', '').replace(' ', '') - res = investigation.register_object(obj_id, obj_type, subtype) + + comment = json_dict.get('comment', '') + # if comment: + # comment = escape(comment) + res = investigation.register_object(obj_id, obj_type, subtype, comment=comment) return res, 200 def api_unregister_object(json_dict): diff --git a/bin/lib/objects/Items.py b/bin/lib/objects/Items.py index 2e35497e..03c6f2cd 100755 --- a/bin/lib/objects/Items.py +++ b/bin/lib/objects/Items.py @@ -264,10 +264,9 @@ class Item(AbstractObject): """ if options is None: options = set() - meta = {'id': self.id, - 'date': self.get_date(separator=True), - 'source': self.get_source(), - 'tags': self.get_tags(r_list=True)} + meta = self.get_default_meta(tags=True) + meta['date'] = self.get_date(separator=True) + meta['source'] = self.get_source() # optional meta fields if 'content' in options: meta['content'] = self.get_content() diff --git a/bin/lib/objects/Screenshots.py b/bin/lib/objects/Screenshots.py index 19ae3754..26f8543f 100755 --- a/bin/lib/objects/Screenshots.py +++ b/bin/lib/objects/Screenshots.py @@ -88,7 +88,7 @@ class Screenshot(AbstractObject): return obj def get_meta(self, options=set()): - meta = {'id': self.id} + meta = self.get_default_meta() meta['img'] = get_screenshot_rel_path(self.id) ######### # TODO: Rename ME ?????? meta['tags'] = self.get_tags(r_list=True) if 'tags_safe' in options: diff --git a/bin/lib/objects/abstract_daterange_object.py b/bin/lib/objects/abstract_daterange_object.py index b96c5ec4..5ec103d0 100755 --- a/bin/lib/objects/abstract_daterange_object.py +++ b/bin/lib/objects/abstract_daterange_object.py @@ -82,9 +82,10 @@ class AbstractDaterangeObject(AbstractObject, ABC): return int(nb) def _get_meta(self, options=[]): - meta_dict = {'first_seen': self.get_first_seen(), - 'last_seen': self.get_last_seen(), - 'nb_seen': self.get_nb_seen()} + meta_dict = self.get_default_meta() + meta_dict['first_seen'] = self.get_first_seen() + meta_dict['last_seen'] = self.get_last_seen() + meta_dict['nb_seen'] = self.get_nb_seen() if 'sparkline' in options: meta_dict['sparkline'] = self.get_sparkline() return meta_dict diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index cb7595ad..2423a294 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -62,7 +62,7 @@ class AbstractObject(ABC): def get_default_meta(self, tags=False): dict_meta = {'id': self.get_id(), 'type': self.get_type(), - 'subtype': self.get_subtype()} + 'subtype': self.get_subtype(r_str=True)} if tags: dict_meta['tags'] = self.get_tags() return dict_meta diff --git a/var/www/blueprints/investigations_b.py b/var/www/blueprints/investigations_b.py index 8c1d592b..cf3cf688 100644 --- a/var/www/blueprints/investigations_b.py +++ b/var/www/blueprints/investigations_b.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* -''' +""" Blueprint Flask: ail_investigations -''' +""" import os import sys @@ -54,7 +54,13 @@ def show_investigation(): investigation_uuid = request.args.get("uuid") investigation = Investigations.Investigation(investigation_uuid) metadata = investigation.get_metadata(r_str=True) - objs = ail_objects.get_objects_meta(investigation.get_objects(), flask_context=True) + objs = [] + for obj in investigation.get_objects(): + obj_meta = ail_objects.get_object_meta(obj["type"], obj["subtype"], obj["id"], flask_context=True) + comment = investigation.get_objects_comment(f'{obj["type"]}:{obj["subtype"]}:{obj["id"]}') + if comment: + obj_meta['comment'] = comment + objs.append(obj_meta) return render_template("view_investigation.html", bootstrap_label=bootstrap_label, metadata=metadata, investigation_objs=objs) @@ -169,10 +175,13 @@ def register_investigation(): object_type = request.args.get('type') object_subtype = request.args.get('subtype') object_id = request.args.get('id') + comment = request.args.get('comment') for investigation_uuid in investigations_uuid: input_dict = {"uuid": investigation_uuid, "id": object_id, "type": object_type, "subtype": object_subtype} + if comment: + input_dict["comment"] = comment res = Investigations.api_register_object(input_dict) if res[1] != 200: return create_json_response(res[0], res[1]) diff --git a/var/www/templates/investigations/view_investigation.html b/var/www/templates/investigations/view_investigation.html index 4cfcd06e..3848b736 100644 --- a/var/www/templates/investigations/view_investigation.html +++ b/var/www/templates/investigations/view_investigation.html @@ -12,8 +12,8 @@ - - + + @@ -125,11 +125,12 @@ - - - + + + - + + @@ -156,6 +157,11 @@ {{ tag }} {% endfor %} + - + @@ -80,6 +80,10 @@ {% endif %} + {% if dict_uuid['description'] %} +
    {{ dict_uuid['description'] }}
    + {% endif %} +
    {% for tag in dict_uuid['tags'] %} @@ -145,6 +149,10 @@ {% endif %} + {% if dict_uuid['description'] %} +
    {{ dict_uuid['description'] }}
    + {% endif %} +
    {% for tag in dict_uuid['tags'] %} From fce63d52a0a20170943d68b4d8de9ec957747c5e Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 29 Jun 2023 09:07:00 +0200 Subject: [PATCH 055/238] fix: [updater] remove old ARDB env --- bin/LAUNCH.sh | 2 +- update/bin/Update_Redis.sh | 4 +--- update/default_update/Update.sh | 6 +++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 547cd76f..1de7f301 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -27,7 +27,7 @@ fi export PATH=$AIL_VENV/bin:$PATH export PATH=$AIL_HOME:$PATH export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH +export PATH=$AIL_KVROCKS:$PATH export PATH=$AIL_BIN:$PATH export PATH=$AIL_FLASK:$PATH diff --git a/update/bin/Update_Redis.sh b/update/bin/Update_Redis.sh index 238d53f7..dc4d394d 100755 --- a/update/bin/Update_Redis.sh +++ b/update/bin/Update_Redis.sh @@ -2,13 +2,11 @@ [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; [ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_BIN. Run the script from the virtual environment." && exit 1; [ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; export PATH=$AIL_HOME:$PATH export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH export PATH=$AIL_BIN:$PATH export PATH=$AIL_FLASK:$PATH diff --git a/update/default_update/Update.sh b/update/default_update/Update.sh index 189ae846..c14c4dac 100755 --- a/update/default_update/Update.sh +++ b/update/default_update/Update.sh @@ -7,13 +7,13 @@ fi [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; [ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_KVROCKS" ] && echo "Needs the env var AIL_KVROCKS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_BIN. Run the script from the virtual environment." && exit 1; [ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; export PATH=$AIL_HOME:$PATH export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH +export PATH=AIL_KVROCKS:$PATH export PATH=$AIL_BIN:$PATH export PATH=$AIL_FLASK:$PATH From 279ec00990ed0cb5104a89f6dbe1516bb831dca9 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 29 Jun 2023 11:40:57 +0200 Subject: [PATCH 056/238] fix: [correlation card decoded meta] mimetype + size --- bin/lib/objects/ail_objects.py | 2 ++ var/www/templates/correlation/metadata_card_decoded.html | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index df598a70..e64d952f 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -177,6 +177,8 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False): if subtype == 'bitcoin' and related_btc: meta["related_btc"] = btc_ail.get_bitcoin_info(obj.id) if obj.get_type() == 'decoded': + meta['mimetype'] = obj.get_mimetype() + meta['size'] = obj.get_size() meta["vt"] = obj.get_meta_vt() meta["vt"]["status"] = obj.is_vt_enabled() # TAGS MODAL diff --git a/var/www/templates/correlation/metadata_card_decoded.html b/var/www/templates/correlation/metadata_card_decoded.html index da57cb21..47a6627d 100644 --- a/var/www/templates/correlation/metadata_card_decoded.html +++ b/var/www/templates/correlation/metadata_card_decoded.html @@ -35,7 +35,7 @@ {{ dict_object["metadata_card"]["icon"]["icon"] }} - {{ dict_object["metadata"]["file_type"] }} + {{ dict_object["metadata"]["mimetype"] }}
    From f545daaa4360af236b2b3312bc387b46852a8a0e Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 29 Jun 2023 11:47:13 +0200 Subject: [PATCH 057/238] fix: [correlation card decoded meta] mimetype + size --- .../templates/correlation/metadata_card_decoded.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/var/www/templates/correlation/metadata_card_decoded.html b/var/www/templates/correlation/metadata_card_decoded.html index 47a6627d..1e292a80 100644 --- a/var/www/templates/correlation/metadata_card_decoded.html +++ b/var/www/templates/correlation/metadata_card_decoded.html @@ -35,12 +35,12 @@ {{ dict_object["metadata_card"]["icon"]["icon"] }} - {{ dict_object["metadata"]["mimetype"] }} + {{ dict_object["metadata_card"]["mimetype"] }} - - - - + + + +
    TypeIdTypeId TagsComment
    + {% if 'comment' in object %} + {{ object['comment']}} + {% endif %} + diff --git a/var/www/templates/modals/investigations_register_obj.html b/var/www/templates/modals/investigations_register_obj.html index edac2853..50a6ca02 100644 --- a/var/www/templates/modals/investigations_register_obj.html +++ b/var/www/templates/modals/investigations_register_obj.html @@ -14,7 +14,10 @@
    - +
    + + +
    @@ -55,8 +58,13 @@ $('#investigations_register_obj_modal').on('shown.bs.modal', function () { }); function Register_Obj() { - var uuids = linvestigations.getValue(); - // TODO: REQUEST - window.location.replace("{{ url_for('investigations_b.register_investigation') }}?uuids=" + uuids + "&type={{ obj_type }}&subtype={{ obj_subtype }}&id={{ obj_id }}"); + var uuids = linvestigations.getValue(); + var comment = $('#inv_obj_comment').val(); + // TODO: REQUEST + var url = "{{ url_for('investigations_b.register_investigation') }}?uuids=" + uuids + "&type={{ obj_type }}&subtype={{ obj_subtype }}&id={{ obj_id }}" + if (comment) { + url += "&comment=" + comment; + } + window.location.replace(url); } From d6a24c035778333f6d31bb42d125716d30d00f27 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 29 Jun 2023 08:48:50 +0200 Subject: [PATCH 054/238] fix: [hunter + misp export] fix misp event json export + retro hunt date search and description --- bin/lib/Tracker.py | 4 ++-- var/www/blueprints/hunters.py | 2 +- var/www/blueprints/import_export.py | 1 + var/www/templates/hunter/trackersManagement.html | 10 +++++++++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 2a5336ad..b7ca5e77 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -650,14 +650,14 @@ def get_user_trackers_meta(user_id, tracker_type=None): metas = [] for tracker_uuid in get_user_trackers(user_id, tracker_type=tracker_type): tracker = Tracker(tracker_uuid) - metas.append(tracker.get_meta(options={'mails', 'sparkline', 'tags'})) + metas.append(tracker.get_meta(options={'description', 'mails', 'sparkline', 'tags'})) return metas def get_global_trackers_meta(tracker_type=None): metas = [] for tracker_uuid in get_global_trackers(tracker_type=tracker_type): tracker = Tracker(tracker_uuid) - metas.append(tracker.get_meta(options={'mails', 'sparkline', 'tags'})) + metas.append(tracker.get_meta(options={'description', 'mails', 'sparkline', 'tags'})) return metas def get_users_trackers_meta(): diff --git a/var/www/blueprints/hunters.py b/var/www/blueprints/hunters.py index b2a2e30b..9a2b6c3e 100644 --- a/var/www/blueprints/hunters.py +++ b/var/www/blueprints/hunters.py @@ -172,7 +172,7 @@ def show_tracker(): typo_squatting = set() if date_from: - date_from, date_to = Date.sanitise_daterange(meta['first_seen'], meta['last_seen']) + date_from, date_to = Date.sanitise_daterange(date_from, date_to) objs = tracker.get_objs_by_daterange(date_from, date_to) meta['objs'] = ail_objects.get_objects_meta(objs, flask_context=True) else: diff --git a/var/www/blueprints/import_export.py b/var/www/blueprints/import_export.py index 312fe0be..bb28d080 100644 --- a/var/www/blueprints/import_export.py +++ b/var/www/blueprints/import_export.py @@ -163,6 +163,7 @@ def objects_misp_export_post(): MISPExporter.delete_user_misp_objects_to_export(user_id) if not export: event_uuid = event[10:46] + event = f'{{"Event": {event}}}' # TODO ADD JAVASCRIPT REFRESH PAGE IF RESP == 200 return send_file(io.BytesIO(event.encode()), as_attachment=True, download_name=f'ail_export_{event_uuid}.json') diff --git a/var/www/templates/hunter/trackersManagement.html b/var/www/templates/hunter/trackersManagement.html index 6bd1cfe4..89407245 100644 --- a/var/www/templates/hunter/trackersManagement.html +++ b/var/www/templates/hunter/trackersManagement.html @@ -60,7 +60,7 @@     		Tracker First seen Last seenEmail notificationEmails sparkline
    {{ dict_object["metadata"]['first_seen'] }} {{ dict_object["metadata"]['last_seen'] }}{{ dict_object["metadata"]['first_seen'] }}{{ dict_object["metadata"]['last_seen'] }}{{ dict_object["metadata"]['size'] }}{{ dict_object["metadata"]['nb_seen'] }}{{ dict_object["metadata_card"]['first_seen'] }}{{ dict_object["metadata_card"]['last_seen'] }}{{ dict_object["metadata_card"]['size'] }}{{ dict_object["metadata_card"]['nb_seen'] }}
    From fa8b4a67d963a1abe8719973f29c3b4d496b9b56 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 29 Jun 2023 15:02:57 +0200 Subject: [PATCH 058/238] fix: [correlation tags] fix tag all objects --- bin/lib/objects/ail_objects.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index e64d952f..c54b7dd4 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -336,7 +336,7 @@ def get_obj_correlations(obj_type, subtype, obj_id): return obj.get_correlations() def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max): - if len(objs) < nb_max or nb_max == -1: + if len(objs) < nb_max or nb_max == 0: if lvl == 0: objs.add((obj_type, subtype, obj_id)) @@ -356,6 +356,7 @@ def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, return objs def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300): + print(nb_max) objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max) # print(objs) for obj_tuple in objs: From 0fa0984ec4ffa3ed19383aef1dd071a8a1e5313b Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 30 Jun 2023 16:19:38 +0200 Subject: [PATCH 059/238] fix: [updater] fix db checker --- bin/LAUNCH.sh | 3 --- update/default_update/Update.sh | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 1de7f301..00c224e4 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -685,9 +685,6 @@ while [ "$1" != "" ]; do -lrv | --launchRedisVerify ) launch_redis; wait_until_redis_is_ready; ;; - -lav | --launchARDBVerify ) launch_ardb; - wait_until_ardb_is_ready; - ;; -lkv | --launchKVORCKSVerify ) launch_kvrocks; wait_until_kvrocks_is_ready; ;; diff --git a/update/default_update/Update.sh b/update/default_update/Update.sh index c14c4dac..ef881805 100755 --- a/update/default_update/Update.sh +++ b/update/default_update/Update.sh @@ -25,7 +25,7 @@ bash ${AIL_BIN}/LAUNCH.sh -ks wait echo "" -bash ${AIL_BIN}/LAUNCH.sh -lav +bash ${AIL_BIN}/LAUNCH.sh -lkv wait echo "" From 35f678245fc5cd53a26d7a2d8d7e23a5780bada1 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 30 Jun 2023 16:22:30 +0200 Subject: [PATCH 060/238] fix: [decoded] fix download file --- bin/lib/objects/Decodeds.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/lib/objects/Decodeds.py b/bin/lib/objects/Decodeds.py index 001f7dfd..fb194be1 100755 --- a/bin/lib/objects/Decodeds.py +++ b/bin/lib/objects/Decodeds.py @@ -138,7 +138,7 @@ class Decoded(AbstractDaterangeObject): with open(filepath, 'rb') as f: content = f.read() return content - elif r_str == 'bytesio': + elif r_type == 'bytesio': with open(filepath, 'rb') as f: content = BytesIO(f.read()) return content @@ -149,7 +149,7 @@ class Decoded(AbstractDaterangeObject): with zipfile.ZipFile(zip_content, "w") as zf: # TODO: Fix password # zf.setpassword(b"infected") - zf.writestr(self.id, self.get_content().getvalue()) + zf.writestr(self.id, self.get_content(r_type='bytesio').getvalue()) zip_content.seek(0) return zip_content From 450ebdd7899c3a1bef6059469f5c9eeafc2f743e Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 6 Jul 2023 11:26:32 +0200 Subject: [PATCH 061/238] chg: [etag] add new etag object --- bin/crawlers/Crawler.py | 5 + bin/lib/ail_core.py | 2 +- bin/lib/correlations_engine.py | 3 +- bin/lib/crawlers.py | 46 +- bin/lib/objects/Etags.py | 121 ++++ bin/lib/objects/ail_objects.py | 5 +- var/www/Flask_server.py | 2 + var/www/blueprints/correlation.py | 3 + var/www/blueprints/objects_cookie_name.py | 2 - var/www/blueprints/objects_etag.py | 86 +++ .../correlation/metadata_card_etag.html | 173 +++++ .../correlation/show_correlation.html | 6 + .../templates/objects/etag/EtagDaterange.html | 602 ++++++++++++++++++ .../templates/sidebars/sidebar_objects.html | 6 + 14 files changed, 1046 insertions(+), 16 deletions(-) create mode 100755 bin/lib/objects/Etags.py create mode 100644 var/www/blueprints/objects_etag.py create mode 100644 var/www/templates/correlation/metadata_card_etag.html create mode 100644 var/www/templates/objects/etag/EtagDaterange.html diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index be615993..2b2f35c6 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -17,6 +17,7 @@ from lib import ail_logger from lib import crawlers from lib.ConfigLoader import ConfigLoader from lib.objects import CookiesNames +from lib.objects import Etags from lib.objects.Domains import Domain from lib.objects.Items import Item from lib.objects import Screenshots @@ -288,6 +289,10 @@ class Crawler(AbstractModule): print(cookie_name) cookie = CookiesNames.create(cookie_name) cookie.add(self.date.replace('/', ''), self.domain.id) + for etag_content in crawlers.extract_etag_from_har(entries['har']): + print(etag_content) + etag = Etags.create(etag_content) + etag.add(self.date.replace('/', ''), self.domain.id) # Next Children entries_children = entries.get('children') diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index c52db274..e0fd3a17 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -15,7 +15,7 @@ config_loader = ConfigLoader() r_serv_db = config_loader.get_db_conn("Kvrocks_DB") config_loader = None -AIL_OBJECTS = sorted({'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'favicon', 'item', 'pgp', +AIL_OBJECTS = sorted({'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'item', 'pgp', 'screenshot', 'title', 'username'}) def get_ail_uuid(): diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index 8e29837d..1a2081ac 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -45,7 +45,8 @@ CORRELATION_TYPES_BY_OBJ = { "cryptocurrency": ["domain", "item"], "cve": ["domain", "item"], "decoded": ["domain", "item"], - "domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "favicon", "item", "pgp", "title", "screenshot", "username"], + "domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "item", "pgp", "title", "screenshot", "username"], + "etag": ["domain"], "favicon": ["domain", "item"], # TODO Decoded "item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], "pgp": ["domain", "item"], diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 6f6bf5b7..7c98537e 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -264,6 +264,7 @@ def extract_author_from_html(html): if keywords: return keywords['content'] return '' + # # # - - # # # @@ -299,9 +300,10 @@ def get_all_har_ids(): except (TypeError, ValueError): pass - for file in [f for f in os.listdir(today_root_dir) if os.path.isfile(os.path.join(today_root_dir, f))]: - har_id = os.path.relpath(os.path.join(today_root_dir, file), HAR_DIR) - har_ids.append(har_id) + if os.path.exists(today_root_dir): + for file in [f for f in os.listdir(today_root_dir) if os.path.isfile(os.path.join(today_root_dir, f))]: + har_id = os.path.relpath(os.path.join(today_root_dir, file), HAR_DIR) + har_ids.append(har_id) for ydir in sorted(dirs_year, reverse=False): search_dear = os.path.join(HAR_DIR, ydir) @@ -312,14 +314,13 @@ def get_all_har_ids(): har_ids.append(har_id) return har_ids -def extract_cookies_names_from_har_by_har_id(har_id): +def get_har_content(har_id): har_path = os.path.join(HAR_DIR, har_id) with open(har_path) as f: try: - har_content = json.loads(f.read()) + return json.loads(f.read()) except json.decoder.JSONDecodeError: - har_content = {} - return extract_cookies_names_from_har(har_content) + return {} def extract_cookies_names_from_har(har): cookies = set() @@ -334,18 +335,41 @@ def extract_cookies_names_from_har(har): cookies.add(name) return cookies -def _reprocess_all_hars(): +def _reprocess_all_hars_cookie_name(): from lib.objects import CookiesNames for har_id in get_all_har_ids(): domain = har_id.split('/')[-1] domain = domain[:-41] date = har_id.split('/') date = f'{date[-4]}{date[-3]}{date[-2]}' - for cookie_name in extract_cookies_names_from_har_by_har_id(har_id): + for cookie_name in extract_cookies_names_from_har(get_har_content(har_id)): print(domain, date, cookie_name) cookie = CookiesNames.create(cookie_name) cookie.add(date, domain) +def extract_etag_from_har(har): # TODO check response url + etags = set() + for entrie in har.get('log', {}).get('entries', []): + for header in entrie.get('response', {}).get('headers', []): + if header.get('name') == 'etag': + # print(header) + etag = header.get('value') + if etag: + etags.add(etag) + return etags + +def _reprocess_all_hars_etag(): + from lib.objects import Etags + for har_id in get_all_har_ids(): + domain = har_id.split('/')[-1] + domain = domain[:-41] + date = har_id.split('/') + date = f'{date[-4]}{date[-3]}{date[-2]}' + for etag_content in extract_etag_from_har(get_har_content(har_id)): + print(domain, date, etag_content) + etag = Etags.create(etag_content) + etag.add(date, domain) + # # # - - # # # ################################################################################ @@ -1913,5 +1937,5 @@ load_blacklist() # temp_url = '' # r = extract_favicon_from_html(content, temp_url) # print(r) -# _reprocess_all_hars() - +# _reprocess_all_hars_cookie_name() +# _reprocess_all_hars_etag() diff --git a/bin/lib/objects/Etags.py b/bin/lib/objects/Etags.py new file mode 100755 index 00000000..eb41f68c --- /dev/null +++ b/bin/lib/objects/Etags.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys + +from hashlib import sha256 +from flask import url_for + +from pymisp import MISPObject + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader +from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects + +config_loader = ConfigLoader() +r_objects = config_loader.get_db_conn("Kvrocks_Objects") +baseurl = config_loader.get_config_str("Notifications", "ail_domain") +config_loader = None + +# TODO NEW ABSTRACT OBJECT -> daterange for all objects ???? + +class Etag(AbstractDaterangeObject): + """ + AIL Etag Object. + """ + + def __init__(self, obj_id): + super(Etag, self).__init__('etag', obj_id) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True), + # 'compress': 'gzip'} + # return payload + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + # # TODO: + pass + + def get_content(self, r_type='str'): + if r_type == 'str': + return self._get_field('content') + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', type=self.type, id=self.id) + else: + url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}' + return url + + # TODO # CHANGE COLOR + def get_svg_icon(self): + return {'style': 'fas', 'icon': '\uf02b', 'color': '#556F65', 'radius': 5} + + def get_misp_object(self): + obj_attrs = [] + obj = MISPObject('etag') + first_seen = self.get_first_seen() + last_seen = self.get_last_seen() + if first_seen: + obj.first_seen = first_seen + if last_seen: + obj.last_seen = last_seen + if not first_seen or not last_seen: + self.logger.warning( + f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}') + + obj_attrs.append(obj.add_attribute('etag', value=self.get_content())) + for obj_attr in obj_attrs: + for tag in self.get_tags(): + obj_attr.add_tag(tag) + return obj + + def get_nb_seen(self): + return self.get_nb_correlation('domain') + + def get_meta(self, options=set()): + meta = self._get_meta(options=options) + meta['id'] = self.id + meta['tags'] = self.get_tags(r_list=True) + meta['content'] = self.get_content() + return meta + + def add(self, date, obj_id): # date = HAR Date + self._add(date, 'domain', '', obj_id) + + def create(self, content, _first_seen=None, _last_seen=None): + if not isinstance(content, str): + content = content.decode() + self._set_field('content', content) + self._create() + + +def create(content): + if isinstance(content, str): + content = content.encode() + obj_id = sha256(content).hexdigest() + etag = Etag(obj_id) + if not etag.exists(): + etag.create(content) + return etag + + +class Etags(AbstractDaterangeObjects): + """ + Etags Objects + """ + def __init__(self): + super().__init__('etag', Etag) + + def sanitize_id_to_search(self, name_to_search): + return name_to_search # TODO + + +# if __name__ == '__main__': +# name_to_search = '98' +# print(search_cves_by_name(name_to_search)) diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index c54b7dd4..4279c776 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -18,6 +18,7 @@ from lib.objects import CookiesNames from lib.objects.Cves import Cve from lib.objects.Decodeds import Decoded, get_all_decodeds_objects, get_nb_decodeds_objects from lib.objects.Domains import Domain +from lib.objects import Etags from lib.objects.Favicons import Favicon from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects from lib.objects import Pgps @@ -57,6 +58,8 @@ def get_object(obj_type, subtype, id): return CookiesNames.CookieName(id) elif obj_type == 'cve': return Cve(id) + elif obj_type == 'etag': + return Etags.Etag(id) elif obj_type == 'favicon': return Favicon(id) elif obj_type == 'screenshot': @@ -168,7 +171,7 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False): obj = get_object(obj_type, subtype, id) meta = obj.get_meta() meta['icon'] = obj.get_svg_icon() - if subtype or obj_type == 'cookie-name' or obj_type == 'cve' or obj_type == 'title' or obj_type == 'favicon': + if subtype or obj_type == 'cookie-name' or obj_type == 'cve' or obj_type == 'etag' or obj_type == 'title' or obj_type == 'favicon': meta['sparkline'] = obj.get_sparkline() if obj_type == 'cve': meta['cve_search'] = obj.get_cve_search() diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index cc110c35..b40b5cc8 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -51,6 +51,7 @@ from blueprints.objects_decoded import objects_decoded from blueprints.objects_subtypes import objects_subtypes from blueprints.objects_title import objects_title from blueprints.objects_cookie_name import objects_cookie_name +from blueprints.objects_etag import objects_etag Flask_dir = os.environ['AIL_FLASK'] @@ -106,6 +107,7 @@ app.register_blueprint(objects_decoded, url_prefix=baseUrl) app.register_blueprint(objects_subtypes, url_prefix=baseUrl) app.register_blueprint(objects_title, url_prefix=baseUrl) app.register_blueprint(objects_cookie_name, url_prefix=baseUrl) +app.register_blueprint(objects_etag, url_prefix=baseUrl) # ========= =========# diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index f6e7feda..d5bb1c82 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -83,6 +83,9 @@ def show_correlation(): correl_option = request.form.get('CookieNameCheck') if correl_option: filter_types.append('cookie-name') + correl_option = request.form.get('EtagCheck') + if correl_option: + filter_types.append('etag') correl_option = request.form.get('CveCheck') if correl_option: filter_types.append('cve') diff --git a/var/www/blueprints/objects_cookie_name.py b/var/www/blueprints/objects_cookie_name.py index ab111ff2..06d6743a 100644 --- a/var/www/blueprints/objects_cookie_name.py +++ b/var/www/blueprints/objects_cookie_name.py @@ -45,8 +45,6 @@ def objects_cookies_names(): else: dict_objects = {} - print(dict_objects) - return render_template("CookieNameDaterange.html", date_from=date_from, date_to=date_to, dict_objects=dict_objects, show_objects=show_objects) diff --git a/var/www/blueprints/objects_etag.py b/var/www/blueprints/objects_etag.py new file mode 100644 index 00000000..ad2b24fd --- /dev/null +++ b/var/www/blueprints/objects_etag.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... +''' + +import os +import sys + +from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort +from flask_login import login_required, current_user + +# Import Role_Manager +from Role_Manager import login_admin, login_analyst, login_read_only + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.objects import Etags +from packages import Date + +# ============ BLUEPRINT ============ +objects_etag = Blueprint('objects_etag', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/etag')) + +# ============ VARIABLES ============ +bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] + + +# ============ FUNCTIONS ============ +@objects_etag.route("/objects/etags", methods=['GET']) +@login_required +@login_read_only +def objects_etags(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + show_objects = request.args.get('show_objects') + date = Date.sanitise_date_range(date_from, date_to) + date_from = date['date_from'] + date_to = date['date_to'] + + if show_objects: + dict_objects = Etags.Etags().api_get_meta_by_daterange(date_from, date_to) + else: + dict_objects = {} + + return render_template("EtagDaterange.html", date_from=date_from, date_to=date_to, + dict_objects=dict_objects, show_objects=show_objects) + +@objects_etag.route("/objects/etag/post", methods=['POST']) +@login_required +@login_read_only +def objects_etags_post(): + date_from = request.form.get('date_from') + date_to = request.form.get('date_to') + show_objects = request.form.get('show_objects') + return redirect(url_for('objects_etag.objects_etags', date_from=date_from, date_to=date_to, show_objects=show_objects)) + +@objects_etag.route("/objects/etag/range/json", methods=['GET']) +@login_required +@login_read_only +def objects_etag_range_json(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + date = Date.sanitise_date_range(date_from, date_to) + date_from = date['date_from'] + date_to = date['date_to'] + return jsonify(Etags.Etags().api_get_chart_nb_by_daterange(date_from, date_to)) + +# @objects_etag.route("/objects/etag/search", methods=['POST']) +# @login_required +# @login_read_only +# def objects_etags_names_search(): +# to_search = request.form.get('object_id') +# +# # TODO SANITIZE ID +# # TODO Search all +# cve = Cves.Cve(to_search) +# if not cve.exists(): +# abort(404) +# else: +# return redirect(cve.get_link(flask_context=True)) + +# ============= ROUTES ============== + diff --git a/var/www/templates/correlation/metadata_card_etag.html b/var/www/templates/correlation/metadata_card_etag.html new file mode 100644 index 00000000..cc599227 --- /dev/null +++ b/var/www/templates/correlation/metadata_card_etag.html @@ -0,0 +1,173 @@ + + + +{% with modal_add_tags=dict_object['metadata_card']['add_tags_modal']%} + {% include 'modals/add_tags.html' %} +{% endwith %} + +{% include 'modals/edit_tag.html' %} + +
    +
    +

    {{ dict_object["metadata"]["content"] }}

    +
    {{ dict_object["correlation_id"] }}
    +
      +
    • +
      +
      + + + + + + + + + + + + + + + + + +
      Object typeFirst seenLast seenNb seen
      + + + + {{ dict_object["metadata_card"]["icon"]["icon"] }} + + + {{ dict_object["object_type"] }} + {{ dict_object["metadata"]['first_seen'] }}{{ dict_object["metadata"]['last_seen'] }}{{ dict_object["metadata"]['nb_seen'] }}
      +
      +
      +
      +
      +
      +
      • + +
    • +
      +
      + Tags: + {% for tag in dict_object["metadata"]['tags'] %} + + {% endfor %} + +
      +
      • +
    + + {% with obj_type='etag', obj_id=dict_object['correlation_id'], obj_subtype='' %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + +
    +
    + + + + + + diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 95aa922c..326b637f 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -117,6 +117,8 @@ {% include 'correlation/metadata_card_title.html' %} {% elif dict_object["object_type"] == "cookie-name" %} {% include 'correlation/metadata_card_cookie_name.html' %} + {% elif dict_object["object_type"] == "etag" %} + {% include 'correlation/metadata_card_etag.html' %} {% elif dict_object["object_type"] == "item" %} {% include 'correlation/metadata_card_item.html' %} {% endif %} @@ -211,6 +213,10 @@
    +
    + + +
    diff --git a/var/www/templates/objects/etag/EtagDaterange.html b/var/www/templates/objects/etag/EtagDaterange.html new file mode 100644 index 00000000..9fc7f3e7 --- /dev/null +++ b/var/www/templates/objects/etag/EtagDaterange.html @@ -0,0 +1,602 @@ + + + + + Etags - AIL + + + + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + +
    +
    +
    + +{# {% include 'etag/block_etag_search.html' %}#} + +
    + + +
    + +
    +
    +
    Select a date range :
    +
    +
    +
    + +
    +
    +
    + +
    +
    + + +
    + +
    +
    +
    + +
    +
    +
    +
    +
    +
    + + {% if dict_objects %} + {% if date_from|string == date_to|string %} +

    {{ date_from }} Etags Name:

    + {% else %} +

    {{ date_from }} to {{ date_to }} Etags Name:

    + {% endif %} + + + + + + + + + + + + {% for obj_id in dict_objects %} + + + + + + + + {% endfor %} + +
    First SeenLast SeenTotalLast days
    {{ dict_objects[obj_id]['content'] }}{{ dict_objects[obj_id]['first_seen'] }}{{ dict_objects[obj_id]['last_seen'] }}{{ dict_objects[obj_id]['nb_seen'] }}
    + + + {% else %} + {% if show_objects %} + {% if date_from|string == date_to|string %} +

    {{ date_from }}, No Etag Name

    + {% else %} +

    {{ date_from }} to {{ date_to }}, No Etag Name

    + {% endif %} + {% endif %} + {% endif %} +
    + +
    +
    + + + + + + + + + + + + + + + + + diff --git a/var/www/templates/sidebars/sidebar_objects.html b/var/www/templates/sidebars/sidebar_objects.html index 89239701..5f7b00b5 100644 --- a/var/www/templates/sidebars/sidebar_objects.html +++ b/var/www/templates/sidebars/sidebar_objects.html @@ -40,6 +40,12 @@ Cookie Name
    • +
  • + + + Etag + +
  • From 482fc21b5e93f37982c2c580fb4e710eeac4d9ff Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 7 Jul 2023 16:29:32 +0200 Subject: [PATCH 062/238] chg: [correlation] correlation graph, add an option to hide an object/node by pressing H + reset correlation graph --- bin/lib/correlations_engine.py | 12 ++++-- bin/lib/objects/ail_objects.py | 17 +++++--- var/www/blueprints/correlation.py | 35 +++++++++++++-- .../correlation/show_correlation.html | 43 ++++++++++++++++++- 4 files changed, 91 insertions(+), 16 deletions(-) diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index 1a2081ac..39ccaa4e 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -170,18 +170,18 @@ def get_obj_str_id(obj_type, subtype, obj_id): subtype = '' return f'{obj_type}:{subtype}:{obj_id}' -def get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, flask_context=False): +def get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set(), flask_context=False): links = set() nodes = set() meta = {'complete': True, 'objs': set()} obj_str_id = get_obj_str_id(obj_type, subtype, obj_id) - _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=filter_types, previous_str_obj='') + _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, previous_str_obj='') return obj_str_id, nodes, links, meta -def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=[], previous_str_obj=''): +def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=[], objs_hidden=set(), previous_str_obj=''): obj_str_id = get_obj_str_id(obj_type, subtype, obj_id) meta['objs'].add(obj_str_id) nodes.add(obj_str_id) @@ -192,6 +192,10 @@ def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, for str_obj in obj_correlations[correl_type]: subtype2, obj2_id = str_obj.split(':', 1) obj2_str_id = get_obj_str_id(correl_type, subtype2, obj2_id) + # filter objects to hide + if obj2_str_id in objs_hidden: + continue + meta['objs'].add(obj2_str_id) if obj2_str_id == previous_str_obj: @@ -205,5 +209,5 @@ def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, if level > 0: next_level = level - 1 - _get_correlations_graph_node(links, nodes, meta, correl_type, subtype2, obj2_id, next_level, max_nodes, filter_types=filter_types, previous_str_obj=obj_str_id) + _get_correlations_graph_node(links, nodes, meta, correl_type, subtype2, obj2_id, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, previous_str_obj=obj_str_id) diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 4279c776..e0eeda63 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -338,7 +338,7 @@ def get_obj_correlations(obj_type, subtype, obj_id): obj = get_object(obj_type, subtype, obj_id) return obj.get_correlations() -def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max): +def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max, objs_hidden): if len(objs) < nb_max or nb_max == 0: if lvl == 0: objs.add((obj_type, subtype, obj_id)) @@ -351,16 +351,17 @@ def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lv for obj2_type in correlations: for str_obj in correlations[obj2_type]: obj2_subtype, obj2_id = str_obj.split(':', 1) - _get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, filter_types, lvl, nb_max) + if get_obj_global_id(obj2_type, obj2_subtype, obj2_id) in objs_hidden: + continue # filter object to hide + _get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, filter_types, lvl, nb_max, objs_hidden) -def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, nb_max=300): +def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, nb_max=300, objs_hidden=set()): objs = set() - _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max) + _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max, objs_hidden) return objs -def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300): - print(nb_max) - objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max) +def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300, objs_hidden=set()): + objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max, objs_hidden=objs_hidden) # print(objs) for obj_tuple in objs: obj1_type, subtype1, id1 = obj_tuple @@ -422,10 +423,12 @@ def create_correlation_graph_nodes(nodes_set, obj_str_id, flask_context=True): def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, + objs_hidden=set(), flask_context=False): obj_str_id, nodes, links, meta = correlations_engine.get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, + objs_hidden=objs_hidden, flask_context=flask_context) # print(meta) meta['objs'] = list(meta['objs']) diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index d5bb1c82..594dc660 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -61,6 +61,13 @@ def sanitise_level(level): level = 2 return level +def sanitise_objs_hidden(objs_hidden): + if objs_hidden: + objs_hidden = set(objs_hidden.split(',')) # TODO sanitize objects + else: + objs_hidden = set() + return objs_hidden + # ============= ROUTES ============== @correlation.route('/correlation/show', methods=['GET', 'POST']) @login_required @@ -130,6 +137,10 @@ def show_correlation(): max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) mode = sanitise_graph_mode(request.args.get('mode')) level = sanitise_level(request.args.get('level')) + objs_hidden = sanitise_objs_hidden(request.args.get('hidden')) + obj_to_hide = request.args.get('hide') + if obj_to_hide: + objs_hidden.add(obj_to_hide) related_btc = bool(request.args.get('related_btc', False)) @@ -139,17 +150,24 @@ def show_correlation(): if not ail_objects.exists_obj(obj_type, subtype, obj_id): return abort(404) # object exist - else: - dict_object = {"object_type": obj_type, - "correlation_id": obj_id, + else: # TODO remove old dict key + dict_object = {"type": obj_type, + "id": obj_id, + "object_type": obj_type, "max_nodes": max_nodes, "mode": mode, "level": level, "filter": filter_types, "filter_str": ",".join(filter_types), + "hidden": objs_hidden, "hidden_str": ",".join(objs_hidden), + + "correlation_id": obj_id, "metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags'}, flask_context=True), "nb_correl": ail_objects.get_obj_nb_correlations(obj_type, subtype, obj_id) } if subtype: + dict_object["subtype"] = subtype dict_object["metadata"]['type_id'] = subtype + else: + dict_object["subtype"] = '' dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id, related_btc=related_btc) return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label, tags_selector_data=Tag.get_tags_selector_data()) @@ -194,9 +212,15 @@ def graph_node_json(): max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) level = sanitise_level(request.args.get('level')) + hidden = request.args.get('hidden') + if hidden: + hidden = set(hidden.split(',')) + else: + hidden = set() + filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(',')) - json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True) + json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, objs_hidden=hidden, flask_context=True) #json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes) return jsonify(json_graph) @@ -224,6 +248,7 @@ def correlation_tags_add(): nb_max = sanitise_nb_max_nodes(request.form.get('tag_nb_max')) level = sanitise_level(request.form.get('tag_level')) filter_types = ail_objects.sanitize_objs_types(request.form.get('tag_filter', '').split(',')) + hidden = sanitise_objs_hidden(request.form.get('tag_hidden')) if not ail_objects.exists_obj(obj_type, subtype, obj_id): return abort(404) @@ -252,9 +277,11 @@ def correlation_tags_add(): if tags: ail_objects.obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=filter_types, + objs_hidden=hidden, lvl=level + 1, nb_max=nb_max) return redirect(url_for('correlation.show_correlation', type=obj_type, subtype=subtype, id=obj_id, level=level, max_nodes=nb_max, + hidden=hidden, hidden_str=",".join(hidden), filter=",".join(filter_types))) diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 326b637f..b243bd35 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -164,6 +164,11 @@  Resize Graph + + +
     Graph Incomplete, Max Nodes Reached.
    @@ -180,6 +185,14 @@ +

    Press H on an object / node to hide it.

    + {% if dict_object["hidden"] %} +
    Hidden objects:
    + {% for obj_hidden in dict_object["hidden"] %} + {{ obj_hidden }}
    + {% endfor %} + {% endif %} +
    @@ -343,6 +356,7 @@ + {% include 'tags/block_tags_selector.html' %} + {% endfor %} + +
    +
    • + + + {% with obj_type='hhhash', obj_id=dict_object['correlation_id'], obj_subtype='' %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + + + + + + + + + diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index b243bd35..11a85cd7 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -119,6 +119,8 @@ {% include 'correlation/metadata_card_cookie_name.html' %} {% elif dict_object["object_type"] == "etag" %} {% include 'correlation/metadata_card_etag.html' %} + {% elif dict_object["object_type"] == "hhhash" %} + {% include 'correlation/metadata_card_hhhash.html' %} {% elif dict_object["object_type"] == "item" %} {% include 'correlation/metadata_card_item.html' %} {% endif %} @@ -230,6 +232,10 @@ +
    + + +
    diff --git a/var/www/templates/objects/hhhash/HHHashDaterange.html b/var/www/templates/objects/hhhash/HHHashDaterange.html new file mode 100644 index 00000000..79e12238 --- /dev/null +++ b/var/www/templates/objects/hhhash/HHHashDaterange.html @@ -0,0 +1,602 @@ + + + + + HHHashs - AIL + + + + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + +
    +
    +
    + +{# {% include 'hhhash/block_hhhash_search.html' %}#} + +
    + + +
    + +
    +
    +
    Select a date range :
    +
    +
    +
    + +
    +
    +
    + +
    +
    + + +
    + +
    +
    +
    + +
    +
    +
    +
    +
    +
    + + {% if dict_objects %} + {% if date_from|string == date_to|string %} +

    {{ date_from }} HHHashs Name:

    + {% else %} +

    {{ date_from }} to {{ date_to }} HHHashs Name:

    + {% endif %} + + + + + + + + + + + + {% for obj_id in dict_objects %} + + + + + + + + {% endfor %} + +
    First SeenLast SeenTotalLast days
    {{ dict_objects[obj_id]['content'] }}{{ dict_objects[obj_id]['first_seen'] }}{{ dict_objects[obj_id]['last_seen'] }}{{ dict_objects[obj_id]['nb_seen'] }}
    + + + {% else %} + {% if show_objects %} + {% if date_from|string == date_to|string %} +

    {{ date_from }}, No HHHash Name

    + {% else %} +

    {{ date_from }} to {{ date_to }}, No HHHash Name

    + {% endif %} + {% endif %} + {% endif %} +
    + +
    +
    + + + + + + + + + + + + + + + + + diff --git a/var/www/templates/sidebars/sidebar_objects.html b/var/www/templates/sidebars/sidebar_objects.html index 5f7b00b5..12b5abc0 100644 --- a/var/www/templates/sidebars/sidebar_objects.html +++ b/var/www/templates/sidebars/sidebar_objects.html @@ -46,6 +46,12 @@ Etag     +
  • + + + HHHash + +
  • From fe2769308b41600bceb5fd17e861aa0b98dfc57d Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 18 Jul 2023 14:30:00 +0200 Subject: [PATCH 074/238] fix: [crawler] fix down domain + domain redirection history --- bin/crawlers/Crawler.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index ab972c68..7f2c3df9 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -60,6 +60,7 @@ class Crawler(AbstractModule): self.root_item = None self.date = None self.items_dir = None + self.original_domain = None self.domain = None # TODO Replace with warning list ??? @@ -190,6 +191,7 @@ class Crawler(AbstractModule): print(domain) self.domain = Domain(domain) + self.original_domain = Domain(domain) epoch = int(time.time()) parent_id = task.get_parent() @@ -212,12 +214,20 @@ class Crawler(AbstractModule): # Origin + History + tags if self.root_item: self.domain.set_last_origin(parent_id) - self.domain.add_history(epoch, root_item=self.root_item) # Tags for tag in task.get_tags(): self.domain.add_tag(tag) - elif self.domain.was_up(): - self.domain.add_history(epoch, root_item=epoch) + self.domain.add_history(epoch, root_item=self.root_item) + + if self.domain != self.original_domain: + self.original_domain.update_daterange(self.date.replace('/', '')) + if self.root_item: + self.original_domain.set_last_origin(parent_id) + # Tags + for tag in task.get_tags(): + self.domain.add_tag(tag) + self.original_domain.add_history(epoch, root_item=self.root_item) + crawlers.update_last_crawled_domain(self.original_domain.get_domain_type(), self.original_domain.id, epoch) crawlers.update_last_crawled_domain(self.domain.get_domain_type(), self.domain.id, epoch) print('capture:', capture.uuid, 'completed') From 68dffcd26b53229525ffe649526e3cef669dbcef Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 25 Jul 2023 15:57:11 +0200 Subject: [PATCH 075/238] chg: [api crawler] fix response + add cookiejar, proxy and frequency parameters --- bin/lib/crawlers.py | 10 +- bin/modules/Onion.py | 8 +- tools/crawler_add_task.py | 120 +++++++++++++++++++++++ var/www/modules/restApi/Flask_restApi.py | 5 +- 4 files changed, 134 insertions(+), 9 deletions(-) create mode 100755 tools/crawler_add_task.py diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 571af921..3e61ed88 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -1723,14 +1723,16 @@ def api_add_crawler_task(data, user_id=None): if frequency: # TODO verify user - return create_schedule(frequency, user_id, url, depth=depth_limit, har=har, screenshot=screenshot, header=None, - cookiejar=cookiejar_uuid, proxy=proxy, user_agent=None, tags=tags), 200 + task_uuid = create_schedule(frequency, user_id, url, depth=depth_limit, har=har, screenshot=screenshot, header=None, + cookiejar=cookiejar_uuid, proxy=proxy, user_agent=None, tags=tags) else: # TODO HEADERS # TODO USER AGENT - return create_task(url, depth=depth_limit, har=har, screenshot=screenshot, header=None, + task_uuid = create_task(url, depth=depth_limit, har=har, screenshot=screenshot, header=None, cookiejar=cookiejar_uuid, proxy=proxy, user_agent=None, tags=tags, - parent='manual', priority=90), 200 + parent='manual', priority=90) + + return {'uuid': task_uuid}, 200 #### #### diff --git a/bin/modules/Onion.py b/bin/modules/Onion.py index 5e76d0fa..303b5bdc 100755 --- a/bin/modules/Onion.py +++ b/bin/modules/Onion.py @@ -42,7 +42,8 @@ class Onion(AbstractModule): self.faup = crawlers.get_faup() # activate_crawler = p.config.get("Crawler", "activate_crawler") - + self.har = config_loader.get_config_boolean('Crawler', 'default_har') + self.screenshot = config_loader.get_config_boolean('Crawler', 'default_screenshot') self.onion_regex = r"((http|https|ftp)?(?:\://)?([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.onion)(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*)" # self.i2p_regex = r"((http|https|ftp)?(?:\://)?([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.i2p)(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*)" @@ -90,8 +91,9 @@ class Onion(AbstractModule): if onion_urls: if crawlers.is_crawler_activated(): - for domain in domains: # TODO LOAD DEFAULT SCREENSHOT + HAR - task_uuid = crawlers.create_task(domain, parent=item.get_id(), priority=0) + for domain in domains: + task_uuid = crawlers.create_task(domain, parent=item.get_id(), priority=0, + har=self.har, screenshot=self.screenshot) if task_uuid: print(f'{domain} added to crawler queue: {task_uuid}') else: diff --git a/tools/crawler_add_task.py b/tools/crawler_add_task.py new file mode 100755 index 00000000..f785f167 --- /dev/null +++ b/tools/crawler_add_task.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +DIR/File Importer Helper +================ + +Import Content + +""" + +import argparse +import os +from pyail import PyAIL +import sys + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader + +def check_frequency(value): + value = int(value) + if value <= 0: + raise argparse.ArgumentTypeError(f'Error: Invalid frequency {value}') + + +if __name__ == "__main__": + + # TODO add c argument for config file + parser = argparse.ArgumentParser(description='Directory or file importer') + parser.add_argument('-u', '--url', type=str, help='URL to crawl', required=True) + parser.add_argument('-k', '--key', type=str, help='AIL API Key', required=True) + parser.add_argument('-a', '--ail', type=str, help='AIL URL') + parser.add_argument('-d', '--depth', type=int, default=1, help='Depth limit') # TODO improve me + parser.add_argument('--cookiejar', type=str, help='Cookiejar uuid') + parser.add_argument('-p', '--proxy', type=str, help='Proxy address to use, "web" and "tor" can be used as shortcut (web is used by default is )') + + group = parser.add_mutually_exclusive_group() + group.add_argument('--har', dest='har', action='store_true', help='Save HAR') + group.add_argument('--no-har', dest='har', action='store_false', help='Don\'t save HAR') + parser.set_defaults(har=None) + + group = parser.add_mutually_exclusive_group() + group.add_argument('--screenshot', dest='screenshot', action='store_true', help='Save screenshot') + group.add_argument('--no-screenshot', dest='screenshot', action='store_false', help='Don\'t save screenshot') + parser.set_defaults(screenshot=None) + + group = parser.add_argument_group('Frequency, create a regular crawler/scheduler') + group.add_argument('-f', '--frequency', type=str, choices=['monthly', 'weekly', 'daily', 'hourly'], + help='monthly, weekly, daily or hourly frequency or specify a custom one with the others arguments') + group.add_argument('--minutes', type=int, help='frequency in minutes') + group.add_argument('--hours', type=int, help='frequency in hours') + group.add_argument('--days', type=int, help='frequency in days') + group.add_argument('--weeks', type=int, help='frequency in weeks') + group.add_argument('--months', type=int, help='frequency in months') + + args = parser.parse_args() + + if not args.url and not args.key: + parser.print_help() + sys.exit(0) + + # Load crawler default config + config_loader = ConfigLoader() + har = args.har + if har is None: + har = config_loader.get_config_boolean('Crawler', 'default_har') + screenshot = args.screenshot + if screenshot is None: + screenshot = config_loader.get_config_boolean('Crawler', 'default_screenshot') + + if args.depth: + depth = args.depth + if depth < 0: + raise argparse.ArgumentTypeError(f'Error: Invalid depth {depth}') + else: + depth = 1 + + # frequency + frequency = {} + if args.frequency: + if args.frequency in ['monthly', 'weekly', 'daily', 'hourly']: + frequency = args.frequency + else: + raise argparse.ArgumentTypeError('Invalid frequency') + elif args.minutes or args.hours or args.days or args.weeks or args.months: + if args.minutes: + check_frequency(args.minutes) + frequency['minutes'] = args.minutes + if args.hours: + check_frequency(args.hours) + frequency['hours'] = args.hours + if args.days: + check_frequency(args.days) + frequency['days'] = args.days + if args.weeks: + check_frequency(args.weeks) + frequency['weeks'] = args.weeks + if args.months: + check_frequency(args.months) + frequency['months'] = args.months + if not frequency: + frequency = None + + proxy = args.proxy + + if args.cookiejar: + cookiejar = args.cookiejar + else: + cookiejar = None + + ail = args.ail + if not ail: + ail = 'https://localhost:7000/' + + client = PyAIL(ail, args.key, ssl=False) + r = client.crawl_url(args.url, har=har, screenshot=screenshot, depth_limit=depth, frequency=frequency, + cookiejar=cookiejar, proxy=proxy) + print(r) diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index e233bb00..0c95d0a0 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* -''' +""" Flask functions and routes for the rest api -''' +""" import os import re @@ -508,6 +508,7 @@ def get_item_cryptocurrency_bitcoin(): # # # # # # # # # # # # # # CRAWLER # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TODO: ADD RESULT JSON Response +# @restApi.route("api/v1/crawler/task/add", methods=['POST']) @restApi.route("api/v1/add/crawler/task", methods=['POST']) @token_required('analyst') def add_crawler_task(): From 6a30d6149bcecd0044f2aa61582b5a2dbc53b48b Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 25 Jul 2023 16:04:45 +0200 Subject: [PATCH 076/238] fix: [tools crawler] improve helper --- tools/crawler_add_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/crawler_add_task.py b/tools/crawler_add_task.py index f785f167..455a12fc 100755 --- a/tools/crawler_add_task.py +++ b/tools/crawler_add_task.py @@ -46,7 +46,7 @@ if __name__ == "__main__": group.add_argument('--no-screenshot', dest='screenshot', action='store_false', help='Don\'t save screenshot') parser.set_defaults(screenshot=None) - group = parser.add_argument_group('Frequency, create a regular crawler/scheduler') + group = parser.add_argument_group('Frequency, create a regular crawler/scheduler. one shot if not specified') group.add_argument('-f', '--frequency', type=str, choices=['monthly', 'weekly', 'daily', 'hourly'], help='monthly, weekly, daily or hourly frequency or specify a custom one with the others arguments') group.add_argument('--minutes', type=int, help='frequency in minutes') From 4f79944b2452073ab197486c551a09d83217830f Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 25 Jul 2023 16:11:36 +0200 Subject: [PATCH 077/238] fix: [tools crawler] fix helper --- tools/crawler_add_task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/crawler_add_task.py b/tools/crawler_add_task.py index 455a12fc..6b59242a 100755 --- a/tools/crawler_add_task.py +++ b/tools/crawler_add_task.py @@ -28,13 +28,13 @@ def check_frequency(value): if __name__ == "__main__": # TODO add c argument for config file - parser = argparse.ArgumentParser(description='Directory or file importer') + parser = argparse.ArgumentParser(description='Send an URL to the crawler - Create a crawler task') parser.add_argument('-u', '--url', type=str, help='URL to crawl', required=True) parser.add_argument('-k', '--key', type=str, help='AIL API Key', required=True) parser.add_argument('-a', '--ail', type=str, help='AIL URL') parser.add_argument('-d', '--depth', type=int, default=1, help='Depth limit') # TODO improve me parser.add_argument('--cookiejar', type=str, help='Cookiejar uuid') - parser.add_argument('-p', '--proxy', type=str, help='Proxy address to use, "web" and "tor" can be used as shortcut (web is used by default is )') + parser.add_argument('-p', '--proxy', type=str, help='Proxy address to use, "web" and "tor" can be used as shortcut (web is used by default if the domain isn\'t an onion)') group = parser.add_mutually_exclusive_group() group.add_argument('--har', dest='har', action='store_true', help='Save HAR') From fc477ebcb0308b65f0d7b86b38d8153539b15421 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 26 Jul 2023 10:14:03 +0200 Subject: [PATCH 078/238] Update crawler_add_task.py --- tools/crawler_add_task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/crawler_add_task.py b/tools/crawler_add_task.py index 6b59242a..5d9604fe 100755 --- a/tools/crawler_add_task.py +++ b/tools/crawler_add_task.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ -DIR/File Importer Helper +Send an URL to the crawler - Create a crawler task ================ -Import Content +Import URL to be crawled by AIL and then analysed """ From 4eb5b4c4b9aeabf353a30c29d2d744ce3d09191a Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 28 Jul 2023 11:10:21 +0200 Subject: [PATCH 079/238] fix: [MailExporter] improve logs + SMTP over SSL: add an option to validate self signed CA This exporter use STARTTLS by defaults. If the SMTP server does not support the STARTTLS extension, the client will switch to SSL instead. --- bin/exporter/MailExporter.py | 33 +++++++++++++++++++++++++++------ configs/core.cfg.sample | 4 ++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/bin/exporter/MailExporter.py b/bin/exporter/MailExporter.py index 4b2d4a3a..c4d3f5b5 100755 --- a/bin/exporter/MailExporter.py +++ b/bin/exporter/MailExporter.py @@ -8,9 +8,12 @@ Import Content """ import os +import logging +import logging.config import sys from abc import ABC +from ssl import create_default_context import smtplib from email.mime.multipart import MIMEMultipart @@ -22,17 +25,22 @@ sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## +from lib import ail_logger from exporter.abstract_exporter import AbstractExporter from lib.ConfigLoader import ConfigLoader # from lib.objects.abstract_object import AbstractObject # from lib.Tracker import Tracker +logging.config.dictConfig(ail_logger.get_config(name='modules')) + class MailExporter(AbstractExporter, ABC): - def __init__(self, host=None, port=None, password=None, user='', sender=''): + def __init__(self, host=None, port=None, password=None, user='', sender='', cert_required=None, ca_file=None): super().__init__() config_loader = ConfigLoader() + self.logger = logging.getLogger(f'{self.__class__.__name__}') + if host: self.host = host self.port = port @@ -45,6 +53,15 @@ class MailExporter(AbstractExporter, ABC): self.pw = config_loader.get_config_str("Notifications", "sender_pw") if self.pw == 'None': self.pw = None + if cert_required is not None: + self.cert_required = bool(cert_required) + self.ca_file = ca_file + else: + self.cert_required = config_loader.get_config_boolean("Notifications", "cert_required") + if self.cert_required: + self.ca_file = config_loader.get_config_str("Notifications", "ca_file") + else: + self.ca_file = None if user: self.user = user else: @@ -67,8 +84,12 @@ class MailExporter(AbstractExporter, ABC): smtp_server = smtplib.SMTP(self.host, self.port) smtp_server.starttls() except smtplib.SMTPNotSupportedError: - print("The server does not support the STARTTLS extension.") - smtp_server = smtplib.SMTP_SSL(self.host, self.port) + self.logger.info(f"The server {self.host}:{self.port} does not support the STARTTLS extension.") + if self.cert_required: + context = create_default_context(cafile=self.ca_file) + else: + context = None + smtp_server = smtplib.SMTP_SSL(self.host, self.port, context=context) smtp_server.ehlo() if self.user is not None: @@ -80,7 +101,7 @@ class MailExporter(AbstractExporter, ABC): return smtp_server # except Exception as err: # traceback.print_tb(err.__traceback__) - # logger.warning(err) + # self.logger.warning(err) def _export(self, recipient, subject, body): mime_msg = MIMEMultipart() @@ -95,8 +116,8 @@ class MailExporter(AbstractExporter, ABC): smtp_client.quit() # except Exception as err: # traceback.print_tb(err.__traceback__) - # logger.warning(err) - print(f'Send notification: {subject} to {recipient}') + # self.logger.warning(err) + self.logger.info(f'Send notification: {subject} to {recipient}') class MailExporterTracker(MailExporter): diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index bb9054fc..9bdf40a4 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -45,6 +45,10 @@ sender = sender@example.com sender_host = smtp.example.com sender_port = 1337 sender_pw = None +# Only needed for SSL if the mail server don't support TLS (used by default). use this option to validate the server certificate. +cert_required = False +# Only needed for the SSL if you want to validate your self signed certificate for SSL +ca_file = # Only needed when the credentials for email server needs a username instead of an email address #sender_user = sender sender_user = From 64252ddf752bacb4cb9c362c3213b81a7de7fdb5 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 31 Jul 2023 16:00:31 +0200 Subject: [PATCH 080/238] fix: [Tracker] edit tracker filters --- bin/lib/Tracker.py | 8 ++++++++ configs/core.cfg.sample | 4 ++-- var/www/templates/hunter/tracker_add.html | 10 +++++++--- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index b7ca5e77..f1588137 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -207,6 +207,13 @@ class Tracker: if filters: self._set_field('filters', json.dumps(filters)) + def del_filters(self, tracker_type, to_track): + filters = self.get_filters() + for obj_type in filters: + r_tracker.srem(f'trackers:objs:{tracker_type}:{obj_type}', to_track) + r_tracker.srem(f'trackers:uuid:{tracker_type}:{to_track}', f'{self.uuid}:{obj_type}') + r_tracker.hdel(f'tracker:{self.uuid}', 'filters') + def get_tracked(self): return self._get_field('tracked') @@ -513,6 +520,7 @@ class Tracker: self._set_mails(mails) # Filters + self.del_filters(old_type, old_to_track) if not filters: filters = {} for obj_type in get_objects_tracked(): diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index 9bdf40a4..3278033f 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -45,9 +45,9 @@ sender = sender@example.com sender_host = smtp.example.com sender_port = 1337 sender_pw = None -# Only needed for SSL if the mail server don't support TLS (used by default). use this option to validate the server certificate. +# Only needed for SMTP over SSL if the mail server don't support TLS (used by default). use this option to validate the server certificate. cert_required = False -# Only needed for the SSL if you want to validate your self signed certificate for SSL +# Only needed for SMTP over SSL if you want to validate your self signed certificate for SSL ca_file = # Only needed when the credentials for email server needs a username instead of an email address #sender_user = sender diff --git a/var/www/templates/hunter/tracker_add.html b/var/www/templates/hunter/tracker_add.html index 9233e4ad..7cc690ba 100644 --- a/var/www/templates/hunter/tracker_add.html +++ b/var/www/templates/hunter/tracker_add.html @@ -76,7 +76,7 @@ {#
    • #}
    - +
    {#
    #} {# #} @@ -84,7 +84,7 @@ {#
    #}
    - +
    @@ -101,7 +101,7 @@
    - +
    @@ -351,6 +351,10 @@ $(document).ready(function(){ }); +$(function () { + $('[data-toggle="tooltip"]').tooltip() +}) + function toggle_sidebar(){ if($('#nav_menu').is(':visible')){ $('#nav_menu').hide(); From 1aa0bd8a0ec20c8d6b4c1f7e26123c90b0926656 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 31 Jul 2023 16:25:28 +0200 Subject: [PATCH 081/238] fix: [settings] fix edit user --- bin/lib/Tracker.py | 3 -- bin/lib/Users.py | 8 ++++- var/www/modules/settings/Flask_settings.py | 37 ++++++++++++++-------- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index f1588137..f1ea8905 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -530,9 +530,6 @@ class Tracker: for obj_type in filters: r_tracker.sadd(f'trackers:objs:{tracker_type}:{obj_type}', to_track) r_tracker.sadd(f'trackers:uuid:{tracker_type}:{to_track}', f'{self.uuid}:{obj_type}') - if tracker_type != old_type: - r_tracker.srem(f'trackers:objs:{old_type}:{obj_type}', old_to_track) - r_tracker.srem(f'trackers:uuid:{old_type}:{old_to_track}', f'{self.uuid}:{obj_type}') # Refresh Trackers trigger_trackers_refresh(tracker_type) diff --git a/bin/lib/Users.py b/bin/lib/Users.py index a61830ef..765b1360 100755 --- a/bin/lib/Users.py +++ b/bin/lib/Users.py @@ -247,7 +247,10 @@ class User(UserMixin): self.id = "__anonymous__" def exists(self): - return self.id != "__anonymous__" + if self.id == "__anonymous__": + return False + else: + return r_serv_db.exists(f'ail:user:metadata:{self.id}') # return True or False # def is_authenticated(): @@ -287,3 +290,6 @@ class User(UserMixin): return True else: return False + + def get_role(self): + return r_serv_db.hget(f'ail:user:metadata:{self.id}', 'role') diff --git a/var/www/modules/settings/Flask_settings.py b/var/www/modules/settings/Flask_settings.py index 4316d490..2b1b8826 100644 --- a/var/www/modules/settings/Flask_settings.py +++ b/var/www/modules/settings/Flask_settings.py @@ -19,7 +19,6 @@ sys.path.append(os.environ['AIL_BIN']) from lib import d4 from lib import Users - # ============ VARIABLES ============ import Flask_config @@ -33,7 +32,6 @@ email_regex = Flask_config.email_regex settings = Blueprint('settings', __name__, template_folder='templates') - # ============ FUNCTIONS ============ def check_email(email): @@ -43,6 +41,7 @@ def check_email(email): else: return False + # ============= ROUTES ============== @settings.route("/settings/edit_profile", methods=['GET']) @@ -52,7 +51,8 @@ def edit_profile(): user_metadata = Users.get_user_metadata(current_user.get_id()) admin_level = current_user.is_in_role('admin') return render_template("edit_profile.html", user_metadata=user_metadata, - admin_level=admin_level) + admin_level=admin_level) + @settings.route("/settings/new_token", methods=['GET']) @login_required @@ -61,6 +61,7 @@ def new_token(): Users.generate_new_token(current_user.get_id()) return redirect(url_for('settings.edit_profile')) + @settings.route("/settings/new_token_user", methods=['POST']) @login_required @login_admin @@ -70,6 +71,7 @@ def new_token_user(): Users.generate_new_token(user_id) return redirect(url_for('settings.users_list')) + @settings.route("/settings/create_user", methods=['GET']) @login_required @login_admin @@ -78,14 +80,15 @@ def create_user(): error = request.args.get('error') error_mail = request.args.get('error_mail') role = None - if r_serv_db.exists('user_metadata:{}'.format(user_id)): - role = r_serv_db.hget('user_metadata:{}'.format(user_id), 'role') - else: - user_id = None + if user_id: + user = Users.User(user_id) + if user.exists(): + role = user.get_role() all_roles = Users.get_all_roles() return render_template("create_user.html", all_roles=all_roles, user_id=user_id, user_role=role, - error=error, error_mail=error_mail, - admin_level=True) + error=error, error_mail=error_mail, + admin_level=True) + @settings.route("/settings/create_user_post", methods=['POST']) @login_required @@ -98,17 +101,19 @@ def create_user_post(): all_roles = Users.get_all_roles() - if email and len(email)< 300 and check_email(email) and role: + if email and len(email) < 300 and check_email(email) and role: if role in all_roles: # password set if password1 and password2: - if password1==password2: + if password1 == password2: if Users.check_password_strength(password1): password = password1 else: - return render_template("create_user.html", all_roles=all_roles, error="Incorrect Password", admin_level=True) + return render_template("create_user.html", all_roles=all_roles, error="Incorrect Password", + admin_level=True) else: - return render_template("create_user.html", all_roles=all_roles, error="Passwords don't match", admin_level=True) + return render_template("create_user.html", all_roles=all_roles, error="Passwords don't match", + admin_level=True) # generate password else: password = Users.gen_password() @@ -127,6 +132,7 @@ def create_user_post(): else: return render_template("create_user.html", all_roles=all_roles, error_mail=True, admin_level=True) + @settings.route("/settings/users_list", methods=['GET']) @login_required @login_admin @@ -140,6 +146,7 @@ def users_list(): new_user_dict['password'] = request.args.get('new_user_password') return render_template("users_list.html", all_users=all_users, new_user=new_user_dict, admin_level=True) + @settings.route("/settings/edit_user", methods=['POST']) @login_required @login_admin @@ -147,6 +154,7 @@ def edit_user(): user_id = request.form.get('user_id') return redirect(url_for('settings.create_user', user_id=user_id)) + @settings.route("/settings/delete_user", methods=['POST']) @login_required @login_admin @@ -163,6 +171,7 @@ def passive_dns(): passivedns_enabled = d4.is_passive_dns_enabled() return render_template("passive_dns.html", passivedns_enabled=passivedns_enabled) + @settings.route("/settings/passivedns/change_state", methods=['GET']) @login_required @login_admin @@ -171,11 +180,13 @@ def passive_dns_change_state(): passivedns_enabled = d4.change_passive_dns_state(new_state) return redirect(url_for('settings.passive_dns')) + @settings.route("/settings/ail", methods=['GET']) @login_required @login_admin def ail_configs(): return render_template("ail_configs.html", passivedns_enabled=None) + # ========= REGISTRATION ========= app.register_blueprint(settings, url_prefix=baseUrl) From 14a76a91d983709ffc19d4fc713e73e01d60c029 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 1 Aug 2023 11:07:06 +0200 Subject: [PATCH 082/238] fix: [tags ui] fix galaxy, get number of tags enabled + add toolip helper --- bin/lib/Tag.py | 2 +- var/www/templates/tags/galaxies.html | 8 ++++---- var/www/templates/tags/taxonomies.html | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index 94b2eca4..64850b3c 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -338,7 +338,7 @@ def get_galaxy_meta(galaxy_name, nb_active_tags=False): else: meta['icon'] = f'fas fa-{icon}' if nb_active_tags: - meta['nb_active_tags'] = get_galaxy_nb_tags_enabled(galaxy) + meta['nb_active_tags'] = get_galaxy_nb_tags_enabled(galaxy.type) meta['nb_tags'] = len(get_galaxy_tags(galaxy.type)) return meta diff --git a/var/www/templates/tags/galaxies.html b/var/www/templates/tags/galaxies.html index ff69c2b5..ac0b443c 100644 --- a/var/www/templates/tags/galaxies.html +++ b/var/www/templates/tags/galaxies.html @@ -41,7 +41,7 @@ Namespace Enabled Active Tags - + @@ -51,7 +51,7 @@ {{ galaxy['description'] }} {{ galaxy['namespace'] }} - {% if galaxy['enebled'] %} + {% if galaxy['enabled'] %}
    @@ -69,7 +69,7 @@ -     + @@ -98,7 +98,7 @@ $(document).ready(function(){ $("#nav_galaxies").addClass("active"); - $('#myTable_').DataTable({ "lengthMenu": [ 5, 10, 25, 50, 100 ], "pageLength": 15, "order": [[ 0, "asc" ]] }); + $('#myTable_').DataTable({ "lengthMenu": [ 5, 10, 25, 50, 100 ], "pageLength": 15, "order": [[ 3, "desc" ]] }); }); diff --git a/var/www/templates/tags/taxonomies.html b/var/www/templates/tags/taxonomies.html index eff480b5..ac3346d7 100644 --- a/var/www/templates/tags/taxonomies.html +++ b/var/www/templates/tags/taxonomies.html @@ -41,7 +41,7 @@ Version Enabled Active Tags - + @@ -69,7 +69,7 @@ - + @@ -98,7 +98,7 @@ $(document).ready(function(){ $("#nav_taxonomies").addClass("active"); - $('#myTable_').DataTable({ "lengthMenu": [ 5, 10, 25, 50, 100 ], "pageLength": 15, "order": [[ 0, "asc" ]] }); + $('#myTable_').DataTable({ "lengthMenu": [ 5, 10, 25, 50, 100 ], "pageLength": 15, "order": [[ 3, "desc" ]] }); }); From 9098ab25a6cd782230ef3268d4624473654f5bee Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 1 Aug 2023 11:30:45 +0200 Subject: [PATCH 083/238] chg: [tracker ui] improve show typo squatting button + add tooltip --- var/www/templates/hunter/tracker_show.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/www/templates/hunter/tracker_show.html b/var/www/templates/hunter/tracker_show.html index c2715ccc..62c387b6 100644 --- a/var/www/templates/hunter/tracker_show.html +++ b/var/www/templates/hunter/tracker_show.html @@ -89,8 +89,8 @@ Tracked {% if meta['type'] == 'typosquatting' %} -
    From ac45c2dd61f41cf61b96fb2b1c6b86d6c8ba7685 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 1 Aug 2023 14:30:36 +0200 Subject: [PATCH 084/238] chg: [crawler ui] last crawled domains, show last check timestamp --- var/www/blueprints/crawler_splash.py | 1 + var/www/templates/crawler/crawler_splash/last_crawled.html | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index 39d84971..1b7f4454 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -272,6 +272,7 @@ def crawlers_last_domains(): domain, epoch = domain_row.split(':', 1) dom = Domains.Domain(domain) meta = dom.get_meta() + meta['last'] = datetime.fromtimestamp(int(epoch)).strftime("%Y/%m/%d %H:%M.%S") meta['epoch'] = epoch meta['status_epoch'] = dom.is_up_by_epoch(epoch) domains.append(meta) diff --git a/var/www/templates/crawler/crawler_splash/last_crawled.html b/var/www/templates/crawler/crawler_splash/last_crawled.html index a097c47a..582e45cb 100644 --- a/var/www/templates/crawler/crawler_splash/last_crawled.html +++ b/var/www/templates/crawler/crawler_splash/last_crawled.html @@ -74,7 +74,7 @@ data-content="epoch: {{domain['epoch']}}
    last status: {{ domain['status'] }}"> {{ domain['domain'] }} {{domain['first_seen']}} - {{domain['last_check']}} + {{domain['last']}} {% if domain['status_epoch'] %}
    From 7d19da0806ec4a353aca085e681ef40d9dd806b2 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 2 Aug 2023 13:50:07 +0200 Subject: [PATCH 085/238] chg: [flask] cleanup, remove unused import --- var/www/Flask_server.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 93d3a47d..c9d35232 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -17,9 +17,6 @@ from flask_login import LoginManager, current_user, login_user, logout_user, log import importlib from os.path import join -# # TODO: put me in lib/Tag -from pytaxonomies import Taxonomies - sys.path.append('./modules/') sys.path.append(os.environ['AIL_BIN']) @@ -255,6 +252,7 @@ for taxonomy in default_taxonomies: Tag.enable_taxonomy_tags(taxonomy) # ========== INITIAL tags auto export ============ +# from pytaxonomies import Taxonomies # taxonomies = Taxonomies() # # infoleak_tags = taxonomies.get('infoleak').machinetags() From 859591b53f0cd5c52864eca94bf4e8dc755f0de3 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 2 Aug 2023 13:51:13 +0200 Subject: [PATCH 086/238] chg: [flask] cleanup --- var/www/Flask_server.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index c9d35232..e6a99350 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -251,17 +251,6 @@ default_taxonomies = ["infoleak", "gdpr", "fpf", "dark-web"] for taxonomy in default_taxonomies: Tag.enable_taxonomy_tags(taxonomy) -# ========== INITIAL tags auto export ============ -# from pytaxonomies import Taxonomies -# taxonomies = Taxonomies() -# -# infoleak_tags = taxonomies.get('infoleak').machinetags() -# infoleak_automatic_tags = [] -# for tag in taxonomies.get('infoleak').machinetags(): -# if tag.split('=')[0][:] == 'infoleak:automatic-detection': -# r_serv_db.sadd('list_export_tags', tag) -# -# r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"') # ============ MAIN ============ if __name__ == "__main__": From 2691000d0cdf15b90030a12c101af02bf22c9009 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 2 Aug 2023 15:49:12 +0200 Subject: [PATCH 087/238] chg: [telegram fedeer] use meta of the new feeder --- bin/importer/feeders/Telegram.py | 34 ++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/bin/importer/feeders/Telegram.py b/bin/importer/feeders/Telegram.py index 3856c88e..c9c448ef 100755 --- a/bin/importer/feeders/Telegram.py +++ b/bin/importer/feeders/Telegram.py @@ -29,8 +29,8 @@ class TelegramFeeder(DefaultFeeder): def get_item_id(self): # TODO use telegram message date date = datetime.date.today().strftime("%Y/%m/%d") - channel_id = str(self.json_data['meta']['channel_id']) - message_id = str(self.json_data['meta']['message_id']) + channel_id = str(self.json_data['meta']['chat']['id']) + message_id = str(self.json_data['meta']['id']) item_id = f'{channel_id}_{message_id}' item_id = os.path.join('telegram', date, item_id) self.item_id = f'{item_id}.gz' @@ -40,17 +40,21 @@ class TelegramFeeder(DefaultFeeder): """ Process JSON meta field. """ - # channel_id = str(self.json_data['meta']['channel_id']) - # message_id = str(self.json_data['meta']['message_id']) - # telegram_id = f'{channel_id}_{message_id}' - # item_basic.add_map_obj_id_item_id(telegram_id, item_id, 'telegram_id') ######################################### - user = None - if self.json_data['meta'].get('user'): - user = str(self.json_data['meta']['user']) - elif self.json_data['meta'].get('channel'): - user = str(self.json_data['meta']['channel'].get('username')) - if user: - date = item_basic.get_item_date(self.item_id) - username = Username(user, 'telegram') - username.add(date, self.item_id) + # message chat + meta = self.json_data['meta'] + if meta.get('chat'): + if meta['chat'].get('username'): + user = meta['chat']['username'] + if user: + date = item_basic.get_item_date(self.item_id) + username = Username(user, 'telegram') + username.add(date, self.item_id) + # message sender + if meta.get('sender'): + if meta['sender'].get('username'): + user = meta['sender']['username'] + if user: + date = item_basic.get_item_date(self.item_id) + username = Username(user, 'telegram') + username.add(date, self.item_id) return None From bd7aa979bd847a516155bb896d6453a961f63116 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 8 Aug 2023 10:36:58 +0200 Subject: [PATCH 088/238] chg: [module extrator] add debug --- bin/lib/module_extractor.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/lib/module_extractor.py b/bin/lib/module_extractor.py index d4ea6c78..54e4d3ab 100755 --- a/bin/lib/module_extractor.py +++ b/bin/lib/module_extractor.py @@ -104,7 +104,11 @@ def _get_word_regex(word): def convert_byte_offset_to_string(b_content, offset): byte_chunk = b_content[:offset + 1] - string_chunk = byte_chunk.decode() + try: + string_chunk = byte_chunk.decode() + except UnicodeDecodeError as e: + logger.error(f'Yara offset coverter error, {e.reason}\n{b_content}\n{offset}') + string_chunk = b_content offset = len(string_chunk) - 1 return offset From 529a24c191299551ccd90959bea8ae382c0fa5da Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 8 Aug 2023 10:40:44 +0200 Subject: [PATCH 089/238] chg: [module extrator] add debug --- bin/lib/module_extractor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/lib/module_extractor.py b/bin/lib/module_extractor.py index 54e4d3ab..cdb67ab6 100755 --- a/bin/lib/module_extractor.py +++ b/bin/lib/module_extractor.py @@ -107,8 +107,8 @@ def convert_byte_offset_to_string(b_content, offset): try: string_chunk = byte_chunk.decode() except UnicodeDecodeError as e: - logger.error(f'Yara offset coverter error, {e.reason}\n{b_content}\n{offset}') - string_chunk = b_content + logger.error(f'Yara offset converter error, {e.reason}\n{byte_chunk}\n{offset}') + string_chunk = byte_chunk offset = len(string_chunk) - 1 return offset From 4dc5527c1a74586a4fd97a646559844cfdbc8da9 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 8 Aug 2023 11:26:16 +0200 Subject: [PATCH 090/238] fix: [module extractor] fix invalid yara offset --- bin/lib/module_extractor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/lib/module_extractor.py b/bin/lib/module_extractor.py index cdb67ab6..681d666e 100755 --- a/bin/lib/module_extractor.py +++ b/bin/lib/module_extractor.py @@ -106,11 +106,11 @@ def convert_byte_offset_to_string(b_content, offset): byte_chunk = b_content[:offset + 1] try: string_chunk = byte_chunk.decode() + offset = len(string_chunk) - 1 + return offset except UnicodeDecodeError as e: - logger.error(f'Yara offset converter error, {e.reason}\n{byte_chunk}\n{offset}') - string_chunk = byte_chunk - offset = len(string_chunk) - 1 - return offset + logger.error(f'Yara offset converter error, {str(e)}\n{offset}/{len(b_content)}') + return convert_byte_offset_to_string(b_content, offset) # TODO RETRO HUNTS From f05c7b6a93ff5a166d338d9d9fdd6a7825c33918 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 8 Aug 2023 11:27:57 +0200 Subject: [PATCH 091/238] fix: [module extractor] fix invalid yara offset --- bin/lib/module_extractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/lib/module_extractor.py b/bin/lib/module_extractor.py index 681d666e..b6254372 100755 --- a/bin/lib/module_extractor.py +++ b/bin/lib/module_extractor.py @@ -110,7 +110,7 @@ def convert_byte_offset_to_string(b_content, offset): return offset except UnicodeDecodeError as e: logger.error(f'Yara offset converter error, {str(e)}\n{offset}/{len(b_content)}') - return convert_byte_offset_to_string(b_content, offset) + return convert_byte_offset_to_string(b_content, offset - 1) # TODO RETRO HUNTS From 3c1813ba02bec0dac4a0dc677cda771be55a23fe Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 18 Aug 2023 11:05:21 +0200 Subject: [PATCH 092/238] chg: [core] add telegram importer + Chat object + message Object + add timeline engine --- bin/importer/FeederImporter.py | 9 +- bin/importer/feeders/Telegram.py | 110 +++++-- bin/lib/ail_core.py | 16 +- bin/lib/correlations_engine.py | 7 +- bin/lib/objects/Chats.py | 288 ++++++++++++++++++ bin/lib/objects/Items.py | 2 + bin/lib/objects/Messages.py | 268 ++++++++++++++++ bin/lib/objects/UsersAccount.py | 154 ++++++++++ bin/lib/objects/abstract_daterange_object.py | 4 +- bin/lib/objects/abstract_object.py | 56 ++++ bin/lib/objects/ail_objects.py | 3 + bin/lib/timeline_engine.py | 157 ++++++++++ configs/6383.conf | 1 + configs/core.cfg.sample | 5 + var/www/Flask_server.py | 2 + var/www/blueprints/objects_chat.py | 58 ++++ var/www/blueprints/objects_subtypes.py | 6 + .../templates/objects/chat/ChatMessages.html | 190 ++++++++++++ 18 files changed, 1307 insertions(+), 29 deletions(-) create mode 100755 bin/lib/objects/Chats.py create mode 100755 bin/lib/objects/Messages.py create mode 100755 bin/lib/objects/UsersAccount.py create mode 100755 bin/lib/timeline_engine.py create mode 100644 var/www/blueprints/objects_chat.py create mode 100644 var/www/templates/objects/chat/ChatMessages.html diff --git a/bin/importer/FeederImporter.py b/bin/importer/FeederImporter.py index e7a06132..021a52e2 100755 --- a/bin/importer/FeederImporter.py +++ b/bin/importer/FeederImporter.py @@ -87,13 +87,16 @@ class FeederImporter(AbstractImporter): feeder_name = feeder.get_name() print(f'importing: {feeder_name} feeder') - item_id = feeder.get_item_id() + item_id = feeder.get_item_id() # TODO replace me with object global id # process meta if feeder.get_json_meta(): feeder.process_meta() - gzip64_content = feeder.get_gzip64_content() - return f'{feeder_name} {item_id} {gzip64_content}' + if feeder_name == 'telegram': + return item_id # TODO support UI dashboard + else: + gzip64_content = feeder.get_gzip64_content() + return f'{feeder_name} {item_id} {gzip64_content}' class FeederModuleImporter(AbstractModule): diff --git a/bin/importer/feeders/Telegram.py b/bin/importer/feeders/Telegram.py index c9c448ef..52eb0a75 100755 --- a/bin/importer/feeders/Telegram.py +++ b/bin/importer/feeders/Telegram.py @@ -16,9 +16,30 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from importer.feeders.Default import DefaultFeeder +from lib.ConfigLoader import ConfigLoader +from lib.objects.Chats import Chat +from lib.objects import Messages +from lib.objects import UsersAccount from lib.objects.Usernames import Username from lib import item_basic +import base64 +import io +import gzip +def gunzip_bytes_obj(bytes_obj): + gunzipped_bytes_obj = None + try: + in_ = io.BytesIO() + in_.write(bytes_obj) + in_.seek(0) + + with gzip.GzipFile(fileobj=in_, mode='rb') as fo: + gunzipped_bytes_obj = fo.read() + except Exception as e: + print(f'Global; Invalid Gzip file: {e}') + + return gunzipped_bytes_obj + class TelegramFeeder(DefaultFeeder): def __init__(self, json_data): @@ -26,14 +47,17 @@ class TelegramFeeder(DefaultFeeder): self.name = 'telegram' # define item id - def get_item_id(self): - # TODO use telegram message date - date = datetime.date.today().strftime("%Y/%m/%d") - channel_id = str(self.json_data['meta']['chat']['id']) + def get_item_id(self): # TODO rename self.item_id + # Get message date + timestamp = self.json_data['meta']['date']['timestamp'] # TODO CREATE DEFAULT TIMESTAMP + # if self.json_data['meta'].get('date'): + # date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp']) + # date = date.strftime('%Y/%m/%d') + # else: + # date = datetime.date.today().strftime("%Y/%m/%d") + chat_id = str(self.json_data['meta']['chat']['id']) message_id = str(self.json_data['meta']['id']) - item_id = f'{channel_id}_{message_id}' - item_id = os.path.join('telegram', date, item_id) - self.item_id = f'{item_id}.gz' + self.item_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) return self.item_id def process_meta(self): @@ -42,19 +66,67 @@ class TelegramFeeder(DefaultFeeder): """ # message chat meta = self.json_data['meta'] + mess_id = self.json_data['meta']['id'] + if meta.get('reply_to'): + reply_to_id = meta['reply_to'] + else: + reply_to_id = None + + timestamp = meta['date']['timestamp'] + date = datetime.datetime.fromtimestamp(timestamp) + date = date.strftime('%Y%m%d') + if meta.get('chat'): - if meta['chat'].get('username'): - user = meta['chat']['username'] - if user: - date = item_basic.get_item_date(self.item_id) - username = Username(user, 'telegram') - username.add(date, self.item_id) + chat = Chat(meta['chat']['id'], 'telegram') + + if meta['chat'].get('username'): # TODO USE ID AND SAVE USERNAME + chat_username = meta['chat']['username'] + + # Chat---Message + chat.add(date, self.item_id) # TODO modify to accept file objects + # message meta ????? who is the user if two user ???? + + if self.json_data.get('translation'): + translation = self.json_data['translation'] + else: + translation = None + decoded = base64.standard_b64decode(self.json_data['data']) + content = gunzip_bytes_obj(decoded) + Messages.create(self.item_id, content, translation=translation) + + chat.add_message(self.item_id, timestamp, mess_id, reply_id=reply_to_id) + else: + chat = None + # message sender - if meta.get('sender'): + if meta.get('sender'): # TODO handle message channel forward + user_id = meta['sender']['id'] + user_account = UsersAccount.UserAccount(user_id, 'telegram') + # UserAccount---Message + user_account.add(date, self.item_id) + # UserAccount---Chat + user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id) + + if meta['sender'].get('firstname'): + user_account.set_first_name(meta['sender']['firstname']) + if meta['sender'].get('lastname'): + user_account.set_last_name(meta['sender']['lastname']) + if meta['sender'].get('phone'): + user_account.set_phone(meta['sender']['phone']) + if meta['sender'].get('username'): - user = meta['sender']['username'] - if user: - date = item_basic.get_item_date(self.item_id) - username = Username(user, 'telegram') - username.add(date, self.item_id) + username = Username(meta['sender']['username'], 'telegram') + user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id) + + # Username---Message + username.add(date, self.item_id) # TODO #################################################################### + if chat: + chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id) + + # if meta.get('fwd_from'): + # if meta['fwd_from'].get('post_author') # user first name + + # TODO reply threads ???? + + return None diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 75520a2b..eeb83a98 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -15,8 +15,8 @@ config_loader = ConfigLoader() r_serv_db = config_loader.get_db_conn("Kvrocks_DB") config_loader = None -AIL_OBJECTS = sorted({'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash', 'item', - 'pgp', 'screenshot', 'title', 'username'}) +AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash', 'item', + 'pgp', 'screenshot', 'title', 'user-account', 'username'}) def get_ail_uuid(): ail_uuid = r_serv_db.get('ail:uuid') @@ -38,9 +38,11 @@ def get_all_objects(): return AIL_OBJECTS def get_objects_with_subtypes(): - return ['cryptocurrency', 'pgp', 'username'] + return ['chat', 'cryptocurrency', 'pgp', 'username'] def get_object_all_subtypes(obj_type): + if obj_type == 'chat': + return ['discord', 'jabber', 'telegram'] if obj_type == 'cryptocurrency': return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash'] if obj_type == 'pgp': @@ -66,6 +68,14 @@ def get_all_objects_with_subtypes_tuple(): str_objs.append((obj_type, '')) return str_objs +def unpack_obj_global_id(global_id, r_type='tuple'): + if r_type == 'dict': + obj = global_id.split(':', 2) + return {'type': obj[0], 'subtype': obj[1], 'id': obj['2']} + else: # tuple(type, subtype, id) + return global_id.split(':', 2) + + ##-- AIL OBJECTS --## #### Redis #### diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index 609aa8c6..94a06773 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -41,6 +41,7 @@ config_loader = None ################################## CORRELATION_TYPES_BY_OBJ = { + "chat": ["item", "username"], # item ??? "cookie-name": ["domain"], "cryptocurrency": ["domain", "item"], "cve": ["domain", "item"], @@ -49,11 +50,11 @@ CORRELATION_TYPES_BY_OBJ = { "etag": ["domain"], "favicon": ["domain", "item"], # TODO Decoded "hhhash": ["domain"], - "item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], + "item": ["chat", "cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], "pgp": ["domain", "item"], "screenshot": ["domain", "item"], "title": ["domain", "item"], - "username": ["domain", "item"], + "username": ["chat", "domain", "item"], } def get_obj_correl_types(obj_type): @@ -65,6 +66,8 @@ def sanityze_obj_correl_types(obj_type, correl_types): correl_types = set(correl_types).intersection(obj_correl_types) if not correl_types: correl_types = obj_correl_types + if not correl_types: + return [] return correl_types def get_nb_correlation_by_correl_type(obj_type, subtype, obj_id, correl_type): diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py new file mode 100755 index 00000000..438acf51 --- /dev/null +++ b/bin/lib/objects/Chats.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys + +from datetime import datetime + +from flask import url_for +# from pymisp import MISPObject + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import ail_core +from lib.ConfigLoader import ConfigLoader +from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id +from lib.data_retention_engine import update_obj_date +from lib.objects import ail_objects +from lib import item_basic + +from lib.correlations_engine import get_correlation_by_correl_type + +config_loader = ConfigLoader() +baseurl = config_loader.get_config_str("Notifications", "ail_domain") +r_object = config_loader.get_db_conn("Kvrocks_Objects") +r_cache = config_loader.get_redis_conn("Redis_Cache") +config_loader = None + + +################################################################################ +################################################################################ +################################################################################ + +class Chat(AbstractSubtypeObject): # TODO # ID == username ????? + """ + AIL Chat Object. (strings) + """ + + def __init__(self, id, subtype): + super(Chat, self).__init__('chat', id, subtype) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True), + # 'compress': 'gzip'} + # return payload + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + # # TODO: + pass + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id) + else: + url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}' + return url + + def get_svg_icon(self): # TODO + # if self.subtype == 'telegram': + # style = 'fab' + # icon = '\uf2c6' + # elif self.subtype == 'discord': + # style = 'fab' + # icon = '\uf099' + # else: + # style = 'fas' + # icon = '\uf007' + style = 'fas' + icon = '\uf086' + return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5} + + def get_meta(self, options=set()): + meta = self._get_meta(options=options) + meta['id'] = self.id + meta['subtype'] = self.subtype + meta['tags'] = self.get_tags(r_list=True) + return meta + + def get_misp_object(self): + # obj_attrs = [] + # if self.subtype == 'telegram': + # obj = MISPObject('telegram-account', standalone=True) + # obj_attrs.append(obj.add_attribute('username', value=self.id)) + # + # elif self.subtype == 'twitter': + # obj = MISPObject('twitter-account', standalone=True) + # obj_attrs.append(obj.add_attribute('name', value=self.id)) + # + # else: + # obj = MISPObject('user-account', standalone=True) + # obj_attrs.append(obj.add_attribute('username', value=self.id)) + # + # first_seen = self.get_first_seen() + # last_seen = self.get_last_seen() + # if first_seen: + # obj.first_seen = first_seen + # if last_seen: + # obj.last_seen = last_seen + # if not first_seen or not last_seen: + # self.logger.warning( + # f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}') + # + # for obj_attr in obj_attrs: + # for tag in self.get_tags(): + # obj_attr.add_tag(tag) + # return obj + return + + ############################################################################ + ############################################################################ + + # others optional metas, ... -> # TODO ALL meta in hset + + def get_name(self): # get username ???? + pass + + # return username correlation + def get_users(self): # get participants ??? -> passive users ??? + pass + + # def get_last_message_id(self): + # + # return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id') + + def get_obj_message_id(self, obj_id): + if obj_id.endswith('.gz'): + obj_id = obj_id[:-3] + return int(obj_id.split('_')[-1]) + + def _get_message_timestamp(self, obj_global_id): + return r_object.zscore(f'messages:{self.type}:{self.subtype}:{self.id}', obj_global_id) + + def _get_messages(self): + return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True) + + def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None): + obj = ail_objects.get_obj_from_global_id(obj_global_id) + mess_dict = obj.get_meta(options={'content', 'link', 'parent'}) + if mess_dict.get('parent') and parent: + mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False) + mess_dict['username'] = {} + user = obj.get_correlation('username').get('username') + if user: + subtype, user = user.pop().split(':', 1) + mess_dict['username']['type'] = 'telegram' + mess_dict['username']['subtype'] = subtype + mess_dict['username']['id'] = user + else: + mess_dict['username']['id'] = 'UNKNOWN' + + if not mess_datetime: + obj_mess_id = self._get_message_timestamp(obj_global_id) + mess_datetime = datetime.fromtimestamp(obj_mess_id) + mess_dict['date'] = mess_datetime.isoformat(' ') + mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S') + return mess_dict + + + def get_messages(self, start=0, page=1, nb=500): # TODO limit nb returned, # TODO add replies + start = 0 + stop = -1 + # r_object.delete(f'messages:{self.type}:{self.subtype}:{self.id}') + + # TODO chat without username ???? -> chat ID ???? + + messages = {} + curr_date = None + for message in self._get_messages(): + date = datetime.fromtimestamp(message[1]) + date_day = date.strftime('%Y/%m/%d') + if date_day != curr_date: + messages[date_day] = [] + curr_date = date_day + mess_dict = self.get_message_meta(message[0], parent=True, mess_datetime=date) + messages[date_day].append(mess_dict) + return messages + + # Zset with ID ??? id -> item id ??? multiple id == media + text + # id -> media id + # How do we handle reply/thread ??? -> separate with new chats name/id ZSET ??? + # Handle media ??? + + # list of message id -> obj_id + # list of obj_id -> + # abuse parent children ??? + + # def add(self, timestamp, obj_id, mess_id=0, username=None, user_id=None): + # date = # TODO get date from object + # self.update_daterange(date) + # update_obj_date(date, self.type, self.subtype) + # + # + # # daily + # r_object.hincrby(f'{self.type}:{self.subtype}:{date}', self.id, 1) + # # all subtypes + # r_object.zincrby(f'{self.type}_all:{self.subtype}', 1, self.id) + # + # ####################################################################### + # ####################################################################### + # + # # Correlations + # self.add_correlation('item', '', item_id) + # # domain + # if is_crawled(item_id): + # domain = get_item_domain(item_id) + # self.add_correlation('domain', '', domain) + + # TODO kvrocks exception if key don't exists + def get_obj_by_message_id(self, mess_id): + return r_object.hget(f'messages:ids:{self.type}:{self.subtype}:{self.id}', mess_id) + + # importer -> use cache for previous reply SET to_add_id: previously_imported : expire SET key -> 30 mn + def add_message(self, obj_global_id, timestamp, mess_id, reply_id=None): + r_object.hset(f'messages:ids:{self.type}:{self.subtype}:{self.id}', mess_id, obj_global_id) + r_object.zadd(f'messages:{self.type}:{self.subtype}:{self.id}', {obj_global_id: timestamp}) + + if reply_id: + reply_obj = self.get_obj_by_message_id(reply_id) + if reply_obj: + self.add_obj_children(reply_obj, obj_global_id) + else: + self.add_message_cached_reply(reply_id, mess_id) + + # ADD cached replies + for reply_obj in self.get_cached_message_reply(mess_id): + self.add_obj_children(obj_global_id, reply_obj) + + def _get_message_cached_reply(self, message_id): + return r_cache.smembers(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{message_id}') + + def get_cached_message_reply(self, message_id): + objs_global_id = [] + for mess_id in self._get_message_cached_reply(message_id): + obj_global_id = self.get_obj_by_message_id(mess_id) + if obj_global_id: + objs_global_id.append(obj_global_id) + return objs_global_id + + def add_message_cached_reply(self, reply_to_id, message_id): + r_cache.sadd(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_to_id}', message_id) + r_cache.expire(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_to_id}', 600) + + # TODO nb replies = nb son ???? what if it create a onion item ??? -> need source filtering + + +# TODO factorize +def get_all_subtypes(): + return ail_core.get_object_all_subtypes('chat') + +def get_all(): + objs = {} + for subtype in get_all_subtypes(): + objs[subtype] = get_all_by_subtype(subtype) + return objs + +def get_all_by_subtype(subtype): + return get_all_id('chat', subtype) + +# # TODO FILTER NAME + Key + mail +# def sanitize_username_name_to_search(name_to_search, subtype): # TODO FILTER NAME +# +# return name_to_search +# +# def search_usernames_by_name(name_to_search, subtype, r_pos=False): +# usernames = {} +# # for subtype in subtypes: +# r_name = sanitize_username_name_to_search(name_to_search, subtype) +# if not name_to_search or isinstance(r_name, dict): +# # break +# return usernames +# r_name = re.compile(r_name) +# for user_name in get_all_usernames_by_subtype(subtype): +# res = re.search(r_name, user_name) +# if res: +# usernames[user_name] = {} +# if r_pos: +# usernames[user_name]['hl-start'] = res.start() +# usernames[user_name]['hl-end'] = res.end() +# return usernames + + +if __name__ == '__main__': + chat = Chat('test', 'telegram') + r = chat.get_messages() + print(r) diff --git a/bin/lib/objects/Items.py b/bin/lib/objects/Items.py index 03c6f2cd..c2edbb40 100755 --- a/bin/lib/objects/Items.py +++ b/bin/lib/objects/Items.py @@ -288,6 +288,8 @@ class Item(AbstractObject): meta['mimetype'] = self.get_mimetype(content=content) if 'investigations' in options: meta['investigations'] = self.get_investigations() + if 'link' in options: + meta['link'] = self.get_link(flask_context=True) # meta['encoding'] = None return meta diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py new file mode 100755 index 00000000..98cc838f --- /dev/null +++ b/bin/lib/objects/Messages.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import re +import sys +import cld3 +import html2text + +from datetime import datetime + +from pymisp import MISPObject + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ail_core import get_ail_uuid +from lib.objects.abstract_object import AbstractObject +from lib.ConfigLoader import ConfigLoader +from lib.data_retention_engine import update_obj_date, get_obj_date_first +# TODO Set all messages ??? + + +from flask import url_for + +config_loader = ConfigLoader() +r_cache = config_loader.get_redis_conn("Redis_Cache") +r_object = config_loader.get_db_conn("Kvrocks_Objects") +r_content = config_loader.get_db_conn("Kvrocks_Content") +baseurl = config_loader.get_config_str("Notifications", "ail_domain") +config_loader = None + + +# TODO SAVE OR EXTRACT MESSAGE SOURCE FOR ICON ????????? +# TODO iterate on all objects +# TODO also add support for small objects ???? + +# CAN Message exists without CHAT -> no convert it to object + +# ID: source:chat_id:message_id ???? +# +# /!\ handle null chat and message id -> chat = uuid and message = timestamp ??? + + +class Message(AbstractObject): + """ + AIL Message Object. (strings) + """ + + def __init__(self, id): # TODO subtype or use source ???? + super(Message, self).__init__('message', id) # message::< telegram/1692189934.380827/ChatID_MessageID > + + def exists(self): + if self.subtype is None: + return r_object.exists(f'meta:{self.type}:{self.id}') + else: + return r_object.exists(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}') + + def get_source(self): + """ + Returns source/feeder name + """ + l_source = self.id.split('/')[:-4] + return os.path.join(*l_source) + + def get_basename(self): + return os.path.basename(self.id) + + def get_content(self, r_type='str'): # TODO ADD cache # TODO Compress content ??????? + """ + Returns content + """ + content = self._get_field('content') + if r_type == 'str': + return content + elif r_type == 'bytes': + return content.encode() + + def get_date(self): + timestamp = self.get_timestamp() + return datetime.fromtimestamp(timestamp).strftime('%Y%m%d') + + def get_timestamp(self): + dirs = self.id.split('/') + return dirs[-2] + + def get_message_id(self): # TODO optimize + message_id = self.get_basename().rsplit('_', 1)[1] + # if message_id.endswith('.gz'): + # message_id = message_id[:-3] + return message_id + + def get_chat_id(self): # TODO optimize + chat_id = self.get_basename().rsplit('_', 1)[0] + # if chat_id.endswith('.gz'): + # chat_id = chat_id[:-3] + return chat_id + + # Update value on import + # reply to -> parent ? + # reply/comment - > children ? + # nb views + # reactions + # nb fowards + # room ??? + # message from channel ??? + # message media + + def get_translation(self): # TODO support multiple translated languages ????? + """ + Returns translated content + """ + return self._get_field('translated') # TODO multiples translation ... -> use set + + def _set_translation(self, translation): + """ + Set translated content + """ + return self._set_field('translated', translation) # translation by hash ??? -> avoid translating multiple time + + def get_html2text_content(self, content=None, ignore_links=False): + if not content: + content = self.get_content() + h = html2text.HTML2Text() + h.ignore_links = ignore_links + h.ignore_images = ignore_links + return h.handle(content) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True)} + # return payload + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', type=self.type, id=self.id) + else: + url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}' + return url + + def get_svg_icon(self): + return {'style': 'fas', 'icon': 'fa-comment-dots', 'color': '#4dffff', 'radius': 5} + + def get_misp_object(self): # TODO + obj = MISPObject('instant-message', standalone=True) + obj_date = self.get_date() + if obj_date: + obj.first_seen = obj_date + else: + self.logger.warning( + f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={obj_date}') + + # obj_attrs = [obj.add_attribute('first-seen', value=obj_date), + # obj.add_attribute('raw-data', value=self.id, data=self.get_raw_content()), + # obj.add_attribute('sensor', value=get_ail_uuid())] + obj_attrs = [] + for obj_attr in obj_attrs: + for tag in self.get_tags(): + obj_attr.add_tag(tag) + return obj + + # def get_url(self): + # return r_object.hget(f'meta:item::{self.id}', 'url') + + # options: set of optional meta fields + def get_meta(self, options=None): + """ + :type options: set + """ + if options is None: + options = set() + meta = self.get_default_meta(tags=True) + meta['date'] = self.get_date() # TODO replace me by timestamp ?????? + meta['source'] = self.get_source() + # optional meta fields + if 'content' in options: + meta['content'] = self.get_content() + if 'parent' in options: + meta['parent'] = self.get_parent() + if 'investigations' in options: + meta['investigations'] = self.get_investigations() + if 'link' in options: + meta['link'] = self.get_link(flask_context=True) + + # meta['encoding'] = None + return meta + + def _languages_cleaner(self, content=None): + if not content: + content = self.get_content() + # REMOVE URLS + regex = r'\b(?:http://|https://)?(?:[a-zA-Z\d-]{,63}(?:\.[a-zA-Z\d-]{,63})+)(?:\:[0-9]+)*(?:/(?:$|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*\b' + url_regex = re.compile(regex) + urls = url_regex.findall(content) + urls = sorted(urls, key=len, reverse=True) + for url in urls: + content = content.replace(url, '') + # REMOVE PGP Blocks + regex_pgp_public_blocs = r'-----BEGIN PGP PUBLIC KEY BLOCK-----[\s\S]+?-----END PGP PUBLIC KEY BLOCK-----' + regex_pgp_signature = r'-----BEGIN PGP SIGNATURE-----[\s\S]+?-----END PGP SIGNATURE-----' + regex_pgp_message = r'-----BEGIN PGP MESSAGE-----[\s\S]+?-----END PGP MESSAGE-----' + re.compile(regex_pgp_public_blocs) + re.compile(regex_pgp_signature) + re.compile(regex_pgp_message) + res = re.findall(regex_pgp_public_blocs, content) + for it in res: + content = content.replace(it, '') + res = re.findall(regex_pgp_signature, content) + for it in res: + content = content.replace(it, '') + res = re.findall(regex_pgp_message, content) + for it in res: + content = content.replace(it, '') + return content + + def detect_languages(self, min_len=600, num_langs=3, min_proportion=0.2, min_probability=0.7): + languages = [] + ## CLEAN CONTENT ## + content = self.get_html2text_content(ignore_links=True) + content = self._languages_cleaner(content=content) + # REMOVE USELESS SPACE + content = ' '.join(content.split()) + # - CLEAN CONTENT - # + if len(content) >= min_len: + for lang in cld3.get_frequent_languages(content, num_langs=num_langs): + if lang.proportion >= min_proportion and lang.probability >= min_probability and lang.is_reliable: + languages.append(lang) + return languages + + # def translate(self, content=None): # TODO translation plugin + # # TODO get text language + # if not content: + # content = self.get_content() + # translated = argostranslate.translate.translate(content, 'ru', 'en') + # # Save translation + # self._set_translation(translated) + # return translated + + def create(self, content, translation, tags): + self._set_field('content', content) + r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content) + if translation: + self._set_translation(translation) + for tag in tags: + self.add_tag(tag) + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + pass + +def create_obj_id(source, chat_id, message_id, timestamp): + return f'{source}/{timestamp}/{chat_id}_{message_id}' + +# TODO Check if already exists +# def create(source, chat_id, message_id, timestamp, content, tags=[]): +def create(obj_id, content, translation=None, tags=[]): + message = Message(obj_id) + if not message.exists(): + message.create(content, translation, tags) + return message + + +# TODO Encode translation + + +if __name__ == '__main__': + r = 'test' + print(r) diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py new file mode 100755 index 00000000..0355806e --- /dev/null +++ b/bin/lib/objects/UsersAccount.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import re + +from flask import url_for +from pymisp import MISPObject + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import ail_core +from lib.ConfigLoader import ConfigLoader +from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id + +config_loader = ConfigLoader() +baseurl = config_loader.get_config_str("Notifications", "ail_domain") +config_loader = None + + +################################################################################ +################################################################################ +################################################################################ + +class UserAccount(AbstractSubtypeObject): + """ + AIL User Object. (strings) + """ + + def __init__(self, id, subtype): + super(UserAccount, self).__init__('user-account', id, subtype) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True), + # 'compress': 'gzip'} + # return payload + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + # # TODO: + pass + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id) + else: + url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}' + return url + + def get_svg_icon(self): # TODO change icon/color + if self.subtype == 'telegram': + style = 'fab' + icon = '\uf2c6' + elif self.subtype == 'twitter': + style = 'fab' + icon = '\uf099' + else: + style = 'fas' + icon = '\uf007' + return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5} + + def get_first_name(self): + return self._get_field('firstname') + + def get_last_name(self): + return self._get_field('lastname') + + def get_phone(self): + return self._get_field('phone') + + def set_first_name(self, firstname): + return self._set_field('firstname', firstname) + + def set_last_name(self, lastname): + return self._set_field('lastname', lastname) + + def set_phone(self, phone): + return self._set_field('phone', phone) + + # TODO REWRITE ADD FUNCTION + + def get_username(self): + return '' + + def get_usernames(self): + usernames = [] + correl = self.get_correlation('username') + for partial_id in correl.get('username', []): + usernames.append(f'username:{partial_id}') + return usernames + + def get_meta(self, options=set()): + meta = self._get_meta(options=options) + meta['id'] = self.id + meta['subtype'] = self.subtype + meta['tags'] = self.get_tags(r_list=True) + if 'username' in options: + meta['username'] = self.get_username() + if 'usernames' in options: + meta['usernames'] = self.get_usernames() + return meta + + def get_misp_object(self): + obj_attrs = [] + if self.subtype == 'telegram': + obj = MISPObject('telegram-account', standalone=True) + obj_attrs.append(obj.add_attribute('username', value=self.id)) + + elif self.subtype == 'twitter': + obj = MISPObject('twitter-account', standalone=True) + obj_attrs.append(obj.add_attribute('name', value=self.id)) + + else: + obj = MISPObject('user-account', standalone=True) + obj_attrs.append(obj.add_attribute('username', value=self.id)) + + first_seen = self.get_first_seen() + last_seen = self.get_last_seen() + if first_seen: + obj.first_seen = first_seen + if last_seen: + obj.last_seen = last_seen + if not first_seen or not last_seen: + self.logger.warning( + f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}') + + for obj_attr in obj_attrs: + for tag in self.get_tags(): + obj_attr.add_tag(tag) + return obj + +def get_user_by_username(): + pass + +def get_all_subtypes(): + return ail_core.get_object_all_subtypes('user-account') + +def get_all(): + users = {} + for subtype in get_all_subtypes(): + users[subtype] = get_all_by_subtype(subtype) + return users + +def get_all_by_subtype(subtype): + return get_all_id('user-account', subtype) + + +# if __name__ == '__main__': +# name_to_search = 'co' +# subtype = 'telegram' +# print(search_usernames_by_name(name_to_search, subtype)) diff --git a/bin/lib/objects/abstract_daterange_object.py b/bin/lib/objects/abstract_daterange_object.py index 5ec103d0..98aa49c2 100755 --- a/bin/lib/objects/abstract_daterange_object.py +++ b/bin/lib/objects/abstract_daterange_object.py @@ -45,10 +45,10 @@ class AbstractDaterangeObject(AbstractObject, ABC): def exists(self): return r_object.exists(f'meta:{self.type}:{self.id}') - def _get_field(self, field): + def _get_field(self, field): # TODO remove me (NEW in abstract) return r_object.hget(f'meta:{self.type}:{self.id}', field) - def _set_field(self, field, value): + def _set_field(self, field, value): # TODO remove me (NEW in abstract) return r_object.hset(f'meta:{self.type}:{self.id}', field, value) def get_first_seen(self, r_int=False): diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index 2423a294..59a7e968 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -20,6 +20,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from lib import ail_logger from lib import Tag +from lib.ConfigLoader import ConfigLoader from lib import Duplicate from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations @@ -27,6 +28,11 @@ from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers logging.config.dictConfig(ail_logger.get_config(name='ail')) +config_loader = ConfigLoader() +# r_cache = config_loader.get_redis_conn("Redis_Cache") +r_object = config_loader.get_db_conn("Kvrocks_Objects") +config_loader = None + class AbstractObject(ABC): """ Abstract Object @@ -67,6 +73,18 @@ class AbstractObject(ABC): dict_meta['tags'] = self.get_tags() return dict_meta + def _get_field(self, field): + if self.subtype is None: + return r_object.hget(f'meta:{self.type}:{self.id}', field) + else: + return r_object.hget(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', field) + + def _set_field(self, field, value): + if self.subtype is None: + return r_object.hset(f'meta:{self.type}:{self.id}', field, value) + else: + return r_object.hset(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', field, value) + ## Tags ## def get_tags(self, r_list=False): tags = Tag.get_object_tags(self.type, self.id, self.get_subtype(r_str=True)) @@ -198,6 +216,8 @@ class AbstractObject(ABC): else: return [] + ## Correlation ## + def _get_external_correlation(self, req_type, req_subtype, req_id, obj_type): """ Get object correlation @@ -253,3 +273,39 @@ class AbstractObject(ABC): Get object correlations """ delete_obj_correlation(self.type, self.subtype, self.id, type2, subtype2, id2) + + ## -Correlation- ## + + ## Parent ## + + def is_parent(self): + return r_object.exists(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}') + + def is_children(self): + return r_object.hexists(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent') + + def get_parent(self): + return r_object.hget(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent') + + def get_children(self): + return r_object.smembers(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}') + + def set_parent(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO ###################### + if not obj_global_id: + if obj_subtype is None: + obj_subtype = '' + obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}' + r_object.hset(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent', obj_global_id) + + def add_children(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO ###################### + if not obj_global_id: + if obj_subtype is None: + obj_subtype = '' + obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}' + r_object.sadd(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', obj_global_id) + + def add_obj_children(self, parent_global_id, son_global_id): + r_object.sadd(f'child:{parent_global_id}', son_global_id) + r_object.hset(f'meta:{son_global_id}', 'parent', parent_global_id) + + ## Parent ## diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index f12708fb..cd2f7225 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -13,6 +13,7 @@ from lib import correlations_engine from lib import btc_ail from lib import Tag +from lib.objects import Chats from lib.objects import CryptoCurrencies from lib.objects import CookiesNames from lib.objects.Cves import Cve @@ -55,6 +56,8 @@ def get_object(obj_type, subtype, id): return Domain(id) elif obj_type == 'decoded': return Decoded(id) + elif obj_type == 'chat': + return Chats.Chat(id, subtype) elif obj_type == 'cookie-name': return CookiesNames.CookieName(id) elif obj_type == 'cve': diff --git a/bin/lib/timeline_engine.py b/bin/lib/timeline_engine.py new file mode 100755 index 00000000..405e7a50 --- /dev/null +++ b/bin/lib/timeline_engine.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys + +from uuid import uuid4 + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader + +config_loader = ConfigLoader() +r_meta = config_loader.get_db_conn("Kvrocks_Timeline") +config_loader = None + +# CORRELATION_TYPES_BY_OBJ = { +# "chat": ["item", "username"], # item ??? +# "cookie-name": ["domain"], +# "cryptocurrency": ["domain", "item"], +# "cve": ["domain", "item"], +# "decoded": ["domain", "item"], +# "domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"], +# "etag": ["domain"], +# "favicon": ["domain", "item"], +# "hhhash": ["domain"], +# "item": ["chat", "cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], +# "pgp": ["domain", "item"], +# "screenshot": ["domain", "item"], +# "title": ["domain", "item"], +# "username": ["chat", "domain", "item"], +# } +# +# def get_obj_correl_types(obj_type): +# return CORRELATION_TYPES_BY_OBJ.get(obj_type) + +# def sanityze_obj_correl_types(obj_type, correl_types): +# obj_correl_types = get_obj_correl_types(obj_type) +# if correl_types: +# correl_types = set(correl_types).intersection(obj_correl_types) +# if not correl_types: +# correl_types = obj_correl_types +# if not correl_types: +# return [] +# return correl_types + +# TODO rename all function + add missing parameters + +def get_bloc_obj_global_id(bloc): + return r_meta.hget('hset:key', bloc) + +def set_bloc_obj_global_id(bloc, global_id): + return r_meta.hset('hset:key', bloc, global_id) + +def get_bloc_timestamp(bloc, position): + return r_meta.zscore('key', f'{position}:{bloc}') + +def add_bloc(global_id, timestamp, end=None): + if end: + timestamp_end = end + else: + timestamp_end = timestamp + new_bloc = str(uuid4()) + r_meta.zadd('key', {f'start:{new_bloc}': timestamp, f'end:{new_bloc}': timestamp_end}) + set_bloc_obj_global_id(new_bloc, global_id) + return new_bloc + +def _update_bloc(bloc, position, timestamp): + r_meta.zadd('key', {f'{position}:{bloc}': timestamp}) + +# score = timestamp +def get_nearest_bloc_inf(timestamp): + return r_meta.zrevrangebyscore('key', timestamp, 0, num=1) + +def get_nearest_bloc_sup(timestamp): + return r_meta.zrangebyscore('key', timestamp, 0, num=1) + +####################################################################################### + +def add_timestamp(timestamp, obj_global_id): + inf = get_nearest_bloc_inf(timestamp) + sup = get_nearest_bloc_sup(timestamp) + if not inf and not sup: + # create new bloc + new_bloc = add_bloc(obj_global_id, timestamp) + return new_bloc + # timestamp < first_seen + elif not inf: + sup_pos, sup_id = inf.split(':') + sup_obj = get_bloc_obj_global_id(sup_pos) + if sup_obj == obj_global_id: + _update_bloc(sup_id, 'start', timestamp) + # create new bloc + else: + new_bloc = add_bloc(obj_global_id, timestamp) + return new_bloc + + # timestamp > first_seen + elif not sup: + inf_pos, inf_id = inf.split(':') + inf_obj = get_bloc_obj_global_id(inf_id) + if inf_obj == obj_global_id: + _update_bloc(inf_id, 'end', timestamp) + # create new bloc + else: + new_bloc = add_bloc(obj_global_id, timestamp) + return new_bloc + + else: + inf_pos, inf_id = inf.split(':') + sup_pos, sup_id = inf.split(':') + inf_obj = get_bloc_obj_global_id(inf_id) + + if inf_id == sup_id: + # reduce bloc + create two new bloc + if obj_global_id != inf_obj: + # get end timestamp + sup_timestamp = get_bloc_timestamp(sup_id, 'end') + # reduce original bloc + _update_bloc(inf_id, 'end', timestamp - 1) + # Insert new bloc + new_bloc = add_bloc(obj_global_id, timestamp) + # Recreate end of the first bloc by a new bloc + add_bloc(inf_obj, timestamp + 1, end=sup_timestamp) + return new_bloc + + # timestamp in existing bloc + else: + return inf_id + + # different blocs: expend sup/inf bloc or create a new bloc if + elif inf_pos == 'end' and sup_pos == 'start': + # Extend inf bloc + if obj_global_id == inf_obj: + _update_bloc(inf_id, 'end', timestamp) + return inf_id + + sup_obj = get_bloc_obj_global_id(sup_pos) + # Extend sup bloc + if obj_global_id == sup_obj: + _update_bloc(sup_id, 'start', timestamp) + return sup_id + + # create new bloc + new_bloc = add_bloc(obj_global_id, timestamp) + return new_bloc + + # inf_pos == 'start' and sup_pos == 'end' + # else raise error ??? + + + + + + diff --git a/configs/6383.conf b/configs/6383.conf index c730003c..a06d4e69 100644 --- a/configs/6383.conf +++ b/configs/6383.conf @@ -663,6 +663,7 @@ namespace.crawl ail_crawlers namespace.db ail_datas namespace.dup ail_dups namespace.obj ail_objs +namespace.tl ail_tls namespace.stat ail_stats namespace.tag ail_tags namespace.track ail_trackers diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index 3278033f..8185b8f7 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -190,6 +190,11 @@ host = localhost port = 6383 password = ail_objs +[Kvrocks_Timeline] +host = localhost +port = 6383 +password = ail_tls + [Kvrocks_Stats] host = localhost port = 6383 diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index e6a99350..c330443b 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -50,6 +50,7 @@ from blueprints.objects_title import objects_title from blueprints.objects_cookie_name import objects_cookie_name from blueprints.objects_etag import objects_etag from blueprints.objects_hhhash import objects_hhhash +from blueprints.objects_chat import objects_chat Flask_dir = os.environ['AIL_FLASK'] @@ -107,6 +108,7 @@ app.register_blueprint(objects_title, url_prefix=baseUrl) app.register_blueprint(objects_cookie_name, url_prefix=baseUrl) app.register_blueprint(objects_etag, url_prefix=baseUrl) app.register_blueprint(objects_hhhash, url_prefix=baseUrl) +app.register_blueprint(objects_chat, url_prefix=baseUrl) # ========= =========# diff --git a/var/www/blueprints/objects_chat.py b/var/www/blueprints/objects_chat.py new file mode 100644 index 00000000..8a1db11f --- /dev/null +++ b/var/www/blueprints/objects_chat.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... +''' + +import os +import sys +import json + +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file +from flask_login import login_required, current_user + +# Import Role_Manager +from Role_Manager import login_admin, login_analyst, login_read_only + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import ail_core +from lib.objects import abstract_subtype_object +from lib.objects import ail_objects +from lib.objects import Chats +from packages import Date + +# ============ BLUEPRINT ============ +objects_chat = Blueprint('objects_chat', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/chat')) + +# ============ VARIABLES ============ +bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] + +def create_json_response(data, status_code): + return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code + +# ============ FUNCTIONS ============ + +# ============= ROUTES ============== + + +@objects_chat.route("/objects/chat/messages", methods=['GET']) +@login_required +@login_read_only +def objects_dashboard_chat(): + chat = request.args.get('id') + subtype = request.args.get('subtype') + chat = Chats.Chat(chat, subtype) + if chat.exists(): + messages = chat.get_messages() + meta = chat.get_meta({'icon'}) + print(meta) + return render_template('ChatMessages.html', meta=meta, messages=messages, bootstrap_label=bootstrap_label) + else: + return abort(404) + + + diff --git a/var/www/blueprints/objects_subtypes.py b/var/www/blueprints/objects_subtypes.py index dc97ffa8..a41066a4 100644 --- a/var/www/blueprints/objects_subtypes.py +++ b/var/www/blueprints/objects_subtypes.py @@ -91,6 +91,12 @@ def subtypes_objects_dashboard(obj_type, f_request): # ============= ROUTES ============== +@objects_subtypes.route("/objects/chats", methods=['GET']) +@login_required +@login_read_only +def objects_dashboard_chat(): + return subtypes_objects_dashboard('chat', request) + @objects_subtypes.route("/objects/cryptocurrencies", methods=['GET']) @login_required @login_read_only diff --git a/var/www/templates/objects/chat/ChatMessages.html b/var/www/templates/objects/chat/ChatMessages.html new file mode 100644 index 00000000..b89a447a --- /dev/null +++ b/var/www/templates/objects/chat/ChatMessages.html @@ -0,0 +1,190 @@ + + + + + Chat Messages - AIL + + + + + + +{# #} + + + + + + + +{# + #} + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + +
    +
    +

    {{ meta["id"] }} :

    +
      +
    • +
      +
      + + + + + + + + + + + + + + + + + +
      Object subtypeFirst seenLast seenNb seen
      + + + + {{ meta["icon"]["icon"] }} + + + {{ meta["subtype"] }} + {{ meta['first_seen'] }}{{ meta['last_seen'] }}{{ meta['nb_seen'] }}
      +
      +
      +
      +
      +
      +
      • +
    • +
      +
      + Tags: + {% for tag in meta['tags'] %} + + {% endfor %} + +
      +
      • +
    + + {% with obj_type='chat', obj_id=meta['id'], obj_subtype=meta['subtype'] %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + +
    +
    + +
    +
    + + {% for date in messages %} +

    {{ date }}

    + {% for mess in messages[date] %} + +
    +
    + {{ mess['username']['id'] }} +
    {{ mess['hour'] }}
    +
    +
    +
    {{ mess['username']['id'] }}
    + {% if mess['reply_to'] %} +
    +
    {{ mess['reply_to']['username']['id'] }}
    + 
    {{ mess['reply_to']['content'] }}
    + {% for tag in mess['reply_to']['tags'] %} + {{ tag }} + {% endfor %} +
    {{ mess['reply_to']['date'] }}
    +{#
    #} +{# #} +{# #} +{#
    #} +
    + {% endif %} + 
    {{ mess['content'] }}
    + {% for tag in mess['tags'] %} + {{ tag }} + {% endfor %} +
    + + +
    +
    +
    + + {% endfor %} +
    + {% endfor %} + +
    +
    + +
    + +
    +
    + + + + + + From 0cb7431e10388439877aa5c5c269f27b7eae8157 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 21 Aug 2023 15:49:32 +0200 Subject: [PATCH 093/238] chg: [modules] crawl pasties domains --- bin/lib/ConfigLoader.py | 1 + bin/lib/regex_helper.py | 28 +++++++ bin/modules/Pasties.py | 144 +++++++++++++++++++++++++++++++++ bin/modules/Zerobins.py | 71 ---------------- bin/modules/abstract_module.py | 3 + configs/modules.cfg | 2 +- 6 files changed, 177 insertions(+), 72 deletions(-) create mode 100755 bin/modules/Pasties.py delete mode 100755 bin/modules/Zerobins.py diff --git a/bin/lib/ConfigLoader.py b/bin/lib/ConfigLoader.py index 5be8f492..6ecd4b02 100755 --- a/bin/lib/ConfigLoader.py +++ b/bin/lib/ConfigLoader.py @@ -83,6 +83,7 @@ class ConfigLoader(object): else: return [] + # # # # Directory Config # # # # config_loader = ConfigLoader() diff --git a/bin/lib/regex_helper.py b/bin/lib/regex_helper.py index 41ba4e98..6f877823 100755 --- a/bin/lib/regex_helper.py +++ b/bin/lib/regex_helper.py @@ -113,6 +113,34 @@ def regex_finditer(r_key, regex, item_id, content, max_time=30): proc.terminate() sys.exit(0) +def _regex_match(r_key, regex, content): + if re.match(regex, content): + r_serv_cache.set(r_key, 1) + r_serv_cache.expire(r_key, 360) + +def regex_match(r_key, regex, item_id, content, max_time=30): + proc = Proc(target=_regex_match, args=(r_key, regex, content)) + try: + proc.start() + proc.join(max_time) + if proc.is_alive(): + proc.terminate() + # Statistics.incr_module_timeout_statistic(r_key) + err_mess = f"{r_key}: processing timeout: {item_id}" + logger.info(err_mess) + return False + else: + if r_serv_cache.exists(r_key): + r_serv_cache.delete(r_key) + return True + else: + r_serv_cache.delete(r_key) + return False + except KeyboardInterrupt: + print("Caught KeyboardInterrupt, terminating regex worker") + proc.terminate() + sys.exit(0) + def _regex_search(r_key, regex, content): if re.search(regex, content): r_serv_cache.set(r_key, 1) diff --git a/bin/modules/Pasties.py b/bin/modules/Pasties.py new file mode 100755 index 00000000..ce2eff10 --- /dev/null +++ b/bin/modules/Pasties.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +The Pasties Module +====================== +This module spots domain-pasties services for further processing +""" + +################################## +# Import External packages +################################## +import os +import sys +import time + +from pyfaup.faup import Faup + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from modules.abstract_module import AbstractModule +from lib.ConfigLoader import ConfigLoader +from lib import crawlers + +# TODO add url validator + +pasties_blocklist_urls = set() +pasties_domains = {} + +class Pasties(AbstractModule): + """ + Pasties module for AIL framework + """ + + def __init__(self): + super(Pasties, self).__init__() + self.faup = Faup() + + config_loader = ConfigLoader() + self.r_cache = config_loader.get_redis_conn("Redis_Cache") + + self.pasties = {} + self.urls_blocklist = set() + self.load_pasties_domains() + + # Send module state to logs + self.logger.info(f'Module {self.module_name} initialized') + + def load_pasties_domains(self): + self.pasties = {} + self.urls_blocklist = set() + + domains_pasties = os.path.join(os.environ['AIL_HOME'], 'files/domains_pasties') + if os.path.exists(domains_pasties): + with open(domains_pasties) as f: + for line in f: + url = line.strip() + if url: # TODO validate line + self.faup.decode(url) + url_decoded = self.faup.get() + host = url_decoded['host'] + # if url_decoded.get('port', ''): + # host = f'{host}:{url_decoded["port"]}' + path = url_decoded.get('resource_path', '') + # print(url_decoded) + if path and path != '/': + if path[-1] != '/': + path = f'{path}/' + else: + path = None + + if host in self.pasties: + if path: + self.pasties[host].add(path) + else: + if path: + self.pasties[host] = {path} + else: + self.pasties[host] = set() + + url_blocklist = os.path.join(os.environ['AIL_HOME'], 'files/domains_pasties_blacklist') + if os.path.exists(url_blocklist): + with open(url_blocklist) as f: + for line in f: + url = line.strip() + self.faup.decode(url) + url_decoded = self.faup.get() + host = url_decoded['host'] + # if url_decoded.get('port', ''): + # host = f'{host}:{url_decoded["port"]}' + path = url_decoded.get('resource_path', '') + url = f'{host}{path}' + if url_decoded['query_string']: + url = url + url_decoded['query_string'] + self.urls_blocklist.add(url) + + def send_to_crawler(self, url, obj_id): + if not self.r_cache.exists(f'{self.module_name}:url:{url}'): + self.r_cache.set(f'{self.module_name}:url:{url}', int(time.time())) + self.r_cache.expire(f'{self.module_name}:url:{url}', 86400) + crawlers.create_task(url, depth=0, har=False, screenshot=False, proxy='force_tor', priority=60, parent=obj_id) + + def compute(self, message): + url, item_id = message.split() + + self.faup.decode(url) + url_decoded = self.faup.get() + # print(url_decoded) + url_host = url_decoded['host'] + # if url_decoded.get('port', ''): + # url_host = f'{url_host}:{url_decoded["port"]}' + path = url_decoded.get('resource_path', '') + if url_host in self.pasties: + if url.startswith('http://'): + if url[7:] in self.urls_blocklist: + return None + elif url.startswith('https://'): + if url[8:] in self.urls_blocklist: + return None + else: + if url in self.urls_blocklist: + return None + + if not self.pasties[url_host]: + if path and path != '/': + print('send to crawler', url_host, url) + self.send_to_crawler(url, item_id) + else: + if path.endswith('/'): + path_end = path[:-1] + else: + path_end = f'{path}/' + for url_path in self.pasties[url_host]: + if path.startswith(url_path): + if url_path != path and url_path != path_end: + print('send to crawler', url_path, url) + self.send_to_crawler(url, item_id) + break + + +if __name__ == '__main__': + module = Pasties() + module.run() diff --git a/bin/modules/Zerobins.py b/bin/modules/Zerobins.py deleted file mode 100755 index f3fcea5a..00000000 --- a/bin/modules/Zerobins.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* -""" -The Zerobins Module -====================== -This module spots zerobins-like services for further processing -""" - -################################## -# Import External packages -################################## -import os -import re -import sys - -sys.path.append(os.environ['AIL_BIN']) -################################## -# Import Project packages -################################## -from modules.abstract_module import AbstractModule -from lib import crawlers - - -class Zerobins(AbstractModule): - """ - Zerobins module for AIL framework - """ - - def __init__(self): - super(Zerobins, self).__init__() - - binz = [ - r'^https:\/\/(zerobin||privatebin)\..*$', # historical ones - ] - - self.regex = re.compile('|'.join(binz)) - - # Pending time between two computation (computeNone) in seconds - self.pending_seconds = 10 - - # Send module state to logs - self.logger.info(f'Module {self.module_name} initialized') - - def computeNone(self): - """ - Compute when no message in queue - """ - self.logger.debug("No message in queue") - - def compute(self, message): - """ - Compute a message in queue - """ - url, item_id = message.split() - - # Extract zerobins addresses - matching_binz = self.regex_findall(self.regex, item_id, url) - - if len(matching_binz) > 0: - for bin_url in matching_binz: - print(f'send {bin_url} to crawler') - # TODO Change priority ??? - crawlers.create_task(bin_url, depth=0, har=False, screenshot=False, proxy='force_tor', - parent='manual', priority=60) - - self.logger.debug("Compute message in queue") - - -if __name__ == '__main__': - module = Zerobins() - module.run() diff --git a/bin/modules/abstract_module.py b/bin/modules/abstract_module.py index 0a1a12cd..164e77b3 100644 --- a/bin/modules/abstract_module.py +++ b/bin/modules/abstract_module.py @@ -92,6 +92,9 @@ class AbstractModule(ABC): def get_available_queues(self): return self.queue.get_out_queues() + def regex_match(self, regex, obj_id, content): + return regex_helper.regex_match(self.r_cache_key, regex, obj_id, content, max_time=self.max_execution_time) + def regex_search(self, regex, obj_id, content): return regex_helper.regex_search(self.r_cache_key, regex, obj_id, content, max_time=self.max_execution_time) diff --git a/configs/modules.cfg b/configs/modules.cfg index b0b1f6df..3ce4f0ae 100644 --- a/configs/modules.cfg +++ b/configs/modules.cfg @@ -162,7 +162,7 @@ publish = Importers,Tags subscribe = Item publish = Tags -[Zerobins] +[Pasties] subscribe = Url # [My_Module_Name] From 045aab6f3425ef9f3b2ca20cf69acbde6e0ae52e Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 21 Aug 2023 15:52:33 +0200 Subject: [PATCH 094/238] fix: [module pasties] fix module name --- bin/LAUNCH.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 00c224e4..39640a71 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -267,7 +267,7 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "LibInjection" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./LibInjection.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Zerobins" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Zerobins.py; read x" + screen -S "Script_AIL" -X screen -t "Pasties" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Pasties.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "MISP_Thehive_Auto_Push" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./MISP_Thehive_Auto_Push.py; read x" From f44c5509da842be5ec0756d042fad0d5d7d0a005 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 23 Aug 2023 11:16:22 +0200 Subject: [PATCH 095/238] chg: [titles] add yara tracker on title + tags domains if unsafe title tags --- bin/crawlers/Crawler.py | 9 +++++++++ bin/lib/Tracker.py | 4 ++-- bin/lib/ail_core.py | 2 +- bin/lib/objects/Titles.py | 3 ++- var/www/templates/hunter/tracker_add.html | 4 ++++ 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index 7f2c3df9..c22f6ccf 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -22,6 +22,7 @@ from lib.objects.Domains import Domain from lib.objects.Items import Item from lib.objects import Screenshots from lib.objects import Titles +from trackers.Tracker_Yara import Tracker_Yara logging.config.dictConfig(ail_logger.get_config(name='crawlers')) @@ -35,6 +36,8 @@ class Crawler(AbstractModule): # Waiting time in seconds between to message processed self.pending_seconds = 1 + self.tracker_yara = Tracker_Yara(queue=False) + config_loader = ConfigLoader() self.default_har = config_loader.get_config_boolean('Crawler', 'default_har') @@ -283,6 +286,12 @@ class Crawler(AbstractModule): if title_content: title = Titles.create_title(title_content) title.add(item.get_date(), item_id) + # Tracker + self.tracker_yara.compute(title.get_id(), obj_type=title.get_type()) + if not title.is_tags_safe(): + unsafe_tag = 'dark-web:topic="pornography-child-exploitation"' + self.domain.add_tag(unsafe_tag) + item.add_tag(unsafe_tag) # SCREENSHOT if self.screenshot: diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index f1ea8905..c06e303d 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -923,7 +923,7 @@ def api_add_tracker(dict_input, user_id): # Filters # TODO MOVE ME filters = dict_input.get('filters', {}) if filters: - if filters.keys() == {'decoded', 'item', 'pgp'} and set(filters['pgp'].get('subtypes', [])) == {'mail', 'name'}: + if filters.keys() == {'decoded', 'item', 'pgp', 'title'} and set(filters['pgp'].get('subtypes', [])) == {'mail', 'name'}: filters = {} for obj_type in filters: if obj_type not in get_objects_tracked(): @@ -998,7 +998,7 @@ def api_edit_tracker(dict_input, user_id): # Filters # TODO MOVE ME filters = dict_input.get('filters', {}) if filters: - if filters.keys() == {'decoded', 'item', 'pgp'} and set(filters['pgp'].get('subtypes', [])) == {'mail', 'name'}: + if filters.keys() == {'decoded', 'item', 'pgp', 'title'} and set(filters['pgp'].get('subtypes', [])) == {'mail', 'name'}: if not filters['decoded'] and not filters['item']: filters = {} for obj_type in filters: diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 75520a2b..9a7d9557 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -50,7 +50,7 @@ def get_object_all_subtypes(obj_type): return [] def get_objects_tracked(): - return ['decoded', 'item', 'pgp'] + return ['decoded', 'item', 'pgp', 'title'] def get_objects_retro_hunted(): return ['decoded', 'item'] diff --git a/bin/lib/objects/Titles.py b/bin/lib/objects/Titles.py index 9f88426c..1a29d58e 100755 --- a/bin/lib/objects/Titles.py +++ b/bin/lib/objects/Titles.py @@ -45,6 +45,8 @@ class Title(AbstractDaterangeObject): def get_content(self, r_type='str'): if r_type == 'str': return self._get_field('content') + elif r_type == 'bytes': + return self._get_field('content').encode() def get_link(self, flask_context=False): if flask_context: @@ -122,4 +124,3 @@ class Titles(AbstractDaterangeObjects): # # print(r) # r = titles.search_by_id('f7d57B', r_pos=True, case_sensitive=False) # print(r) - diff --git a/var/www/templates/hunter/tracker_add.html b/var/www/templates/hunter/tracker_add.html index 7cc690ba..05266fa4 100644 --- a/var/www/templates/hunter/tracker_add.html +++ b/var/www/templates/hunter/tracker_add.html @@ -132,6 +132,10 @@
    +
    + + +
    {#
    #} {# #} From 46c721590d83301b46999fed645ce16c1cfaff40 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 23 Aug 2023 11:21:22 +0200 Subject: [PATCH 096/238] fix: [tracker objs filter] fix title icon --- var/www/templates/hunter/tracker_add.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/templates/hunter/tracker_add.html b/var/www/templates/hunter/tracker_add.html index 05266fa4..4f8c6f3e 100644 --- a/var/www/templates/hunter/tracker_add.html +++ b/var/www/templates/hunter/tracker_add.html @@ -134,7 +134,7 @@
    - +
    {#
    #} From 2145eb7b8a89fafd4c7631a23f3de01bd1a87570 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 23 Aug 2023 11:46:37 +0200 Subject: [PATCH 097/238] fix: [title] fix None title --- bin/lib/crawlers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 3e61ed88..6e9132d2 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -234,7 +234,9 @@ def extract_title_from_html(html): soup = BeautifulSoup(html, 'html.parser') title = soup.title if title: - return str(title.string) + title = title.string + if title: + return str(title) return '' def extract_description_from_html(html): @@ -2022,4 +2024,4 @@ if __name__ == '__main__': # _reprocess_all_hars_cookie_name() # _reprocess_all_hars_etag() # _gzip_all_hars() - _reprocess_all_hars_hhhashs() + # _reprocess_all_hars_hhhashs() From 4e3784922c3dc420828f95cfe6afa63e772194c0 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 23 Aug 2023 11:47:39 +0200 Subject: [PATCH 098/238] fix: typo --- bin/lib/crawlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 6e9132d2..18b1eeac 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -2012,7 +2012,7 @@ def test_ail_crawlers(): # TODO MOVE ME IN CRAWLER OR FLASK load_blacklist() -if __name__ == '__main__': +# if __name__ == '__main__': # delete_captures() # item_id = 'crawled/2023/02/20/data.gz' From 843b2d3134e96d8e11cdaaf72044961ee391d65d Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 23 Aug 2023 16:13:20 +0200 Subject: [PATCH 099/238] fix: correlations --- bin/importer/feeders/Telegram.py | 7 +++++-- bin/lib/objects/Chats.py | 12 +++++++++--- bin/lib/objects/Messages.py | 2 +- bin/lib/objects/UsersAccount.py | 4 +--- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/bin/importer/feeders/Telegram.py b/bin/importer/feeders/Telegram.py index 52eb0a75..2900a46d 100755 --- a/bin/importer/feeders/Telegram.py +++ b/bin/importer/feeders/Telegram.py @@ -79,7 +79,7 @@ class TelegramFeeder(DefaultFeeder): if meta.get('chat'): chat = Chat(meta['chat']['id'], 'telegram') - if meta['chat'].get('username'): # TODO USE ID AND SAVE USERNAME + if meta['chat'].get('username'): # SAVE USERNAME chat_username = meta['chat']['username'] # Chat---Message @@ -99,7 +99,7 @@ class TelegramFeeder(DefaultFeeder): chat = None # message sender - if meta.get('sender'): # TODO handle message channel forward + if meta.get('sender'): # TODO handle message channel forward - check if is user user_id = meta['sender']['id'] user_account = UsersAccount.UserAccount(user_id, 'telegram') # UserAccount---Message @@ -117,10 +117,13 @@ class TelegramFeeder(DefaultFeeder): if meta['sender'].get('username'): username = Username(meta['sender']['username'], 'telegram') user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id) + # TODO Update user_account<--->username timeline # Username---Message username.add(date, self.item_id) # TODO #################################################################### + if chat: + # Chat---Username chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id) # if meta.get('fwd_from'): diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index 438acf51..a3d1721c 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -117,9 +117,15 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? def get_name(self): # get username ???? pass - # return username correlation - def get_users(self): # get participants ??? -> passive users ??? - pass + # users that send at least a message else participants/spectator + # correlation created by messages + def get_users(self): + users = set() + accounts = self.get_correlation('user-account').get('user-account', []) + for account in accounts: + users.add(account[1:]) + return users + # def get_last_message_id(self): # diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 98cc838f..302f0d0a 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -91,7 +91,7 @@ class Message(AbstractObject): # message_id = message_id[:-3] return message_id - def get_chat_id(self): # TODO optimize + def get_chat_id(self): # TODO optimize -> use me to tag Chat chat_id = self.get_basename().rsplit('_', 1)[0] # if chat_id.endswith('.gz'): # chat_id = chat_id[:-3] diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py index 0355806e..f4f71d05 100755 --- a/bin/lib/objects/UsersAccount.py +++ b/bin/lib/objects/UsersAccount.py @@ -87,9 +87,7 @@ class UserAccount(AbstractSubtypeObject): def get_usernames(self): usernames = [] - correl = self.get_correlation('username') - for partial_id in correl.get('username', []): - usernames.append(f'username:{partial_id}') + # TODO TIMELINE return usernames def get_meta(self, options=set()): From c01b806ae30c6304bee5203b6fd46b389cbf1c2b Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 24 Aug 2023 11:11:57 +0200 Subject: [PATCH 100/238] chg: [mail exporter] add obj content extract for each yara rule match --- bin/exporter/MailExporter.py | 20 ++++++++++---- bin/lib/Tracker.py | 3 +- bin/trackers/Tracker_Yara.py | 53 ++++++++++++++++++++++++++++++++++-- 3 files changed, 68 insertions(+), 8 deletions(-) diff --git a/bin/exporter/MailExporter.py b/bin/exporter/MailExporter.py index c4d3f5b5..40ee1708 100755 --- a/bin/exporter/MailExporter.py +++ b/bin/exporter/MailExporter.py @@ -124,16 +124,26 @@ class MailExporterTracker(MailExporter): def __init__(self, host=None, port=None, password=None, user='', sender=''): super().__init__(host=host, port=port, password=password, user=user, sender=sender) - def export(self, tracker, obj): # TODO match + def export(self, tracker, obj, matches=[]): tracker_type = tracker.get_type() tracker_name = tracker.get_tracked() - subject = f'AIL Framework Tracker: {tracker_name}' # TODO custom subject + description = tracker.get_description() + if not description: + description = tracker_name + + subject = f'AIL Framework Tracker: {description}' body = f"AIL Framework, New occurrence for {tracker_type} tracker: {tracker_name}\n" body += f'Item: {obj.id}\nurl:{obj.get_link()}' - # TODO match option - # if match: - # body += f'Tracker Match:\n\n{escape(match)}' + if matches: + body += '\n' + nb = 1 + for match in matches: + body += f'\nMatch {nb}: {match[0]}\nExtract:\n{match[1]}\n\n' + nb += 1 + else: + body = f"AIL Framework, New occurrence for {tracker_type} tracker: {tracker_name}\n" + body += f'Item: {obj.id}\nurl:{obj.get_link()}' for mail in tracker.get_mails(): self._export(mail, subject, body) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index c06e303d..4baa3e5f 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -248,7 +248,8 @@ class Tracker: return self._get_field('user_id') def webhook_export(self): - return r_tracker.hexists(f'tracker:{self.uuid}', 'webhook') + webhook = self.get_webhook() + return webhook is not None and webhook def get_webhook(self): return r_tracker.hget(f'tracker:{self.uuid}', 'webhook') diff --git a/bin/trackers/Tracker_Yara.py b/bin/trackers/Tracker_Yara.py index fab397d1..1cebeaa6 100755 --- a/bin/trackers/Tracker_Yara.py +++ b/bin/trackers/Tracker_Yara.py @@ -73,8 +73,56 @@ class Tracker_Yara(AbstractModule): print(f'{self.obj.get_id()}: yara scanning timed out') self.redis_logger.info(f'{self.obj.get_id()}: yara scanning timed out') + def convert_byte_offset_to_string(self, b_content, offset): + byte_chunk = b_content[:offset + 1] + try: + string_chunk = byte_chunk.decode() + offset = len(string_chunk) - 1 + return offset + except UnicodeDecodeError: + return self.convert_byte_offset_to_string(b_content, offset - 1) + + def extract_matches(self, data, limit=500, lines=5): + matches = [] + content = self.obj.get_content() + l_content = len(content) + b_content = content.encode() + for string_match in data.get('strings'): + for string_match_instance in string_match.instances: + start = string_match_instance.offset + value = string_match_instance.matched_data.decode() + end = start + string_match_instance.matched_length + # str + start = self.convert_byte_offset_to_string(b_content, start) + end = self.convert_byte_offset_to_string(b_content, end) + + # Start + if start > limit: + i_start = start - limit + else: + i_start = 0 + str_start = content[i_start:start].splitlines() + if len(str_start) > lines: + str_start = '\n'.join(str_start[-lines + 1:]) + else: + str_start = content[i_start:start] + + # End + if end + limit > l_content: + i_end = l_content + else: + i_end = end + limit + str_end = content[end:i_end].splitlines() + if len(str_end) > lines: + str_end = '\n'.join(str_end[:lines + 1]) + else: + str_end = content[end:i_end] + matches.append((value, f'{str_start}{value}{str_end}')) + return matches + def yara_rules_match(self, data): tracker_name = data['namespace'] + matches = None obj_id = self.obj.get_id() for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type('yara', self.obj.get_type(), tracker_name): tracker = Tracker.Tracker(tracker_uuid) @@ -96,8 +144,9 @@ class Tracker_Yara(AbstractModule): # Mails if tracker.mail_export(): - # TODO add matches + custom subjects - self.exporters['mail'].export(tracker, self.obj) + if not matches: + matches = self.extract_matches(data) + self.exporters['mail'].export(tracker, self.obj, matches) # Webhook if tracker.webhook_export(): From 546d6538fd25cbf701b220b4440699f776367cb7 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 24 Aug 2023 14:37:50 +0200 Subject: [PATCH 101/238] chg: [mail exporter] add obj content extract for each regex match --- bin/exporter/MailExporter.py | 1 + bin/trackers/Tracker_Regex.py | 51 ++++++++++++++++++++++++++++++----- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/bin/exporter/MailExporter.py b/bin/exporter/MailExporter.py index 40ee1708..41074d7b 100755 --- a/bin/exporter/MailExporter.py +++ b/bin/exporter/MailExporter.py @@ -145,5 +145,6 @@ class MailExporterTracker(MailExporter): body = f"AIL Framework, New occurrence for {tracker_type} tracker: {tracker_name}\n" body += f'Item: {obj.id}\nurl:{obj.get_link()}' + # print(body) for mail in tracker.get_mails(): self._export(mail, subject, body) diff --git a/bin/trackers/Tracker_Regex.py b/bin/trackers/Tracker_Regex.py index 5cc06410..db35f239 100755 --- a/bin/trackers/Tracker_Regex.py +++ b/bin/trackers/Tracker_Regex.py @@ -41,6 +41,8 @@ class Tracker_Regex(AbstractModule): self.tracked_regexs = Tracker.get_tracked_regexs() self.last_refresh = time.time() + self.obj = None + # Exporter self.exporters = {'mail': MailExporterTracker(), 'webhook': WebHookExporterTracker()} @@ -56,6 +58,7 @@ class Tracker_Regex(AbstractModule): print('Tracked regex refreshed') obj = ail_objects.get_object(obj_type, subtype, obj_id) + self.obj = obj obj_id = obj.get_id() obj_type = obj.get_type() @@ -66,12 +69,46 @@ class Tracker_Regex(AbstractModule): content = obj.get_content() for dict_regex in self.tracked_regexs[obj_type]: - matched = self.regex_findall(dict_regex['regex'], obj_id, content) - if matched: - self.new_tracker_found(dict_regex['tracked'], 'regex', obj) + matches = self.regex_finditer(dict_regex['regex'], obj_id, content) + if matches: + self.new_tracker_found(dict_regex['tracked'], 'regex', obj, matches) - def new_tracker_found(self, tracker_name, tracker_type, obj): + def extract_matches(self, re_matches, limit=500, lines=5): + matches = [] + content = self.obj.get_content() + l_content = len(content) + for match in re_matches: + start = match[0] + value = match[2] + end = match[1] + + # Start + if start > limit: + i_start = start - limit + else: + i_start = 0 + str_start = content[i_start:start].splitlines() + if len(str_start) > lines: + str_start = '\n'.join(str_start[-lines + 1:]) + else: + str_start = content[i_start:start] + + # End + if end + limit > l_content: + i_end = l_content + else: + i_end = end + limit + str_end = content[end:i_end].splitlines() + if len(str_end) > lines: + str_end = '\n'.join(str_end[:lines + 1]) + else: + str_end = content[end:i_end] + matches.append((value, f'{str_start}{value}{str_end}')) + return matches + + def new_tracker_found(self, tracker_name, tracker_type, obj, re_matches): obj_id = obj.get_id() + matches = None for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type(tracker_type, obj.get_type(), tracker_name): tracker = Tracker.Tracker(tracker_uuid) @@ -93,8 +130,9 @@ class Tracker_Regex(AbstractModule): obj.add_tag(tag) if tracker.mail_export(): - # TODO add matches + custom subjects - self.exporters['mail'].export(tracker, obj) + if not matches: + matches = self.extract_matches(re_matches) + self.exporters['mail'].export(tracker, obj, matches) if tracker.webhook_export(): self.exporters['webhook'].export(tracker, obj) @@ -103,4 +141,3 @@ class Tracker_Regex(AbstractModule): if __name__ == "__main__": module = Tracker_Regex() module.run() - # module.compute('submitted/2023/05/02/submitted_b1e518f1-703b-40f6-8238-d1c22888197e.gz') From b32f1102851c9be585f26867a2db97b72fba1348 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 28 Aug 2023 16:29:38 +0200 Subject: [PATCH 102/238] chg: [chat + user-account] correlations + usernames timeline --- bin/crawlers/Crawler.py | 6 +- bin/importer/feeders/Jabber.py | 12 +- bin/importer/feeders/Telegram.py | 46 ++-- bin/importer/feeders/Twitter.py | 8 +- bin/lib/correlations_engine.py | 16 +- bin/lib/crawlers.py | 6 +- bin/lib/objects/Chats.py | 35 ++- bin/lib/objects/Messages.py | 19 +- bin/lib/objects/UsersAccount.py | 15 +- bin/lib/objects/abstract_object.py | 4 +- bin/lib/objects/abstract_subtype_object.py | 18 +- bin/lib/objects/ail_objects.py | 6 + bin/lib/timeline_engine.py | 231 +++++++++++------- bin/modules/Cryptocurrencies.py | 2 +- bin/modules/PgpDump.py | 6 +- bin/modules/Telegram.py | 4 +- .../templates/objects/chat/ChatMessages.html | 18 +- 17 files changed, 279 insertions(+), 173 deletions(-) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index 7f2c3df9..d16ad9f7 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -282,7 +282,7 @@ class Crawler(AbstractModule): title_content = crawlers.extract_title_from_html(entries['html']) if title_content: title = Titles.create_title(title_content) - title.add(item.get_date(), item_id) + title.add(item.get_date(), item) # SCREENSHOT if self.screenshot: @@ -306,11 +306,11 @@ class Crawler(AbstractModule): for cookie_name in crawlers.extract_cookies_names_from_har(entries['har']): print(cookie_name) cookie = CookiesNames.create(cookie_name) - cookie.add(self.date.replace('/', ''), self.domain.id) + cookie.add(self.date.replace('/', ''), self.domain) for etag_content in crawlers.extract_etag_from_har(entries['har']): print(etag_content) etag = Etags.create(etag_content) - etag.add(self.date.replace('/', ''), self.domain.id) + etag.add(self.date.replace('/', ''), self.domain) crawlers.extract_hhhash(entries['har'], self.domain.id, self.date.replace('/', '')) # Next Children diff --git a/bin/importer/feeders/Jabber.py b/bin/importer/feeders/Jabber.py index 79d0950f..8c90adfd 100755 --- a/bin/importer/feeders/Jabber.py +++ b/bin/importer/feeders/Jabber.py @@ -17,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from importer.feeders.Default import DefaultFeeder from lib.objects.Usernames import Username -from lib import item_basic +from lib.objects.Items import Item class JabberFeeder(DefaultFeeder): @@ -36,7 +36,7 @@ class JabberFeeder(DefaultFeeder): self.item_id = f'{item_id}.gz' return self.item_id - def process_meta(self): + def process_meta(self): # TODO replace me by message """ Process JSON meta field. """ @@ -44,10 +44,12 @@ class JabberFeeder(DefaultFeeder): # item_basic.add_map_obj_id_item_id(jabber_id, item_id, 'jabber_id') ############################################## to = str(self.json_data['meta']['jabber:to']) fr = str(self.json_data['meta']['jabber:from']) - date = item_basic.get_item_date(item_id) + + item = Item(self.item_id) + date = item.get_date() user_to = Username(to, 'jabber') user_fr = Username(fr, 'jabber') - user_to.add(date, self.item_id) - user_fr.add(date, self.item_id) + user_to.add(date, item) + user_fr.add(date, item) return None diff --git a/bin/importer/feeders/Telegram.py b/bin/importer/feeders/Telegram.py index 2900a46d..2cc6a127 100755 --- a/bin/importer/feeders/Telegram.py +++ b/bin/importer/feeders/Telegram.py @@ -21,7 +21,6 @@ from lib.objects.Chats import Chat from lib.objects import Messages from lib.objects import UsersAccount from lib.objects.Usernames import Username -from lib import item_basic import base64 import io @@ -57,7 +56,7 @@ class TelegramFeeder(DefaultFeeder): # date = datetime.date.today().strftime("%Y/%m/%d") chat_id = str(self.json_data['meta']['chat']['id']) message_id = str(self.json_data['meta']['id']) - self.item_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) + self.item_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) # TODO rename self.item_id return self.item_id def process_meta(self): @@ -68,7 +67,7 @@ class TelegramFeeder(DefaultFeeder): meta = self.json_data['meta'] mess_id = self.json_data['meta']['id'] if meta.get('reply_to'): - reply_to_id = meta['reply_to'] + reply_to_id = meta['reply_to']['id'] else: reply_to_id = None @@ -76,25 +75,24 @@ class TelegramFeeder(DefaultFeeder): date = datetime.datetime.fromtimestamp(timestamp) date = date.strftime('%Y%m%d') + if self.json_data.get('translation'): + translation = self.json_data['translation'] + else: + translation = None + decoded = base64.standard_b64decode(self.json_data['data']) + content = gunzip_bytes_obj(decoded) + message = Messages.create(self.item_id, content, translation=translation) + if meta.get('chat'): chat = Chat(meta['chat']['id'], 'telegram') - if meta['chat'].get('username'): # SAVE USERNAME - chat_username = meta['chat']['username'] + if meta['chat'].get('username'): + chat_username = Username(meta['chat']['username'], 'telegram') + chat.update_username_timeline(chat_username.get_global_id(), timestamp) # Chat---Message - chat.add(date, self.item_id) # TODO modify to accept file objects - # message meta ????? who is the user if two user ???? - - if self.json_data.get('translation'): - translation = self.json_data['translation'] - else: - translation = None - decoded = base64.standard_b64decode(self.json_data['data']) - content = gunzip_bytes_obj(decoded) - Messages.create(self.item_id, content, translation=translation) - - chat.add_message(self.item_id, timestamp, mess_id, reply_id=reply_to_id) + chat.add(date) + chat.add_message(message.get_global_id(), timestamp, mess_id, reply_id=reply_to_id) else: chat = None @@ -103,7 +101,7 @@ class TelegramFeeder(DefaultFeeder): user_id = meta['sender']['id'] user_account = UsersAccount.UserAccount(user_id, 'telegram') # UserAccount---Message - user_account.add(date, self.item_id) + user_account.add(date, obj=message) # UserAccount---Chat user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id) @@ -116,20 +114,22 @@ class TelegramFeeder(DefaultFeeder): if meta['sender'].get('username'): username = Username(meta['sender']['username'], 'telegram') + # TODO timeline or/and correlation ???? user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id) - # TODO Update user_account<--->username timeline + user_account.update_username_timeline(username.get_global_id(), timestamp) # Username---Message - username.add(date, self.item_id) # TODO #################################################################### + username.add(date) # TODO # correlation message ??? - if chat: - # Chat---Username - chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id) + # if chat: # TODO Chat---Username correlation ??? + # # Chat---Username + # chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id) # if meta.get('fwd_from'): # if meta['fwd_from'].get('post_author') # user first name # TODO reply threads ???? + # message edit ???? return None diff --git a/bin/importer/feeders/Twitter.py b/bin/importer/feeders/Twitter.py index d5040c65..1c719e73 100755 --- a/bin/importer/feeders/Twitter.py +++ b/bin/importer/feeders/Twitter.py @@ -17,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from importer.feeders.Default import DefaultFeeder from lib.objects.Usernames import Username -from lib import item_basic +from lib.objects.Items import Item class TwitterFeeder(DefaultFeeder): @@ -40,9 +40,9 @@ class TwitterFeeder(DefaultFeeder): ''' # tweet_id = str(self.json_data['meta']['twitter:tweet_id']) # item_basic.add_map_obj_id_item_id(tweet_id, item_id, 'twitter_id') ############################################ - - date = item_basic.get_item_date(self.item_id) + item = Item(self.item_id) + date = item.get_date() user = str(self.json_data['meta']['twitter:id']) username = Username(user, 'twitter') - username.add(date, item_id) + username.add(date, item) return None diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index 94a06773..f7b13f61 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -41,20 +41,22 @@ config_loader = None ################################## CORRELATION_TYPES_BY_OBJ = { - "chat": ["item", "username"], # item ??? + "chat": ["user-account"], # message or direct correlation like cve, bitcoin, ... ??? "cookie-name": ["domain"], - "cryptocurrency": ["domain", "item"], - "cve": ["domain", "item"], - "decoded": ["domain", "item"], + "cryptocurrency": ["domain", "item", "message"], + "cve": ["domain", "item", "message"], + "decoded": ["domain", "item", "message"], "domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"], "etag": ["domain"], "favicon": ["domain", "item"], # TODO Decoded "hhhash": ["domain"], - "item": ["chat", "cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], - "pgp": ["domain", "item"], + "item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], # chat ??? + "message": ["cve", "cryptocurrency", "decoded", "pgp", "user-account"], # chat ?? + "pgp": ["domain", "item", "message"], "screenshot": ["domain", "item"], "title": ["domain", "item"], - "username": ["chat", "domain", "item"], + "user-account": ["chat", "message"], + "username": ["domain", "item", "message"], # TODO chat-user/account } def get_obj_correl_types(obj_type): diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 3e61ed88..6387c76f 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -342,7 +342,7 @@ def _reprocess_all_hars_cookie_name(): for cookie_name in extract_cookies_names_from_har(get_har_content(har_id)): print(domain, date, cookie_name) cookie = CookiesNames.create(cookie_name) - cookie.add(date, domain) + cookie.add(date, Domain(domain)) def extract_etag_from_har(har): # TODO check response url etags = set() @@ -365,7 +365,7 @@ def _reprocess_all_hars_etag(): for etag_content in extract_etag_from_har(get_har_content(har_id)): print(domain, date, etag_content) etag = Etags.create(etag_content) - etag.add(date, domain) + etag.add(date, Domain(domain)) def extract_hhhash_by_id(har_id, domain, date): return extract_hhhash(get_har_content(har_id), domain, date) @@ -395,7 +395,7 @@ def extract_hhhash(har, domain, date): # ----- obj = HHHashs.create(hhhash_header, hhhash) - obj.add(date, domain) + obj.add(date, Domain(domain)) hhhashs.add(hhhash) urls.add(url) diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index a3d1721c..bb27413d 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -18,7 +18,7 @@ from lib.ConfigLoader import ConfigLoader from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id from lib.data_retention_engine import update_obj_date from lib.objects import ail_objects -from lib import item_basic +from lib.timeline_engine import Timeline from lib.correlations_engine import get_correlation_by_correl_type @@ -126,6 +126,18 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? users.add(account[1:]) return users + def _get_timeline_username(self): + return Timeline(self.get_global_id(), 'username') + + def get_username(self): + return self._get_timeline_username().get_last_obj_id() + + def get_usernames(self): + return self._get_timeline_username().get_objs_ids() + + def update_username_timeline(self, username_global_id, timestamp): + self._get_timeline_username().add_timestamp(timestamp, username_global_id) + # def get_last_message_id(self): # @@ -144,18 +156,21 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None): obj = ail_objects.get_obj_from_global_id(obj_global_id) - mess_dict = obj.get_meta(options={'content', 'link', 'parent'}) + mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'}) if mess_dict.get('parent') and parent: mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False) - mess_dict['username'] = {} - user = obj.get_correlation('username').get('username') - if user: - subtype, user = user.pop().split(':', 1) - mess_dict['username']['type'] = 'telegram' - mess_dict['username']['subtype'] = subtype - mess_dict['username']['id'] = user + if mess_dict.get('user-account'): + user_account = ail_objects.get_obj_from_global_id(mess_dict['user-account']) + mess_dict['user-account'] = {} + mess_dict['user-account']['type'] = user_account.get_type() + mess_dict['user-account']['subtype'] = user_account.get_subtype(r_str=True) + mess_dict['user-account']['id'] = user_account.get_id() + username = user_account.get_username() + if username: + username = ail_objects.get_obj_from_global_id(username).get_default_meta(link=False) + mess_dict['user-account']['username'] = username # TODO get username at the given timestamp ??? else: - mess_dict['username']['id'] = 'UNKNOWN' + mess_dict['user-account']['id'] = 'UNKNOWN' if not mess_datetime: obj_mess_id = self._get_message_timestamp(obj_global_id) diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 302f0d0a..b724f854 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -27,7 +27,7 @@ from flask import url_for config_loader = ConfigLoader() r_cache = config_loader.get_redis_conn("Redis_Cache") r_object = config_loader.get_db_conn("Kvrocks_Objects") -r_content = config_loader.get_db_conn("Kvrocks_Content") +# r_content = config_loader.get_db_conn("Kvrocks_Content") baseurl = config_loader.get_config_str("Notifications", "ail_domain") config_loader = None @@ -61,7 +61,7 @@ class Message(AbstractObject): """ Returns source/feeder name """ - l_source = self.id.split('/')[:-4] + l_source = self.id.split('/')[:-2] return os.path.join(*l_source) def get_basename(self): @@ -79,7 +79,7 @@ class Message(AbstractObject): def get_date(self): timestamp = self.get_timestamp() - return datetime.fromtimestamp(timestamp).strftime('%Y%m%d') + return datetime.fromtimestamp(float(timestamp)).strftime('%Y%m%d') def get_timestamp(self): dirs = self.id.split('/') @@ -92,11 +92,16 @@ class Message(AbstractObject): return message_id def get_chat_id(self): # TODO optimize -> use me to tag Chat - chat_id = self.get_basename().rsplit('_', 1)[0] + chat_id = self.get_basename().rsplit('_', 1)[0] # if chat_id.endswith('.gz'): # chat_id = chat_id[:-3] return chat_id + def get_user_account(self): + user_account = self.get_correlation('user-account') + if user_account.get('user-account'): + return f'user-account:{user_account["user-account"].pop()}' + # Update value on import # reply to -> parent ? # reply/comment - > children ? @@ -139,7 +144,7 @@ class Message(AbstractObject): return url def get_svg_icon(self): - return {'style': 'fas', 'icon': 'fa-comment-dots', 'color': '#4dffff', 'radius': 5} + return {'style': 'fas', 'icon': '\uf4ad', 'color': '#4dffff', 'radius': 5} def get_misp_object(self): # TODO obj = MISPObject('instant-message', standalone=True) @@ -181,6 +186,8 @@ class Message(AbstractObject): meta['investigations'] = self.get_investigations() if 'link' in options: meta['link'] = self.get_link(flask_context=True) + if 'user-account' in options: + meta['user-account'] = self.get_user_account() # meta['encoding'] = None return meta @@ -238,7 +245,7 @@ class Message(AbstractObject): def create(self, content, translation, tags): self._set_field('content', content) - r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content) + # r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content) if translation: self._set_translation(translation) for tag in tags: diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py index f4f71d05..5bc94a9c 100755 --- a/bin/lib/objects/UsersAccount.py +++ b/bin/lib/objects/UsersAccount.py @@ -3,7 +3,7 @@ import os import sys -import re +# import re from flask import url_for from pymisp import MISPObject @@ -15,6 +15,7 @@ sys.path.append(os.environ['AIL_BIN']) from lib import ail_core from lib.ConfigLoader import ConfigLoader from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id +from lib.timeline_engine import Timeline config_loader = ConfigLoader() baseurl = config_loader.get_config_str("Notifications", "ail_domain") @@ -80,15 +81,17 @@ class UserAccount(AbstractSubtypeObject): def set_phone(self, phone): return self._set_field('phone', phone) - # TODO REWRITE ADD FUNCTION + def _get_timeline_username(self): + return Timeline(self.get_global_id(), 'username') def get_username(self): - return '' + return self._get_timeline_username().get_last_obj_id() def get_usernames(self): - usernames = [] - # TODO TIMELINE - return usernames + return self._get_timeline_username().get_objs_ids() + + def update_username_timeline(self, username_global_id, timestamp): + self._get_timeline_username().add_timestamp(timestamp, username_global_id) def get_meta(self, options=set()): meta = self._get_meta(options=options) diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index 59a7e968..a3f25216 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -65,12 +65,14 @@ class AbstractObject(ABC): def get_global_id(self): return f'{self.get_type()}:{self.get_subtype(r_str=True)}:{self.get_id()}' - def get_default_meta(self, tags=False): + def get_default_meta(self, tags=False, link=False): dict_meta = {'id': self.get_id(), 'type': self.get_type(), 'subtype': self.get_subtype(r_str=True)} if tags: dict_meta['tags'] = self.get_tags() + if link: + dict_meta['link'] = self.get_link() return dict_meta def _get_field(self, field): diff --git a/bin/lib/objects/abstract_subtype_object.py b/bin/lib/objects/abstract_subtype_object.py index 82bb85f6..007f716b 100755 --- a/bin/lib/objects/abstract_subtype_object.py +++ b/bin/lib/objects/abstract_subtype_object.py @@ -151,7 +151,7 @@ class AbstractSubtypeObject(AbstractObject, ABC): # # - def add(self, date, item_id): + def add(self, date, obj=None): self.update_daterange(date) update_obj_date(date, self.type, self.subtype) # daily @@ -162,20 +162,22 @@ class AbstractSubtypeObject(AbstractObject, ABC): ####################################################################### ####################################################################### - # Correlations - self.add_correlation('item', '', item_id) - # domain - if is_crawled(item_id): - domain = get_item_domain(item_id) - self.add_correlation('domain', '', domain) + if obj: + # Correlations + self.add_correlation(obj.type, obj.get_subtype(r_str=True), obj.get_id()) + if obj.type == 'item': # TODO same for message->chat ??? + item_id = obj.get_id() + # domain + if is_crawled(item_id): + domain = get_item_domain(item_id) + self.add_correlation('domain', '', domain) # TODO:ADD objects + Stats def create(self, first_seen, last_seen): self.set_first_seen(first_seen) self.set_last_seen(last_seen) - def _delete(self): pass diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index cd2f7225..3a19060a 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -23,9 +23,11 @@ from lib.objects import Etags from lib.objects.Favicons import Favicon from lib.objects import HHHashs from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects +from lib.objects.Messages import Message from lib.objects import Pgps from lib.objects.Screenshots import Screenshot from lib.objects import Titles +from lib.objects.UsersAccount import UserAccount from lib.objects import Usernames config_loader = ConfigLoader() @@ -68,6 +70,8 @@ def get_object(obj_type, subtype, id): return Favicon(id) elif obj_type == 'hhhash': return HHHashs.HHHash(id) + elif obj_type == 'message': + return Message(id) elif obj_type == 'screenshot': return Screenshot(id) elif obj_type == 'cryptocurrency': @@ -76,6 +80,8 @@ def get_object(obj_type, subtype, id): return Pgps.Pgp(id, subtype) elif obj_type == 'title': return Titles.Title(id) + elif obj_type == 'user-account': + return UserAccount(id, subtype) elif obj_type == 'username': return Usernames.Username(id, subtype) diff --git a/bin/lib/timeline_engine.py b/bin/lib/timeline_engine.py index 405e7a50..58c222f6 100755 --- a/bin/lib/timeline_engine.py +++ b/bin/lib/timeline_engine.py @@ -46,112 +46,167 @@ config_loader = None # return [] # return correl_types -# TODO rename all function + add missing parameters +class Timeline: -def get_bloc_obj_global_id(bloc): - return r_meta.hget('hset:key', bloc) + def __init__(self, global_id, name): + self.id = global_id + self.name = name -def set_bloc_obj_global_id(bloc, global_id): - return r_meta.hset('hset:key', bloc, global_id) + def _get_block_obj_global_id(self, block): + return r_meta.hget(f'block:{self.id}:{self.name}', block) -def get_bloc_timestamp(bloc, position): - return r_meta.zscore('key', f'{position}:{bloc}') + def _set_block_obj_global_id(self, block, global_id): + return r_meta.hset(f'block:{self.id}:{self.name}', block, global_id) -def add_bloc(global_id, timestamp, end=None): - if end: - timestamp_end = end - else: - timestamp_end = timestamp - new_bloc = str(uuid4()) - r_meta.zadd('key', {f'start:{new_bloc}': timestamp, f'end:{new_bloc}': timestamp_end}) - set_bloc_obj_global_id(new_bloc, global_id) - return new_bloc + def _get_block_timestamp(self, block, position): + return r_meta.zscore(f'line:{self.id}:{self.name}', f'{position}:{block}') -def _update_bloc(bloc, position, timestamp): - r_meta.zadd('key', {f'{position}:{bloc}': timestamp}) + def _get_nearest_bloc_inf(self, timestamp): + inf = r_meta.zrevrangebyscore(f'line:{self.id}:{self.name}', float(timestamp), 0, start=0, num=1, withscores=True) + if inf: + inf, score = inf[0] + if inf.startswith('end'): + inf_key = f'start:{inf[4:]}' + inf_score = r_meta.zscore(f'line:{self.id}:{self.name}', inf_key) + if inf_score == score: + inf = inf_key + return inf + else: + return None -# score = timestamp -def get_nearest_bloc_inf(timestamp): - return r_meta.zrevrangebyscore('key', timestamp, 0, num=1) + def _get_nearest_bloc_sup(self, timestamp): + sup = r_meta.zrangebyscore(f'line:{self.id}:{self.name}', float(timestamp), '+inf', start=0, num=1, withscores=True) + if sup: + sup, score = sup[0] + if sup.startswith('start'): + sup_key = f'end:{sup[6:]}' + sup_score = r_meta.zscore(f'line:{self.id}:{self.name}', sup_key) + if score == sup_score: + sup = sup_key + return sup + else: + return None -def get_nearest_bloc_sup(timestamp): - return r_meta.zrangebyscore('key', timestamp, 0, num=1) + def get_first_obj_id(self): + first = r_meta.zrange(f'line:{self.id}:{self.name}', 0, 0) + if first: # start:block + first = first[0] + if first.startswith('start:'): + first = first[6:] + else: + first = first[4:] + return self._get_block_obj_global_id(first) -####################################################################################### + def get_last_obj_id(self): + last = r_meta.zrevrange(f'line:{self.id}:{self.name}', 0, 0) + if last: # end:block + last = last[0] + if last.startswith('end:'): + last = last[4:] + else: + last = last[6:] + return self._get_block_obj_global_id(last) -def add_timestamp(timestamp, obj_global_id): - inf = get_nearest_bloc_inf(timestamp) - sup = get_nearest_bloc_sup(timestamp) - if not inf and not sup: - # create new bloc - new_bloc = add_bloc(obj_global_id, timestamp) + def get_objs_ids(self): + objs = set() + for block in r_meta.zrange(f'line:{self.id}:{self.name}', 0, -1): + if block: + if block.startswith('start:'): + objs.add(self._get_block_obj_global_id(block[6:])) + return objs + + # def get_objs_ids(self): + # objs = {} + # last_obj_id = None + # for block, timestamp in r_meta.zrange(f'line:{self.id}:{self.name}', 0, -1, withscores=True): + # if block: + # if block.startswith('start:'): + # last_obj_id = self._get_block_obj_global_id(block[6:]) + # objs[last_obj_id] = {'first_seen': timestamp} + # else: + # objs[last_obj_id]['last_seen'] = timestamp + # return objs + + def _update_bloc(self, block, position, timestamp): + r_meta.zadd(f'line:{self.id}:{self.name}', {f'{position}:{block}': timestamp}) + + def _add_bloc(self, obj_global_id, timestamp, end=None): + if end: + timestamp_end = end + else: + timestamp_end = timestamp + new_bloc = str(uuid4()) + r_meta.zadd(f'line:{self.id}:{self.name}', {f'start:{new_bloc}': timestamp, f'end:{new_bloc}': timestamp_end}) + self._set_block_obj_global_id(new_bloc, obj_global_id) return new_bloc - # timestamp < first_seen - elif not inf: - sup_pos, sup_id = inf.split(':') - sup_obj = get_bloc_obj_global_id(sup_pos) - if sup_obj == obj_global_id: - _update_bloc(sup_id, 'start', timestamp) - # create new bloc - else: - new_bloc = add_bloc(obj_global_id, timestamp) + + def add_timestamp(self, timestamp, obj_global_id): + inf = self._get_nearest_bloc_inf(timestamp) + sup = self._get_nearest_bloc_sup(timestamp) + if not inf and not sup: + # create new bloc + new_bloc = self._add_bloc(obj_global_id, timestamp) return new_bloc - - # timestamp > first_seen - elif not sup: - inf_pos, inf_id = inf.split(':') - inf_obj = get_bloc_obj_global_id(inf_id) - if inf_obj == obj_global_id: - _update_bloc(inf_id, 'end', timestamp) - # create new bloc - else: - new_bloc = add_bloc(obj_global_id, timestamp) - return new_bloc - - else: - inf_pos, inf_id = inf.split(':') - sup_pos, sup_id = inf.split(':') - inf_obj = get_bloc_obj_global_id(inf_id) - - if inf_id == sup_id: - # reduce bloc + create two new bloc - if obj_global_id != inf_obj: - # get end timestamp - sup_timestamp = get_bloc_timestamp(sup_id, 'end') - # reduce original bloc - _update_bloc(inf_id, 'end', timestamp - 1) - # Insert new bloc - new_bloc = add_bloc(obj_global_id, timestamp) - # Recreate end of the first bloc by a new bloc - add_bloc(inf_obj, timestamp + 1, end=sup_timestamp) + # timestamp < first_seen + elif not inf: + sup_pos, sup_id = sup.split(':') + sup_obj = self._get_block_obj_global_id(sup_id) + if sup_obj == obj_global_id: + self._update_bloc(sup_id, 'start', timestamp) + # create new bloc + else: + new_bloc = self._add_bloc(obj_global_id, timestamp) return new_bloc - # timestamp in existing bloc - else: - return inf_id - - # different blocs: expend sup/inf bloc or create a new bloc if - elif inf_pos == 'end' and sup_pos == 'start': - # Extend inf bloc - if obj_global_id == inf_obj: - _update_bloc(inf_id, 'end', timestamp) - return inf_id - - sup_obj = get_bloc_obj_global_id(sup_pos) - # Extend sup bloc - if obj_global_id == sup_obj: - _update_bloc(sup_id, 'start', timestamp) - return sup_id - + # timestamp > first_seen + elif not sup: + inf_pos, inf_id = inf.split(':') + inf_obj = self._get_block_obj_global_id(inf_id) + if inf_obj == obj_global_id: + self._update_bloc(inf_id, 'end', timestamp) # create new bloc - new_bloc = add_bloc(obj_global_id, timestamp) - return new_bloc + else: + new_bloc = self._add_bloc(obj_global_id, timestamp) + return new_bloc - # inf_pos == 'start' and sup_pos == 'end' - # else raise error ??? + else: + inf_pos, inf_id = inf.split(':') + sup_pos, sup_id = sup.split(':') + inf_obj = self._get_block_obj_global_id(inf_id) + if inf_id == sup_id: + # reduce bloc + create two new bloc + if obj_global_id != inf_obj: + # get end timestamp + sup_timestamp = self._get_block_timestamp(sup_id, 'end') + # reduce original bloc + self._update_bloc(inf_id, 'end', timestamp - 1) + # Insert new bloc + new_bloc = self._add_bloc(obj_global_id, timestamp) + # Recreate end of the first bloc by a new bloc + self._add_bloc(inf_obj, timestamp + 1, end=sup_timestamp) + return new_bloc + # timestamp in existing bloc + else: + return inf_id + # different blocs: expend sup/inf bloc or create a new bloc if + elif inf_pos == 'end' and sup_pos == 'start': + # Extend inf bloc + if obj_global_id == inf_obj: + self._update_bloc(inf_id, 'end', timestamp) + return inf_id + sup_obj = self._get_block_obj_global_id(sup_id) + # Extend sup bloc + if obj_global_id == sup_obj: + self._update_bloc(sup_id, 'start', timestamp) + return sup_id + # create new bloc + new_bloc = self._add_bloc(obj_global_id, timestamp) + return new_bloc + # inf_pos == 'start' and sup_pos == 'end' + # else raise error ??? diff --git a/bin/modules/Cryptocurrencies.py b/bin/modules/Cryptocurrencies.py index 6197f8a1..318cdd88 100755 --- a/bin/modules/Cryptocurrencies.py +++ b/bin/modules/Cryptocurrencies.py @@ -130,7 +130,7 @@ class Cryptocurrencies(AbstractModule, ABC): if crypto.is_valid_address(): # print(address) is_valid_address = True - crypto.add(date, item_id) + crypto.add(date, item) # Check private key if is_valid_address: diff --git a/bin/modules/PgpDump.py b/bin/modules/PgpDump.py index 0647e897..53c89a19 100755 --- a/bin/modules/PgpDump.py +++ b/bin/modules/PgpDump.py @@ -210,18 +210,18 @@ class PgpDump(AbstractModule): date = item.get_date() for key in self.keys: pgp = Pgps.Pgp(key, 'key') - pgp.add(date, self.item_id) + pgp.add(date, item) print(f' key: {key}') for name in self.names: pgp = Pgps.Pgp(name, 'name') - pgp.add(date, self.item_id) + pgp.add(date, item) print(f' name: {name}') self.tracker_term.compute(name, obj_type='pgp', subtype='name') self.tracker_regex.compute(name, obj_type='pgp', subtype='name') self.tracker_yara.compute(name, obj_type='pgp', subtype='name') for mail in self.mails: pgp = Pgps.Pgp(mail, 'mail') - pgp.add(date, self.item_id) + pgp.add(date, item) print(f' mail: {mail}') self.tracker_term.compute(mail, obj_type='pgp', subtype='mail') self.tracker_regex.compute(mail, obj_type='pgp', subtype='mail') diff --git a/bin/modules/Telegram.py b/bin/modules/Telegram.py index 31d90878..42feaa09 100755 --- a/bin/modules/Telegram.py +++ b/bin/modules/Telegram.py @@ -58,7 +58,7 @@ class Telegram(AbstractModule): user_id = dict_url.get('username') if user_id: username = Username(user_id, 'telegram') - username.add(item_date, item.id) + username.add(item_date, item) print(f'username: {user_id}') invite_hash = dict_url.get('invite_hash') if invite_hash: @@ -73,7 +73,7 @@ class Telegram(AbstractModule): user_id = dict_url.get('username') if user_id: username = Username(user_id, 'telegram') - username.add(item_date, item.id) + username.add(item_date, item) print(f'username: {user_id}') invite_hash = dict_url.get('invite_hash') if invite_hash: diff --git a/var/www/templates/objects/chat/ChatMessages.html b/var/www/templates/objects/chat/ChatMessages.html index b89a447a..aa0bdf35 100644 --- a/var/www/templates/objects/chat/ChatMessages.html +++ b/var/www/templates/objects/chat/ChatMessages.html @@ -120,14 +120,26 @@
    - {{ mess['username']['id'] }} + {{ mess['user-account']['id'] }}
    {{ mess['hour'] }}
    -
    {{ mess['username']['id'] }}
    +
    + {% if mess['user-account']['username'] %} + {{ mess['user-account']['username']['id'] }} + {% else %} + {{ mess['user-account']['id'] }} + {% endif %} +
    {% if mess['reply_to'] %}
    -
    {{ mess['reply_to']['username']['id'] }}
    +
    + {% if mess['reply_to']['user-account']['username'] %} + {{ mess['reply_to']['user-account']['username']['id'] }} + {% else %} + {{ mess['reply_to']['user-account']['id'] }} + {% endif %} + 
    {{ mess['reply_to']['content'] }}
    {% for tag in mess['reply_to']['tags'] %} {{ tag }} From 24969610cc4d5c04845e65dfaf9a5592487a0954 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 29 Aug 2023 11:59:39 +0200 Subject: [PATCH 103/238] fix: [items source] fix empty sources list --- bin/lib/item_basic.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/lib/item_basic.py b/bin/lib/item_basic.py index 71fa5378..b35d126e 100755 --- a/bin/lib/item_basic.py +++ b/bin/lib/item_basic.py @@ -209,7 +209,10 @@ def _get_dir_source_name(directory, source_name=None, l_sources_name=set(), filt l_dir = os.listdir(directory) # empty directory if not l_dir: - return l_sources_name.add(source_name) + if source_name: + return l_sources_name.add(source_name) + else: + return l_sources_name else: for src_name in l_dir: if len(src_name) == 4: From 099253f8546237b6164f90e78b16d5444fbf3fbb Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 29 Aug 2023 13:50:16 +0200 Subject: [PATCH 104/238] fix: [json importer] fix empty source name --- bin/importer/feeders/Default.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/importer/feeders/Default.py b/bin/importer/feeders/Default.py index 482d06b4..100ed1e6 100755 --- a/bin/importer/feeders/Default.py +++ b/bin/importer/feeders/Default.py @@ -24,8 +24,12 @@ class DefaultFeeder: Return feeder name. first part of the item_id and display in the UI """ if not self.name: - return self.get_source() - return self.name + name = self.get_source() + else: + name = self.name + if not name: + name = 'default' + return name def get_source(self): return self.json_data.get('source') From 7c73f0944a1a4b8ba052563f6bc0b03374c6ffdf Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 29 Aug 2023 14:03:26 +0200 Subject: [PATCH 105/238] fix: [items source] filter invalid item sources --- bin/lib/item_basic.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/lib/item_basic.py b/bin/lib/item_basic.py index b35d126e..25420106 100755 --- a/bin/lib/item_basic.py +++ b/bin/lib/item_basic.py @@ -204,7 +204,11 @@ def _get_dir_source_name(directory, source_name=None, l_sources_name=set(), filt if not l_sources_name: l_sources_name = set() if source_name: - l_dir = os.listdir(os.path.join(directory, source_name)) + path = os.path.join(directory, source_name) + if os.path.isdir(path): + l_dir = os.listdir(os.path.join(directory, source_name)) + else: + l_dir = [] else: l_dir = os.listdir(directory) # empty directory @@ -215,7 +219,7 @@ def _get_dir_source_name(directory, source_name=None, l_sources_name=set(), filt return l_sources_name else: for src_name in l_dir: - if len(src_name) == 4: + if len(src_name) == 4 and source_name: # try: int(src_name) to_add = os.path.join(source_name) From ed0423118e9facb55fff0d3ef381e688aeb0ade0 Mon Sep 17 00:00:00 2001 From: Jean-Louis Huynen Date: Thu, 31 Aug 2023 15:42:44 +0200 Subject: [PATCH 106/238] chg: [crawlers] submit a single cookie to the crawler task API --- bin/lib/crawlers.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 18b1eeac..3a0a9f19 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -1692,6 +1692,18 @@ def api_add_crawler_task(data, user_id=None): return {'error': 'The access to this cookiejar is restricted'}, 403 cookiejar_uuid = cookiejar.uuid + cookie = data.get('cookie', None) + if not cookiejar_uuid and cookie: + # Create new cookiejar + cookiejar_uuid = create_cookiejar(user_id, "single-shot cookiejar", 1, None) + cookiejar = Cookiejar(cookiejar_uuid) + try: + name = cookie.get('name') + value = cookie.get('value') + cookiejar.add_cookie(name, value, None, None, None, None, None) + except KeyError: + return {'error': 'Invalid cookie key, please submit a valid JSON', 'cookiejar_uuid': cookiejar_uuid}, 400 + frequency = data.get('frequency', None) if frequency: if frequency not in ['monthly', 'weekly', 'daily', 'hourly']: From 68c17c3fbcc20b9e63a3b97d0faac092a970dd10 Mon Sep 17 00:00:00 2001 From: Jean-Louis Huynen Date: Thu, 31 Aug 2023 16:13:20 +0200 Subject: [PATCH 107/238] chg: [crawlers] submit cookies to the crawler task API --- bin/lib/crawlers.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 3a0a9f19..67f868f0 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -1692,17 +1692,18 @@ def api_add_crawler_task(data, user_id=None): return {'error': 'The access to this cookiejar is restricted'}, 403 cookiejar_uuid = cookiejar.uuid - cookie = data.get('cookie', None) - if not cookiejar_uuid and cookie: + cookies = data.get('cookies', None) + if not cookiejar_uuid and cookies: # Create new cookiejar cookiejar_uuid = create_cookiejar(user_id, "single-shot cookiejar", 1, None) cookiejar = Cookiejar(cookiejar_uuid) - try: - name = cookie.get('name') - value = cookie.get('value') - cookiejar.add_cookie(name, value, None, None, None, None, None) - except KeyError: - return {'error': 'Invalid cookie key, please submit a valid JSON', 'cookiejar_uuid': cookiejar_uuid}, 400 + for cookie in cookies: + try: + name = cookie.get('name') + value = cookie.get('value') + cookiejar.add_cookie(name, value, None, None, None, None, None) + except KeyError: + return {'error': 'Invalid cookie key, please submit a valid JSON', 'cookiejar_uuid': cookiejar_uuid}, 400 frequency = data.get('frequency', None) if frequency: From bb3dad2873e3f68079a65fbdbad5ccc28597d612 Mon Sep 17 00:00:00 2001 From: terrtia Date: Thu, 7 Sep 2023 10:38:03 +0200 Subject: [PATCH 108/238] chg: [objs processed] xxhash messages --- bin/lib/ail_queues.py | 36 ++++++++++++++++++------------------ configs/core.cfg.sample | 5 +++++ requirements.txt | 1 + 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/bin/lib/ail_queues.py b/bin/lib/ail_queues.py index 3ab68708..38c5a42d 100755 --- a/bin/lib/ail_queues.py +++ b/bin/lib/ail_queues.py @@ -6,7 +6,7 @@ import sys import datetime import time -from hashlib import sha256 +import xxhash sys.path.append(os.environ['AIL_BIN']) ################################## @@ -80,12 +80,12 @@ class AILQueue: # raise Exception(f'Error: queue {self.name}, no AIL object provided') else: obj_global_id, mess = row_mess - sha256_mess = sha256(message.encode()).hexdigest() - add_processed_obj(obj_global_id, sha256_mess, module=self.name) - return obj_global_id, sha256_mess, mess + m_hash = xxhash.xxh3_64_hexdigest(message) + add_processed_obj(obj_global_id, m_hash, module=self.name) + return obj_global_id, m_hash, mess - def end_message(self, obj_global_id, sha256_mess): - end_processed_obj(obj_global_id, sha256_mess, module=self.name) + def end_message(self, obj_global_id, m_hash): + end_processed_obj(obj_global_id, m_hash, module=self.name) def send_message(self, obj_global_id, message='', queue_name=None): if not self.subscribers_modules: @@ -100,14 +100,14 @@ class AILQueue: message = f'{obj_global_id};{message}' if obj_global_id != '::': - sha256_mess = sha256(message.encode()).hexdigest() + m_hash = xxhash.xxh3_64_hexdigest(message) else: - sha256_mess = None + m_hash = None # Add message to all modules for module_name in self.subscribers_modules[queue_name]: - if sha256_mess: - add_processed_obj(obj_global_id, sha256_mess, queue=module_name) + if m_hash: + add_processed_obj(obj_global_id, m_hash, queue=module_name) r_queues.rpush(f'queue:{module_name}:in', message) # stats @@ -192,23 +192,23 @@ def get_processed_obj_queues(obj_global_id): def get_processed_obj(obj_global_id): return {'modules': get_processed_obj_modules(obj_global_id), 'queues': get_processed_obj_queues(obj_global_id)} -def add_processed_obj(obj_global_id, sha256_mess, module=None, queue=None): +def add_processed_obj(obj_global_id, m_hash, module=None, queue=None): obj_type = obj_global_id.split(':', 1)[0] new_obj = r_obj_process.sadd(f'objs:process', obj_global_id) # first process: if new_obj: r_obj_process.zadd(f'objs:process:{obj_type}', {obj_global_id: int(time.time())}) if queue: - r_obj_process.zadd(f'obj:queues:{obj_global_id}', {f'{queue}:{sha256_mess}': int(time.time())}) + r_obj_process.zadd(f'obj:queues:{obj_global_id}', {f'{queue}:{m_hash}': int(time.time())}) if module: - r_obj_process.zadd(f'obj:modules:{obj_global_id}', {f'{module}:{sha256_mess}': int(time.time())}) - r_obj_process.zrem(f'obj:queues:{obj_global_id}', f'{module}:{sha256_mess}') + r_obj_process.zadd(f'obj:modules:{obj_global_id}', {f'{module}:{m_hash}': int(time.time())}) + r_obj_process.zrem(f'obj:queues:{obj_global_id}', f'{module}:{m_hash}') -def end_processed_obj(obj_global_id, sha256_mess, module=None, queue=None): +def end_processed_obj(obj_global_id, m_hash, module=None, queue=None): if queue: - r_obj_process.zrem(f'obj:queues:{obj_global_id}', f'{queue}:{sha256_mess}') + r_obj_process.zrem(f'obj:queues:{obj_global_id}', f'{queue}:{m_hash}') if module: - r_obj_process.zrem(f'obj:modules:{obj_global_id}', f'{module}:{sha256_mess}') + r_obj_process.zrem(f'obj:modules:{obj_global_id}', f'{module}:{m_hash}') # TODO HANDLE QUEUE DELETE # process completed @@ -322,7 +322,7 @@ def save_queue_digraph(): if __name__ == '__main__': # clear_modules_queues_stats() # save_queue_digraph() - oobj_global_id = 'item::submitted/2023/06/22/submitted_f656119e-f2ea-49d7-9beb-fb97077f8fe5.gz' + oobj_global_id = 'item::submitted/2023/09/06/submitted_75fb9ff2-8c91-409d-8bd6-31769d73db8f.gz' while True: print(get_processed_obj(oobj_global_id)) time.sleep(0.5) diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index bb9054fc..62e9efc3 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -154,6 +154,11 @@ host = localhost port = 6381 db = 0 +[Redis_Process] +host = localhost +port = 6381 +db = 2 + [Redis_Mixer_Cache] host = localhost port = 6381 diff --git a/requirements.txt b/requirements.txt index 8bb16553..30cd6c3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ websockets>9.0 crcmod mmh3>2.5 ssdeep>3.3 +xxhash>3.1.0 # ZMQ zmq From fee3332edbe223106eb5a233746198fe7f174679 Mon Sep 17 00:00:00 2001 From: terrtia Date: Fri, 29 Sep 2023 15:43:37 +0200 Subject: [PATCH 109/238] fix: [tracker] delete yara rule, fix filter by object type --- bin/lib/Tracker.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 4baa3e5f..9c4702ae 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -2,6 +2,8 @@ # -*-coding:UTF-8 -* import json import os +import logging +import logging.config import re import sys import time @@ -24,11 +26,16 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from packages import Date from lib.ail_core import get_objects_tracked, get_object_all_subtypes, get_objects_retro_hunted +from lib import ail_logger from lib import ConfigLoader from lib import item_basic from lib import Tag from lib.Users import User +# LOGS +logging.config.dictConfig(ail_logger.get_config(name='modules')) +logger = logging.getLogger() + config_loader = ConfigLoader.ConfigLoader() r_cache = config_loader.get_redis_conn("Redis_Cache") @@ -561,9 +568,7 @@ class Tracker: os.remove(filepath) # Filters - filters = self.get_filters() - if not filters: - filters = get_objects_tracked() + filters = get_objects_tracked() for obj_type in filters: r_tracker.srem(f'trackers:objs:{tracker_type}:{obj_type}', tracked) r_tracker.srem(f'trackers:uuid:{tracker_type}:{tracked}', f'{self.uuid}:{obj_type}') @@ -1152,7 +1157,11 @@ def get_tracked_yara_rules(): for obj_type in get_objects_tracked(): rules = {} for tracked in _get_tracked_by_obj_type('yara', obj_type): - rules[tracked] = os.path.join(get_yara_rules_dir(), tracked) + rule = os.path.join(get_yara_rules_dir(), tracked) + if not os.path.exists(rule): + logger.critical(f"Yara rule don't exists {tracked} : {obj_type}") + else: + rules[tracked] = rule to_track[obj_type] = yara.compile(filepaths=rules) print(to_track) return to_track From fb4a74b45a49dc968bf823866e06a75fdc8f92d5 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Tue, 3 Oct 2023 11:56:01 +0200 Subject: [PATCH 110/238] fix: [dep] Pinning flask to < 3.0 due to Werkzeug 3.0 issues: https://stackoverflow.com/a/77215455 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8bb16553..8e9bb803 100644 --- a/requirements.txt +++ b/requirements.txt @@ -67,7 +67,7 @@ pylibinjection>=0.2.4 phonenumbers>8.12.1 # Web -flask>=1.1.4 +flask==2.3.3 flask-login bcrypt>3.1.6 From daf9f6fb5dfec8e1ef14714918b37ab2ee95a3ee Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 4 Oct 2023 14:40:13 +0200 Subject: [PATCH 111/238] fix: [chats] message css + reply ID --- bin/importer/feeders/Telegram.py | 2 +- var/www/templates/objects/chat/ChatMessages.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/importer/feeders/Telegram.py b/bin/importer/feeders/Telegram.py index 2cc6a127..313a8c9b 100755 --- a/bin/importer/feeders/Telegram.py +++ b/bin/importer/feeders/Telegram.py @@ -67,7 +67,7 @@ class TelegramFeeder(DefaultFeeder): meta = self.json_data['meta'] mess_id = self.json_data['meta']['id'] if meta.get('reply_to'): - reply_to_id = meta['reply_to']['id'] + reply_to_id = int(meta['reply_to']) else: reply_to_id = None diff --git a/var/www/templates/objects/chat/ChatMessages.html b/var/www/templates/objects/chat/ChatMessages.html index aa0bdf35..dd99b250 100644 --- a/var/www/templates/objects/chat/ChatMessages.html +++ b/var/www/templates/objects/chat/ChatMessages.html @@ -118,7 +118,7 @@

    {{ date }}

    {% for mess in messages[date] %} -
    +
    {{ mess['user-account']['id'] }}
    {{ mess['hour'] }}
    From eae57fb813b3664bd8bd1fb5df151ee80fc212c8 Mon Sep 17 00:00:00 2001 From: terrtia Date: Thu, 5 Oct 2023 16:24:28 +0200 Subject: [PATCH 112/238] chg: [importers obj_type] importers queues: add feeder source + object global ID --- bin/crawlers/Crawler.py | 4 ++-- bin/importer/FeederImporter.py | 10 +++++----- bin/importer/FileImporter.py | 5 +++-- bin/importer/PystemonImporter.py | 1 + bin/importer/ZMQImporter.py | 18 +++++++++++++++--- bin/importer/abstract_importer.py | 4 +++- bin/importer/feeders/Default.py | 22 ++++++++++++++++------ bin/importer/feeders/Telegram.py | 14 ++++++++------ bin/modules/Mixer.py | 25 +++++++++++-------------- bin/modules/SubmitPaste.py | 2 +- 10 files changed, 65 insertions(+), 40 deletions(-) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index d16ad9f7..adaf7bbf 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -265,11 +265,11 @@ class Crawler(AbstractModule): print(item_id) gzip64encoded = crawlers.get_gzipped_b64_item(item_id, entries['html']) # send item to Global - relay_message = f'crawler {item_id} {gzip64encoded}' + relay_message = f'crawler item::{item_id} {gzip64encoded}' self.add_message_to_queue(relay_message, 'Importers') # Tag - msg = f'infoleak:submission="crawler";{item_id}' + msg = f'infoleak:submission="crawler";{item_id}' # TODO FIXME self.add_message_to_queue(msg, 'Tags') crawlers.create_item_metadata(item_id, last_url, parent_id) diff --git a/bin/importer/FeederImporter.py b/bin/importer/FeederImporter.py index dc2dfb7d..8c8b08cb 100755 --- a/bin/importer/FeederImporter.py +++ b/bin/importer/FeederImporter.py @@ -87,16 +87,16 @@ class FeederImporter(AbstractImporter): feeder_name = feeder.get_name() print(f'importing: {feeder_name} feeder') - item_id = feeder.get_item_id() # TODO replace me with object global id + obj = feeder.get_obj() # TODO replace by a list of objects to import ???? # process meta if feeder.get_json_meta(): feeder.process_meta() - if feeder_name == 'telegram': - return item_id # TODO support UI dashboard - else: + if obj.type == 'item': # object save on disk as file (Items) gzip64_content = feeder.get_gzip64_content() - return f'{feeder_name} {item_id} {gzip64_content}' + return f'{feeder_name} {obj.get_global_id()} {gzip64_content}' + else: # Messages save on DB + return f'{feeder_name} {obj.get_global_id()}' class FeederModuleImporter(AbstractModule): diff --git a/bin/importer/FileImporter.py b/bin/importer/FileImporter.py index 4a926a41..820e7f53 100755 --- a/bin/importer/FileImporter.py +++ b/bin/importer/FileImporter.py @@ -19,7 +19,7 @@ sys.path.append(os.environ['AIL_BIN']) from importer.abstract_importer import AbstractImporter # from modules.abstract_module import AbstractModule from lib import ail_logger -from lib.ail_queues import AILQueue +# from lib.ail_queues import AILQueue from lib import ail_files # TODO RENAME ME logging.config.dictConfig(ail_logger.get_config(name='modules')) @@ -41,9 +41,10 @@ class FileImporter(AbstractImporter): gzipped = False if mimetype == 'application/gzip': gzipped = True - elif not ail_files.is_text(mimetype): + elif not ail_files.is_text(mimetype): # # # # return None + # TODO handle multiple objects message = self.create_message(item_id, content, gzipped=gzipped, source='dir_import') if message: self.add_message_to_queue(message=message) diff --git a/bin/importer/PystemonImporter.py b/bin/importer/PystemonImporter.py index 1a0e68d8..69733ed0 100755 --- a/bin/importer/PystemonImporter.py +++ b/bin/importer/PystemonImporter.py @@ -52,6 +52,7 @@ class PystemonImporter(AbstractImporter): else: gzipped = False + # TODO handle multiple objects return self.create_message(item_id, content, gzipped=gzipped, source='pystemon') except IOError as e: diff --git a/bin/importer/ZMQImporter.py b/bin/importer/ZMQImporter.py index 91728cf9..bb86880f 100755 --- a/bin/importer/ZMQImporter.py +++ b/bin/importer/ZMQImporter.py @@ -56,6 +56,8 @@ class ZMQModuleImporter(AbstractModule): super().__init__() config_loader = ConfigLoader() + self.default_feeder_name = config_loader.get_config_str("Module_Mixer", "default_unnamed_feed_name") + addresses = config_loader.get_config_str('ZMQ_Global', 'address') addresses = addresses.split(',') channel = config_loader.get_config_str('ZMQ_Global', 'channel') @@ -63,7 +65,6 @@ class ZMQModuleImporter(AbstractModule): for address in addresses: self.zmq_importer.add(address.strip(), channel) - # TODO MESSAGE SOURCE - UI def get_message(self): for message in self.zmq_importer.importer(): # remove channel from message @@ -72,8 +73,19 @@ class ZMQModuleImporter(AbstractModule): def compute(self, messages): for message in messages: message = message.decode() - print(message.split(' ', 1)[0]) - self.add_message_to_queue(message=message) + + obj_id, gzip64encoded = message.split(' ', 1) # TODO ADD LOGS + splitted = obj_id.split('>>', 1) + if splitted == 2: + feeder_name, obj_id = splitted + else: + feeder_name = self.default_feeder_name + + # f'{source} item::{obj_id} {content}' + relay_message = f'{feeder_name} item::{obj_id} {gzip64encoded}' + + print(f'feeder_name item::{obj_id}') + self.add_message_to_queue(message=relay_message) if __name__ == '__main__': diff --git a/bin/importer/abstract_importer.py b/bin/importer/abstract_importer.py index 1c4b458d..ebe32c63 100755 --- a/bin/importer/abstract_importer.py +++ b/bin/importer/abstract_importer.py @@ -98,5 +98,7 @@ class AbstractImporter(ABC): # TODO ail queues source = self.name self.logger.info(f'{source} {obj_id}') # self.logger.debug(f'{source} {obj_id} {content}') - return f'{source} {obj_id} {content}' + + # TODO handle multiple objects + return f'{source} item::{obj_id} {content}' diff --git a/bin/importer/feeders/Default.py b/bin/importer/feeders/Default.py index 482d06b4..f4313707 100755 --- a/bin/importer/feeders/Default.py +++ b/bin/importer/feeders/Default.py @@ -9,14 +9,21 @@ Process Feeder Json (example: Twitter feeder) """ import os import datetime +import sys import uuid +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.objects import ail_objects + class DefaultFeeder: """Default Feeder""" def __init__(self, json_data): self.json_data = json_data - self.item_id = None + self.obj = None self.name = None def get_name(self): @@ -52,14 +59,17 @@ class DefaultFeeder: return self.json_data.get('data') ## OVERWRITE ME ## - def get_item_id(self): + def get_obj(self): """ - Return item id. define item id + Return obj global id. define obj global id + Default == item object """ date = datetime.date.today().strftime("%Y/%m/%d") - item_id = os.path.join(self.get_name(), date, str(uuid.uuid4())) - self.item_id = f'{item_id}.gz' - return self.item_id + obj_id = os.path.join(self.get_name(), date, str(uuid.uuid4())) + obj_id = f'{obj_id}.gz' + obj_id = f'item::{obj_id}' + self.obj = ail_objects.get_obj_from_global_id(obj_id) + return self.obj ## OVERWRITE ME ## def process_meta(self): diff --git a/bin/importer/feeders/Telegram.py b/bin/importer/feeders/Telegram.py index 313a8c9b..4eea63da 100755 --- a/bin/importer/feeders/Telegram.py +++ b/bin/importer/feeders/Telegram.py @@ -17,6 +17,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from importer.feeders.Default import DefaultFeeder from lib.ConfigLoader import ConfigLoader +from lib.objects import ail_objects from lib.objects.Chats import Chat from lib.objects import Messages from lib.objects import UsersAccount @@ -25,6 +26,7 @@ from lib.objects.Usernames import Username import base64 import io import gzip + def gunzip_bytes_obj(bytes_obj): gunzipped_bytes_obj = None try: @@ -45,8 +47,7 @@ class TelegramFeeder(DefaultFeeder): super().__init__(json_data) self.name = 'telegram' - # define item id - def get_item_id(self): # TODO rename self.item_id + def get_obj(self): # TODO handle others objects -> images, pdf, ... # Get message date timestamp = self.json_data['meta']['date']['timestamp'] # TODO CREATE DEFAULT TIMESTAMP # if self.json_data['meta'].get('date'): @@ -56,8 +57,10 @@ class TelegramFeeder(DefaultFeeder): # date = datetime.date.today().strftime("%Y/%m/%d") chat_id = str(self.json_data['meta']['chat']['id']) message_id = str(self.json_data['meta']['id']) - self.item_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) # TODO rename self.item_id - return self.item_id + obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) + obj_id = f'message:telegram:{obj_id}' + self.obj = ail_objects.get_obj_from_global_id(obj_id) + return self.obj def process_meta(self): """ @@ -81,7 +84,7 @@ class TelegramFeeder(DefaultFeeder): translation = None decoded = base64.standard_b64decode(self.json_data['data']) content = gunzip_bytes_obj(decoded) - message = Messages.create(self.item_id, content, translation=translation) + message = Messages.create(self.obj.id, content, translation=translation) if meta.get('chat'): chat = Chat(meta['chat']['id'], 'telegram') @@ -131,5 +134,4 @@ class TelegramFeeder(DefaultFeeder): # TODO reply threads ???? # message edit ???? - return None diff --git a/bin/modules/Mixer.py b/bin/modules/Mixer.py index 62c427e3..7ca0985d 100755 --- a/bin/modules/Mixer.py +++ b/bin/modules/Mixer.py @@ -139,22 +139,19 @@ class Mixer(AbstractModule): def compute(self, message): self.refresh_stats() splitted = message.split() - # Old Feeder name "feeder>>item_id gzip64encoded" - if len(splitted) == 2: - item_id, gzip64encoded = splitted - try: - feeder_name, item_id = item_id.split('>>') - feeder_name.replace(" ", "") - if 'import_dir' in feeder_name: - feeder_name = feeder_name.split('/')[1] - except ValueError: - feeder_name = self.default_feeder_name - # Feeder name in message: "feeder item_id gzip64encoded" - elif len(splitted) == 3: - feeder_name, item_id, gzip64encoded = splitted + # message -> # feeder_name - object - content + # or # message -> # feeder_name - object + + # feeder_name - object + if len(splitted) == 2: # feeder_name - object (content already saved) + feeder_name, obj_id = splitted + + # Feeder name in message: "feeder obj_id gzip64encoded" + elif len(splitted) == 3: # gzip64encoded content + feeder_name, obj_id, gzip64encoded = splitted else: print('Invalid message: not processed') - self.logger.debug(f'Invalid Item: {splitted[0]} not processed') + self.logger.debug(f'Invalid Item: {splitted[0]} not processed') # TODO return None # remove absolute path diff --git a/bin/modules/SubmitPaste.py b/bin/modules/SubmitPaste.py index 740090ea..e83a0856 100755 --- a/bin/modules/SubmitPaste.py +++ b/bin/modules/SubmitPaste.py @@ -277,7 +277,7 @@ class SubmitPaste(AbstractModule): self.redis_logger.debug(f"relative path {rel_item_path}") # send paste to Global module - relay_message = f"submitted {rel_item_path} {gzip64encoded}" + relay_message = f"submitted item::{rel_item_path} {gzip64encoded}" self.add_message_to_queue(message=relay_message) # add tags From 676b0f84effea7e08196591ca0f2e56ca2865c76 Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 11 Oct 2023 12:06:01 +0200 Subject: [PATCH 113/238] chg: [module + queues] track + rename object global ID by module --- bin/core/Sync_importer.py | 9 +-- bin/core/Sync_module.py | 53 ++++++++++++++-- bin/crawlers/Crawler.py | 19 +++--- bin/importer/FeederImporter.py | 10 +-- bin/importer/FileImporter.py | 10 ++- bin/importer/PystemonImporter.py | 14 ++++- bin/importer/ZMQImporter.py | 15 ++--- bin/importer/abstract_importer.py | 40 +++++++----- bin/importer/feeders/Telegram.py | 1 + bin/lib/ail_core.py | 4 ++ bin/lib/ail_queues.py | 51 +++++++++++++-- bin/lib/objects/Items.py | 21 ++++++- bin/lib/objects/ail_objects.py | 2 + bin/modules/Global.py | 101 +++++++++++++----------------- bin/modules/Mixer.py | 43 ++++++++----- bin/modules/SubmitPaste.py | 8 ++- bin/modules/Tags.py | 3 - bin/modules/abstract_module.py | 3 +- 18 files changed, 267 insertions(+), 140 deletions(-) diff --git a/bin/core/Sync_importer.py b/bin/core/Sync_importer.py index bf70b67e..8bb11669 100755 --- a/bin/core/Sync_importer.py +++ b/bin/core/Sync_importer.py @@ -23,7 +23,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from core import ail_2_ail from modules.abstract_module import AbstractModule -# from lib.ConfigLoader import ConfigLoader +from lib.objects.Items import Item #### CONFIG #### # config_loader = ConfigLoader() @@ -76,10 +76,11 @@ class Sync_importer(AbstractModule): # # TODO: create default id item_id = ail_stream['meta']['ail:id'] + item = Item(item_id) - message = f'sync {item_id} {b64_gzip_content}' - print(item_id) - self.add_message_to_queue(message, 'Importers') + message = f'sync {b64_gzip_content}' + print(item.id) + self.add_message_to_queue(obj=item, message=message, queue='Importers') if __name__ == '__main__': diff --git a/bin/core/Sync_module.py b/bin/core/Sync_module.py index 22f814f4..857efaa3 100755 --- a/bin/core/Sync_module.py +++ b/bin/core/Sync_module.py @@ -15,17 +15,20 @@ This module . import os import sys import time +import traceback sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## from core import ail_2_ail -from lib.objects.Items import Item +from lib.ail_queues import get_processed_end_obj +from lib.exceptions import ModuleQueueError +from lib.objects import ail_objects from modules.abstract_module import AbstractModule -class Sync_module(AbstractModule): +class Sync_module(AbstractModule): # TODO KEEP A QUEUE ??????????????????????????????????????????????? """ Sync_module module for AIL framework """ @@ -53,7 +56,7 @@ class Sync_module(AbstractModule): print('sync queues refreshed') print(self.dict_sync_queues) - obj = self.get_obj() + obj = ail_objects.get_obj_from_global_id(message) tags = obj.get_tags() @@ -67,10 +70,52 @@ class Sync_module(AbstractModule): obj_dict = obj.get_default_meta() # send to queue push and/or pull for dict_ail in self.dict_sync_queues[queue_uuid]['ail_instances']: - print(f'ail_uuid: {dict_ail["ail_uuid"]} obj: {message}') + print(f'ail_uuid: {dict_ail["ail_uuid"]} obj: {obj.type}:{obj.get_subtype(r_str=True)}:{obj.id}') ail_2_ail.add_object_to_sync_queue(queue_uuid, dict_ail['ail_uuid'], obj_dict, push=dict_ail['push'], pull=dict_ail['pull']) + def run(self): + """ + Run Module endless process + """ + + # Endless loop processing messages from the input queue + while self.proceed: + # Get one message (paste) from the QueueIn (copy of Redis_Global publish) + global_id = get_processed_end_obj() + if global_id: + try: + # Module processing with the message from the queue + self.compute(global_id) + except Exception as err: + if self.debug: + self.queue.error() + raise err + + # LOG ERROR + trace = traceback.format_tb(err.__traceback__) + trace = ''.join(trace) + self.logger.critical(f"Error in module {self.module_name}: {__name__} : {err}") + self.logger.critical(f"Module {self.module_name} input message: {global_id}") + self.logger.critical(trace) + + if isinstance(err, ModuleQueueError): + self.queue.error() + raise err + # remove from set_module + ## check if item process == completed + + if self.obj: + self.queue.end_message(self.obj.get_global_id(), self.sha256_mess) + self.obj = None + self.sha256_mess = None + + else: + self.computeNone() + # Wait before next process + self.logger.debug(f"{self.module_name}, waiting for new message, Idling {self.pending_seconds}s") + time.sleep(self.pending_seconds) + if __name__ == '__main__': diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index adaf7bbf..3332299d 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -262,23 +262,24 @@ class Crawler(AbstractModule): if 'html' in entries and entries.get('html'): item_id = crawlers.create_item_id(self.items_dir, self.domain.id) - print(item_id) - gzip64encoded = crawlers.get_gzipped_b64_item(item_id, entries['html']) + item = Item(item_id) + print(item.id) + + gzip64encoded = crawlers.get_gzipped_b64_item(item.id, entries['html']) # send item to Global - relay_message = f'crawler item::{item_id} {gzip64encoded}' - self.add_message_to_queue(relay_message, 'Importers') + relay_message = f'crawler {gzip64encoded}' + self.add_message_to_queue(obj=item, message=relay_message, queue='Importers') - # Tag - msg = f'infoleak:submission="crawler";{item_id}' # TODO FIXME - self.add_message_to_queue(msg, 'Tags') + # Tag # TODO replace me with metadata to tags + msg = f'infoleak:submission="crawler"' # TODO FIXME + self.add_message_to_queue(obj=item, message=msg, queue='Tags') + # TODO replace me with metadata to add crawlers.create_item_metadata(item_id, last_url, parent_id) if self.root_item is None: self.root_item = item_id parent_id = item_id - item = Item(item_id) - title_content = crawlers.extract_title_from_html(entries['html']) if title_content: title = Titles.create_title(title_content) diff --git a/bin/importer/FeederImporter.py b/bin/importer/FeederImporter.py index 8c8b08cb..24b9bcb8 100755 --- a/bin/importer/FeederImporter.py +++ b/bin/importer/FeederImporter.py @@ -94,9 +94,9 @@ class FeederImporter(AbstractImporter): if obj.type == 'item': # object save on disk as file (Items) gzip64_content = feeder.get_gzip64_content() - return f'{feeder_name} {obj.get_global_id()} {gzip64_content}' + return obj, f'{feeder_name} {gzip64_content}' else: # Messages save on DB - return f'{feeder_name} {obj.get_global_id()}' + return obj, f'{feeder_name}' class FeederModuleImporter(AbstractModule): @@ -115,8 +115,10 @@ class FeederModuleImporter(AbstractModule): def compute(self, message): # TODO HANDLE Invalid JSON json_data = json.loads(message) - relay_message = self.importer.importer(json_data) - self.add_message_to_queue(message=relay_message) + # TODO multiple objs + messages + obj, relay_message = self.importer.importer(json_data) + #### + self.add_message_to_queue(obj=obj, message=relay_message) # Launch Importer diff --git a/bin/importer/FileImporter.py b/bin/importer/FileImporter.py index 820e7f53..d9a31ba2 100755 --- a/bin/importer/FileImporter.py +++ b/bin/importer/FileImporter.py @@ -22,6 +22,8 @@ from lib import ail_logger # from lib.ail_queues import AILQueue from lib import ail_files # TODO RENAME ME +from lib.objects.Items import Item + logging.config.dictConfig(ail_logger.get_config(name='modules')) class FileImporter(AbstractImporter): @@ -44,10 +46,12 @@ class FileImporter(AbstractImporter): elif not ail_files.is_text(mimetype): # # # # return None - # TODO handle multiple objects - message = self.create_message(item_id, content, gzipped=gzipped, source='dir_import') + source = 'dir_import' + message = self.create_message(content, gzipped=gzipped, source=source) + self.logger.info(f'{source} {item_id}') + obj = Item(item_id) if message: - self.add_message_to_queue(message=message) + self.add_message_to_queue(obj, message=message) class DirImporter(AbstractImporter): def __init__(self): diff --git a/bin/importer/PystemonImporter.py b/bin/importer/PystemonImporter.py index 69733ed0..1c0692b9 100755 --- a/bin/importer/PystemonImporter.py +++ b/bin/importer/PystemonImporter.py @@ -22,6 +22,8 @@ from importer.abstract_importer import AbstractImporter from modules.abstract_module import AbstractModule from lib.ConfigLoader import ConfigLoader +from lib.objects.Items import Item + class PystemonImporter(AbstractImporter): def __init__(self, pystemon_dir, host='localhost', port=6379, db=10): super().__init__() @@ -53,10 +55,13 @@ class PystemonImporter(AbstractImporter): gzipped = False # TODO handle multiple objects - return self.create_message(item_id, content, gzipped=gzipped, source='pystemon') + source = 'pystemon' + message = self.create_message(content, gzipped=gzipped, source=source) + self.logger.info(f'{source} {item_id}') + return item_id, message except IOError as e: - print(f'Error: {full_item_path}, IOError') + self.logger.error(f'Error {e}: {full_item_path}, IOError') return None @@ -80,7 +85,10 @@ class PystemonModuleImporter(AbstractModule): return self.importer.importer() def compute(self, message): - self.add_message_to_queue(message=message) + if message: + item_id, message = message + item = Item(item_id) + self.add_message_to_queue(obj=item, message=message) if __name__ == '__main__': diff --git a/bin/importer/ZMQImporter.py b/bin/importer/ZMQImporter.py index bb86880f..509b136e 100755 --- a/bin/importer/ZMQImporter.py +++ b/bin/importer/ZMQImporter.py @@ -4,15 +4,13 @@ Importer Class ================ -Import Content +ZMQ Importer """ import os import sys - import zmq - sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages @@ -21,6 +19,8 @@ from importer.abstract_importer import AbstractImporter from modules.abstract_module import AbstractModule from lib.ConfigLoader import ConfigLoader +from lib.objects.Items import Item + class ZMQImporters(AbstractImporter): def __init__(self): super().__init__() @@ -74,18 +74,19 @@ class ZMQModuleImporter(AbstractModule): for message in messages: message = message.decode() - obj_id, gzip64encoded = message.split(' ', 1) # TODO ADD LOGS + obj_id, gzip64encoded = message.split(' ', 1) # TODO ADD LOGS splitted = obj_id.split('>>', 1) if splitted == 2: feeder_name, obj_id = splitted else: feeder_name = self.default_feeder_name - # f'{source} item::{obj_id} {content}' - relay_message = f'{feeder_name} item::{obj_id} {gzip64encoded}' + obj = Item(obj_id) + # f'{source} {content}' + relay_message = f'{feeder_name} {gzip64encoded}' print(f'feeder_name item::{obj_id}') - self.add_message_to_queue(message=relay_message) + self.add_message_to_queue(obj=obj, message=relay_message) if __name__ == '__main__': diff --git a/bin/importer/abstract_importer.py b/bin/importer/abstract_importer.py index ebe32c63..11cb0f67 100755 --- a/bin/importer/abstract_importer.py +++ b/bin/importer/abstract_importer.py @@ -54,16 +54,22 @@ class AbstractImporter(ABC): # TODO ail queues """ return self.__class__.__name__ - def add_message_to_queue(self, message, queue_name=None): + def add_message_to_queue(self, obj, message='', queue=None): """ Add message to queue + :param obj: AILObject :param message: message to send in queue - :param queue_name: queue or module name + :param queue: queue name or module name ex: add_message_to_queue(item_id, 'Mail') """ - if message: - self.queue.send_message(message, queue_name) + if not obj: + raise Exception(f'Invalid AIL object, {obj}') + obj_global_id = obj.get_global_id() + self.queue.send_message(obj_global_id, message, queue) + + def get_available_queues(self): + return self.queue.get_out_queues() @staticmethod def b64(content): @@ -85,20 +91,20 @@ class AbstractImporter(ABC): # TODO ail queues self.logger.warning(e) return '' - def create_message(self, obj_id, content, b64=False, gzipped=False, source=None): - if not gzipped: - content = self.b64_gzip(content) - elif not b64: - content = self.b64(content) - if not content: - return None - if isinstance(content, bytes): - content = content.decode() + def create_message(self, content, b64=False, gzipped=False, source=None): if not source: source = self.name - self.logger.info(f'{source} {obj_id}') - # self.logger.debug(f'{source} {obj_id} {content}') - # TODO handle multiple objects - return f'{source} item::{obj_id} {content}' + if content: + if not gzipped: + content = self.b64_gzip(content) + elif not b64: + content = self.b64(content) + if not content: + return None + if isinstance(content, bytes): + content = content.decode() + return f'{source} {content}' + else: + return f'{source}' diff --git a/bin/importer/feeders/Telegram.py b/bin/importer/feeders/Telegram.py index 4eea63da..5ef58b5f 100755 --- a/bin/importer/feeders/Telegram.py +++ b/bin/importer/feeders/Telegram.py @@ -27,6 +27,7 @@ import base64 import io import gzip +# TODO remove compression ??? def gunzip_bytes_obj(bytes_obj): gunzipped_bytes_obj = None try: diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index eeb83a98..9eaaca97 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -93,6 +93,10 @@ def zscan_iter(r_redis, name): # count ??? ## -- Redis -- ## +def rreplace(s, old, new, occurrence): + li = s.rsplit(old, occurrence) + return new.join(li) + def paginate_iterator(iter_elems, nb_obj=50, page=1): dict_page = {'nb_all_elem': len(iter_elems)} nb_pages = dict_page['nb_all_elem'] / nb_obj diff --git a/bin/lib/ail_queues.py b/bin/lib/ail_queues.py index 38c5a42d..06ad6b0d 100755 --- a/bin/lib/ail_queues.py +++ b/bin/lib/ail_queues.py @@ -84,6 +84,18 @@ class AILQueue: add_processed_obj(obj_global_id, m_hash, module=self.name) return obj_global_id, m_hash, mess + def rename_message_obj(self, new_id, old_id): + # restrict rename function + if self.name == 'Mixer' or self.name == 'Global': + rename_processed_obj(new_id, old_id) + else: + raise ModuleQueueError('This Module can\'t rename an object ID') + + # condition -> not in any queue + # TODO EDIT meta + + + def end_message(self, obj_global_id, m_hash): end_processed_obj(obj_global_id, m_hash, module=self.name) @@ -171,6 +183,12 @@ def clear_modules_queues_stats(): def get_processed_objs(): return r_obj_process.smembers(f'objs:process') +def get_processed_end_objs(): + return r_obj_process.smembers(f'objs:processed') + +def get_processed_end_obj(): + return r_obj_process.spop(f'objs:processed') + def get_processed_objs_by_type(obj_type): return r_obj_process.zrange(f'objs:process:{obj_type}', 0, -1) @@ -219,6 +237,28 @@ def end_processed_obj(obj_global_id, m_hash, module=None, queue=None): r_obj_process.sadd(f'objs:processed', obj_global_id) # TODO use list ?????? +def rename_processed_obj(new_id, old_id): + module = get_processed_obj_modules(old_id) + # currently in a module + if len(module) == 1: + module, x_hash = module[0].split(':', 1) + obj_type = old_id.split(':', 1)[0] + r_obj_process.zrem(f'obj:modules:{old_id}', f'{module}:{x_hash}') + r_obj_process.zrem(f'objs:process:{obj_type}', old_id) + r_obj_process.srem(f'objs:process', old_id) + add_processed_obj(new_id, x_hash, module=module) + +def delete_processed_obj(obj_global_id): + for q in get_processed_obj_queues(obj_global_id): + queue, x_hash = q.split(':', 1) + r_obj_process.zrem(f'obj:queues:{obj_global_id}', f'{queue}:{x_hash}') + for m in get_processed_obj_modules(obj_global_id): + module, x_hash = m.split(':', 1) + r_obj_process.zrem(f'obj:modules:{obj_global_id}', f'{module}:{x_hash}') + obj_type = obj_global_id.split(':', 1)[0] + r_obj_process.zrem(f'objs:process:{obj_type}', obj_global_id) + r_obj_process.srem(f'objs:process', obj_global_id) + ################################################################################### @@ -322,7 +362,10 @@ def save_queue_digraph(): if __name__ == '__main__': # clear_modules_queues_stats() # save_queue_digraph() - oobj_global_id = 'item::submitted/2023/09/06/submitted_75fb9ff2-8c91-409d-8bd6-31769d73db8f.gz' - while True: - print(get_processed_obj(oobj_global_id)) - time.sleep(0.5) + oobj_global_id = 'item::submitted/2023/10/11/submitted_b5440009-05d5-4494-a807-a6d8e4a900cf.gz' + # print(get_processed_obj(oobj_global_id)) + # delete_processed_obj(oobj_global_id) + # while True: + # print(get_processed_obj(oobj_global_id)) + # time.sleep(0.5) + print(get_processed_end_objs()) diff --git a/bin/lib/objects/Items.py b/bin/lib/objects/Items.py index c2edbb40..8aaab0b2 100755 --- a/bin/lib/objects/Items.py +++ b/bin/lib/objects/Items.py @@ -11,6 +11,7 @@ import cld3 import html2text from io import BytesIO +from uuid import uuid4 from pymisp import MISPObject @@ -18,7 +19,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## -from lib.ail_core import get_ail_uuid +from lib.ail_core import get_ail_uuid, rreplace from lib.objects.abstract_object import AbstractObject from lib.ConfigLoader import ConfigLoader from lib import item_basic @@ -137,9 +138,23 @@ class Item(AbstractObject): #################################################################################### #################################################################################### - def sanitize_id(self): - pass + # TODO ADD function to check if ITEM (content + file) already exists + def sanitize_id(self): + if ITEMS_FOLDER in self.id: + self.id = self.id.replace(ITEMS_FOLDER, '', 1) + + # limit filename length + basename = self.get_basename() + if len(basename) > 255: + new_basename = f'{basename[:215]}{str(uuid4())}.gz' + self.id = rreplace(self.id, basename, new_basename, 1) + + + + + + return self.id # # TODO: sanitize_id # # TODO: check if already exists ? diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 89be336f..0c29d668 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -84,6 +84,8 @@ def get_object(obj_type, subtype, obj_id): return UserAccount(obj_id, subtype) elif obj_type == 'username': return Usernames.Username(obj_id, subtype) + else: + raise Exception(f'Unknown AIL object: {obj_type} {subtype} {obj_id}') def get_objects(objects): objs = set() diff --git a/bin/modules/Global.py b/bin/modules/Global.py index 1c05fcf3..dd3cd900 100755 --- a/bin/modules/Global.py +++ b/bin/modules/Global.py @@ -79,73 +79,56 @@ class Global(AbstractModule): self.time_last_stats = time.time() self.processed_item = 0 - def compute(self, message, r_result=False): - # Recovering the streamed message informations - splitted = message.split() + def compute(self, message, r_result=False): # TODO move OBJ ID sanitization to importer + # Recovering the streamed message infos + gzip64encoded = message - if len(splitted) == 2: - item, gzip64encoded = splitted + if self.obj.type == 'item': + if gzip64encoded: - # Remove ITEMS_FOLDER from item path (crawled item + submitted) - if self.ITEMS_FOLDER in item: - item = item.replace(self.ITEMS_FOLDER, '', 1) + # Creating the full filepath + filename = os.path.join(self.ITEMS_FOLDER, self.obj.id) + filename = os.path.realpath(filename) - file_name_item = item.split('/')[-1] - if len(file_name_item) > 255: - new_file_name_item = '{}{}.gz'.format(file_name_item[:215], str(uuid4())) - item = self.rreplace(item, file_name_item, new_file_name_item, 1) + # Incorrect filename + if not os.path.commonprefix([filename, self.ITEMS_FOLDER]) == self.ITEMS_FOLDER: + self.logger.warning(f'Global; Path traversal detected {filename}') + print(f'Global; Path traversal detected {filename}') - # Creating the full filepath - filename = os.path.join(self.ITEMS_FOLDER, item) - filename = os.path.realpath(filename) + else: + # Decode compressed base64 + decoded = base64.standard_b64decode(gzip64encoded) + new_file_content = self.gunzip_bytes_obj(filename, decoded) - # Incorrect filename - if not os.path.commonprefix([filename, self.ITEMS_FOLDER]) == self.ITEMS_FOLDER: - self.logger.warning(f'Global; Path traversal detected {filename}') - print(f'Global; Path traversal detected {filename}') + # TODO REWRITE ME + if new_file_content: + filename = self.check_filename(filename, new_file_content) + + if filename: + # create subdir + dirname = os.path.dirname(filename) + if not os.path.exists(dirname): + os.makedirs(dirname) + + with open(filename, 'wb') as f: + f.write(decoded) + + update_obj_date(self.obj.get_date(), 'item') + + self.add_message_to_queue(obj=self.obj, queue='Item') + self.processed_item += 1 + + print(self.obj.id) + if r_result: + return self.obj.id else: - # Decode compressed base64 - decoded = base64.standard_b64decode(gzip64encoded) - new_file_content = self.gunzip_bytes_obj(filename, decoded) - - if new_file_content: - filename = self.check_filename(filename, new_file_content) - - if filename: - # create subdir - dirname = os.path.dirname(filename) - if not os.path.exists(dirname): - os.makedirs(dirname) - - with open(filename, 'wb') as f: - f.write(decoded) - - item_id = filename - # remove self.ITEMS_FOLDER from - if self.ITEMS_FOLDER in item_id: - item_id = item_id.replace(self.ITEMS_FOLDER, '', 1) - - item = Item(item_id) - - update_obj_date(item.get_date(), 'item') - - self.add_message_to_queue(obj=item, queue='Item') - self.processed_item += 1 - - # DIRTY FIX AIL SYNC - SEND TO SYNC MODULE - # # FIXME: DIRTY FIX - message = f'{item.get_type()};{item.get_subtype(r_str=True)};{item.get_id()}' - print(message) - self.add_message_to_queue(obj=item, queue='Sync') - - print(item_id) - if r_result: - return item_id - + self.logger.info(f"Empty Item: {message} not processed") + elif self.obj: + # TODO send to specific object queue => image, ... + self.add_message_to_queue(obj=self.obj, queue='Item') else: - self.logger.debug(f"Empty Item: {message} not processed") - print(f"Empty Item: {message} not processed") + self.logger.critical(f"Empty obj: {self.obj} {message} not processed") def check_filename(self, filename, new_file_content): """ diff --git a/bin/modules/Mixer.py b/bin/modules/Mixer.py index 7ca0985d..8d9d513c 100755 --- a/bin/modules/Mixer.py +++ b/bin/modules/Mixer.py @@ -9,7 +9,7 @@ This module is consuming the Redis-list created by the ZMQ_Feed_Q Module. This module take all the feeds provided in the config. -Depending on the configuration, this module will process the feed as follow: +Depending on the configuration, this module will process the feed as follows: operation_mode 1: "Avoid any duplicate from any sources" - The module maintain a list of content for each item - If the content is new, process it @@ -64,9 +64,6 @@ class Mixer(AbstractModule): self.ttl_key = config_loader.get_config_int("Module_Mixer", "ttl_duplicate") self.default_feeder_name = config_loader.get_config_str("Module_Mixer", "default_unnamed_feed_name") - self.ITEMS_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/' - self.ITEMS_FOLDER = os.path.join(os.path.realpath(self.ITEMS_FOLDER), '') - self.nb_processed_items = 0 self.feeders_processed = {} self.feeders_duplicate = {} @@ -138,27 +135,38 @@ class Mixer(AbstractModule): def compute(self, message): self.refresh_stats() + # obj = self.obj + # TODO CHECK IF NOT self.object -> get object global ID from message + splitted = message.split() - # message -> # feeder_name - object - content - # or # message -> # feeder_name - object + # message -> feeder_name - content + # or message -> feeder_name # feeder_name - object - if len(splitted) == 2: # feeder_name - object (content already saved) - feeder_name, obj_id = splitted + if len(splitted) == 1: # feeder_name - object (content already saved) + feeder_name = message + gzip64encoded = None # Feeder name in message: "feeder obj_id gzip64encoded" - elif len(splitted) == 3: # gzip64encoded content - feeder_name, obj_id, gzip64encoded = splitted + elif len(splitted) == 2: # gzip64encoded content + feeder_name, gzip64encoded = splitted else: - print('Invalid message: not processed') - self.logger.debug(f'Invalid Item: {splitted[0]} not processed') # TODO + self.logger.warning(f'Invalid Message: {splitted} not processed') return None - # remove absolute path - item_id = item_id.replace(self.ITEMS_FOLDER, '', 1) + if self.obj.type == 'item': + # Remove ITEMS_FOLDER from item path (crawled item + submitted) + # Limit basename length + obj_id = self.obj.id + self.obj.sanitize_id() + if self.obj.id != obj_id: + self.queue.rename_message_obj(self.obj.id, obj_id) - relay_message = f'{item_id} {gzip64encoded}' + relay_message = gzip64encoded + # print(relay_message) + + # TODO only work for item object # Avoid any duplicate coming from any sources if self.operation_mode == 1: digest = hashlib.sha1(gzip64encoded.encode('utf8')).hexdigest() @@ -207,7 +215,10 @@ class Mixer(AbstractModule): # No Filtering else: self.increase_stat_processed(feeder_name) - self.add_message_to_queue(relay_message) + if self.obj.type == 'item': + self.add_message_to_queue(obj=self.obj, message=gzip64encoded) + else: + self.add_message_to_queue(obj=self.obj) if __name__ == "__main__": diff --git a/bin/modules/SubmitPaste.py b/bin/modules/SubmitPaste.py index e83a0856..4e264a73 100755 --- a/bin/modules/SubmitPaste.py +++ b/bin/modules/SubmitPaste.py @@ -25,7 +25,7 @@ from modules.abstract_module import AbstractModule from lib.objects.Items import ITEMS_FOLDER from lib import ConfigLoader from lib import Tag - +from lib.objects.Items import Item class SubmitPaste(AbstractModule): """ @@ -276,9 +276,11 @@ class SubmitPaste(AbstractModule): rel_item_path = save_path.replace(self.PASTES_FOLDER, '', 1) self.redis_logger.debug(f"relative path {rel_item_path}") + item = Item(rel_item_path) + # send paste to Global module - relay_message = f"submitted item::{rel_item_path} {gzip64encoded}" - self.add_message_to_queue(message=relay_message) + relay_message = f"submitted {gzip64encoded}" + self.add_message_to_queue(obj=item, message=relay_message) # add tags for tag in ltags: diff --git a/bin/modules/Tags.py b/bin/modules/Tags.py index 760cb138..33ea1c80 100755 --- a/bin/modules/Tags.py +++ b/bin/modules/Tags.py @@ -46,9 +46,6 @@ class Tags(AbstractModule): # Forward message to channel self.add_message_to_queue(message=tag, queue='Tag_feed') - self.add_message_to_queue(queue='Sync') - - if __name__ == '__main__': module = Tags() module.run() diff --git a/bin/modules/abstract_module.py b/bin/modules/abstract_module.py index 05e253d5..ed2fe7d3 100644 --- a/bin/modules/abstract_module.py +++ b/bin/modules/abstract_module.py @@ -96,7 +96,8 @@ class AbstractModule(ABC): self.obj = None return None - def add_message_to_queue(self, message='', obj=None, queue=None): + # TODO ADD META OBJ ???? + def add_message_to_queue(self, obj=None, message='', queue=None): """ Add message to queue :param obj: AILObject From 623ba455ff40e7c1012d4b28d0b6f5064576cdd0 Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 11 Oct 2023 14:31:13 +0200 Subject: [PATCH 114/238] fix: [queues] fix ended duplicate + sync queue --- bin/core/Sync_module.py | 13 +++---------- bin/lib/ail_queues.py | 2 +- configs/modules.cfg | 11 +++++------ 3 files changed, 9 insertions(+), 17 deletions(-) diff --git a/bin/core/Sync_module.py b/bin/core/Sync_module.py index 857efaa3..c6bdfeeb 100755 --- a/bin/core/Sync_module.py +++ b/bin/core/Sync_module.py @@ -33,8 +33,8 @@ class Sync_module(AbstractModule): # TODO KEEP A QUEUE ????????????????????????? Sync_module module for AIL framework """ - def __init__(self): - super(Sync_module, self).__init__() + def __init__(self, queue=False): # FIXME MODIFY/ADD QUEUE + super(Sync_module, self).__init__(queue=queue) # Waiting time in seconds between to message processed self.pending_seconds = 10 @@ -102,13 +102,6 @@ class Sync_module(AbstractModule): # TODO KEEP A QUEUE ????????????????????????? if isinstance(err, ModuleQueueError): self.queue.error() raise err - # remove from set_module - ## check if item process == completed - - if self.obj: - self.queue.end_message(self.obj.get_global_id(), self.sha256_mess) - self.obj = None - self.sha256_mess = None else: self.computeNone() @@ -119,5 +112,5 @@ class Sync_module(AbstractModule): # TODO KEEP A QUEUE ????????????????????????? if __name__ == '__main__': - module = Sync_module() + module = Sync_module(queue=False) # FIXME MODIFY/ADD QUEUE module.run() diff --git a/bin/lib/ail_queues.py b/bin/lib/ail_queues.py index 06ad6b0d..9ce647e0 100755 --- a/bin/lib/ail_queues.py +++ b/bin/lib/ail_queues.py @@ -199,7 +199,7 @@ def is_processed_obj_moduled(obj_global_id): return r_obj_process.exists(f'obj:modules:{obj_global_id}') def is_processed_obj(obj_global_id): - return is_processed_obj_queued(obj_global_id) and is_processed_obj_moduled(obj_global_id) + return is_processed_obj_queued(obj_global_id) or is_processed_obj_moduled(obj_global_id) def get_processed_obj_modules(obj_global_id): return r_obj_process.zrange(f'obj:modules:{obj_global_id}', 0, -1) diff --git a/configs/modules.cfg b/configs/modules.cfg index b0b1f6df..d64f6431 100644 --- a/configs/modules.cfg +++ b/configs/modules.cfg @@ -24,7 +24,7 @@ publish = Importers,Tags [Global] subscribe = SaveObj -publish = Item,Sync +publish = Item [Duplicates] subscribe = Duplicate @@ -108,11 +108,7 @@ publish = Tags [Tags] subscribe = Tags -publish = Tag_feed,Sync - -# dirty fix -[Sync_module] -subscribe = Sync +publish = Tag_feed [MISP_Thehive_Auto_Push] subscribe = Tag_feed @@ -165,6 +161,9 @@ publish = Tags [Zerobins] subscribe = Url +#[Sync_module] +#publish = Sync + # [My_Module_Name] # subscribe = Global # Queue name # publish = Tags # Queue name From 6978764b022ed64102174ba590aa74a2fea89f8e Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 11 Oct 2023 14:53:12 +0200 Subject: [PATCH 115/238] fix: [module] fix module obj type: language + mail --- bin/modules/Languages.py | 12 +++++++----- bin/modules/Mail.py | 6 +++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/bin/modules/Languages.py b/bin/modules/Languages.py index b0b87230..53c3c999 100755 --- a/bin/modules/Languages.py +++ b/bin/modules/Languages.py @@ -25,11 +25,13 @@ class Languages(AbstractModule): self.logger.info(f'Module {self.module_name} initialized') def compute(self, message): - item = self.get_obj() - if item.is_crawled(): - domain = Domain(item.get_domain()) - for lang in item.get_languages(min_probability=0.8): - domain.add_language(lang.language) + obj = self.get_obj() + + if obj.type == 'item': + if item.is_crawled(): + domain = Domain(item.get_domain()) + for lang in item.get_languages(min_probability=0.8): + domain.add_language(lang.language) if __name__ == '__main__': diff --git a/bin/modules/Mail.py b/bin/modules/Mail.py index bbdbcfce..82cac546 100755 --- a/bin/modules/Mail.py +++ b/bin/modules/Mail.py @@ -139,7 +139,7 @@ class Mail(AbstractModule): item = self.get_obj() item_date = item.get_date() - mails = self.regex_findall(self.email_regex, item_id, item.get_content()) + mails = self.regex_findall(self.email_regex, item.id, item.get_content()) mxdomains_email = {} for mail in mails: mxdomain = mail.rsplit('@', 1)[1].lower() @@ -172,9 +172,9 @@ class Mail(AbstractModule): # for tld in mx_tlds: # Statistics.add_module_tld_stats_by_date('mail', item_date, tld, mx_tlds[tld]) - msg = f'Mails;{item.get_source()};{item_date};{item.get_basename()};Checked {num_valid_email} e-mail(s);{item_id}' + msg = f'Mails;{item.get_source()};{item_date};{item.get_basename()};Checked {num_valid_email} e-mail(s);{item.id}' if num_valid_email > self.mail_threshold: - print(f'{item_id} Checked {num_valid_email} e-mail(s)') + print(f'{item.id} Checked {num_valid_email} e-mail(s)') self.redis_logger.warning(msg) # Tags tag = 'infoleak:automatic-detection="mail"' From 912511976450ad27219fac88d3cc4c77b7f714a7 Mon Sep 17 00:00:00 2001 From: terrtia Date: Thu, 2 Nov 2023 16:28:33 +0100 Subject: [PATCH 116/238] chg: [chats] add chats explorer v0 --- bin/importer/FeederImporter.py | 2 + bin/importer/feeders/Telegram.py | 116 +------ bin/importer/feeders/abstract_chats_feeder.py | 249 ++++++++++++++ bin/lib/ail_core.py | 7 +- bin/lib/chats_viewer.py | 305 ++++++++++++++++++ bin/lib/objects/ChatSubChannels.py | 153 +++++++++ bin/lib/objects/ChatThreads.py | 103 ++++++ bin/lib/objects/Chats.py | 123 ++----- bin/lib/objects/Messages.py | 22 +- bin/lib/objects/abstract_chat_object.py | 241 ++++++++++++++ bin/lib/objects/abstract_object.py | 7 +- bin/lib/objects/abstract_subtype_object.py | 7 +- var/www/Flask_server.py | 4 +- var/www/blueprints/chats_explorer.py | 132 ++++++++ var/www/blueprints/objects_chat.py | 58 ---- .../templates/chats_explorer/ChatMessage.html | 209 ++++++++++++ .../chat => chats_explorer}/ChatMessages.html | 20 +- .../chats_explorer/SubChannelMessages.html | 211 ++++++++++++ .../chats_explorer/chat_instance.html | 123 +++++++ .../templates/chats_explorer/chat_viewer.html | 133 ++++++++ .../chats_explorer/chats_instance.html | 80 +++++ .../chats_explorer/chats_protocols.html | 80 +++++ 22 files changed, 2109 insertions(+), 276 deletions(-) create mode 100755 bin/importer/feeders/abstract_chats_feeder.py create mode 100755 bin/lib/chats_viewer.py create mode 100755 bin/lib/objects/ChatSubChannels.py create mode 100755 bin/lib/objects/ChatThreads.py create mode 100755 bin/lib/objects/abstract_chat_object.py create mode 100644 var/www/blueprints/chats_explorer.py delete mode 100644 var/www/blueprints/objects_chat.py create mode 100644 var/www/templates/chats_explorer/ChatMessage.html rename var/www/templates/{objects/chat => chats_explorer}/ChatMessages.html (90%) create mode 100644 var/www/templates/chats_explorer/SubChannelMessages.html create mode 100644 var/www/templates/chats_explorer/chat_instance.html create mode 100644 var/www/templates/chats_explorer/chat_viewer.html create mode 100644 var/www/templates/chats_explorer/chats_instance.html create mode 100644 var/www/templates/chats_explorer/chats_protocols.html diff --git a/bin/importer/FeederImporter.py b/bin/importer/FeederImporter.py index 24b9bcb8..cc1e3ea8 100755 --- a/bin/importer/FeederImporter.py +++ b/bin/importer/FeederImporter.py @@ -56,6 +56,8 @@ class FeederImporter(AbstractImporter): feeders = [f[:-3] for f in os.listdir(feeder_dir) if os.path.isfile(os.path.join(feeder_dir, f))] self.feeders = {} for feeder in feeders: + if feeder == 'abstract_chats_feeder': + continue print(feeder) part = feeder.split('.')[-1] # import json importer class diff --git a/bin/importer/feeders/Telegram.py b/bin/importer/feeders/Telegram.py index 5ef58b5f..8764aa8b 100755 --- a/bin/importer/feeders/Telegram.py +++ b/bin/importer/feeders/Telegram.py @@ -15,7 +15,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## -from importer.feeders.Default import DefaultFeeder +from importer.feeders.abstract_chats_feeder import AbstractChatFeeder from lib.ConfigLoader import ConfigLoader from lib.objects import ail_objects from lib.objects.Chats import Chat @@ -24,115 +24,15 @@ from lib.objects import UsersAccount from lib.objects.Usernames import Username import base64 -import io -import gzip -# TODO remove compression ??? -def gunzip_bytes_obj(bytes_obj): - gunzipped_bytes_obj = None - try: - in_ = io.BytesIO() - in_.write(bytes_obj) - in_.seek(0) - - with gzip.GzipFile(fileobj=in_, mode='rb') as fo: - gunzipped_bytes_obj = fo.read() - except Exception as e: - print(f'Global; Invalid Gzip file: {e}') - - return gunzipped_bytes_obj - -class TelegramFeeder(DefaultFeeder): +class TelegramFeeder(AbstractChatFeeder): def __init__(self, json_data): - super().__init__(json_data) - self.name = 'telegram' + super().__init__('telegram', json_data) - def get_obj(self): # TODO handle others objects -> images, pdf, ... - # Get message date - timestamp = self.json_data['meta']['date']['timestamp'] # TODO CREATE DEFAULT TIMESTAMP - # if self.json_data['meta'].get('date'): - # date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp']) - # date = date.strftime('%Y/%m/%d') - # else: - # date = datetime.date.today().strftime("%Y/%m/%d") - chat_id = str(self.json_data['meta']['chat']['id']) - message_id = str(self.json_data['meta']['id']) - obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) - obj_id = f'message:telegram:{obj_id}' - self.obj = ail_objects.get_obj_from_global_id(obj_id) - return self.obj + # def get_obj(self):. + # obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) + # obj_id = f'message:telegram:{obj_id}' + # self.obj = ail_objects.get_obj_from_global_id(obj_id) + # return self.obj - def process_meta(self): - """ - Process JSON meta field. - """ - # message chat - meta = self.json_data['meta'] - mess_id = self.json_data['meta']['id'] - if meta.get('reply_to'): - reply_to_id = int(meta['reply_to']) - else: - reply_to_id = None - - timestamp = meta['date']['timestamp'] - date = datetime.datetime.fromtimestamp(timestamp) - date = date.strftime('%Y%m%d') - - if self.json_data.get('translation'): - translation = self.json_data['translation'] - else: - translation = None - decoded = base64.standard_b64decode(self.json_data['data']) - content = gunzip_bytes_obj(decoded) - message = Messages.create(self.obj.id, content, translation=translation) - - if meta.get('chat'): - chat = Chat(meta['chat']['id'], 'telegram') - - if meta['chat'].get('username'): - chat_username = Username(meta['chat']['username'], 'telegram') - chat.update_username_timeline(chat_username.get_global_id(), timestamp) - - # Chat---Message - chat.add(date) - chat.add_message(message.get_global_id(), timestamp, mess_id, reply_id=reply_to_id) - else: - chat = None - - # message sender - if meta.get('sender'): # TODO handle message channel forward - check if is user - user_id = meta['sender']['id'] - user_account = UsersAccount.UserAccount(user_id, 'telegram') - # UserAccount---Message - user_account.add(date, obj=message) - # UserAccount---Chat - user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id) - - if meta['sender'].get('firstname'): - user_account.set_first_name(meta['sender']['firstname']) - if meta['sender'].get('lastname'): - user_account.set_last_name(meta['sender']['lastname']) - if meta['sender'].get('phone'): - user_account.set_phone(meta['sender']['phone']) - - if meta['sender'].get('username'): - username = Username(meta['sender']['username'], 'telegram') - # TODO timeline or/and correlation ???? - user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id) - user_account.update_username_timeline(username.get_global_id(), timestamp) - - # Username---Message - username.add(date) # TODO # correlation message ??? - - # if chat: # TODO Chat---Username correlation ??? - # # Chat---Username - # chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id) - - # if meta.get('fwd_from'): - # if meta['fwd_from'].get('post_author') # user first name - - # TODO reply threads ???? - # message edit ???? - - return None diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py new file mode 100755 index 00000000..dfc559b7 --- /dev/null +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +Abstract Chat JSON Feeder Importer Module +================ + +Process Feeder Json (example: Twitter feeder) + +""" +import datetime +import os +import sys + +from abc import abstractmethod, ABC + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from importer.feeders.Default import DefaultFeeder +from lib.objects.Chats import Chat +from lib.objects import ChatSubChannels +from lib.objects import Messages +from lib.objects import UsersAccount +from lib.objects.Usernames import Username +from lib import chats_viewer + +import base64 +import io +import gzip + +# TODO remove compression ??? +def _gunzip_bytes_obj(bytes_obj): + gunzipped_bytes_obj = None + try: + in_ = io.BytesIO() + in_.write(bytes_obj) + in_.seek(0) + + with gzip.GzipFile(fileobj=in_, mode='rb') as fo: + gunzipped_bytes_obj = fo.read() + except Exception as e: + print(f'Global; Invalid Gzip file: {e}') + + return gunzipped_bytes_obj + +class AbstractChatFeeder(DefaultFeeder, ABC): + + def __init__(self, name, json_data): + super().__init__(json_data) + self.obj = None + self.name = name + + def get_chat_protocol(self): # TODO # # # # # # # # # # # # # + return self.name + + def get_chat_network(self): + self.json_data['meta'].get('network', None) + + def get_chat_address(self): + self.json_data['meta'].get('address', None) + + def get_chat_instance_uuid(self): + chat_instance_uuid = chats_viewer.create_chat_service_instance(self.get_chat_protocol(), + network=self.get_chat_network(), + address=self.get_chat_address()) + # TODO SET + return chat_instance_uuid + + def get_chat_id(self): # TODO RAISE ERROR IF NONE + return self.json_data['meta']['chat']['id'] + + def get_channel_id(self): + pass + + def get_subchannels(self): + pass + + def get_thread_id(self): + pass + + def get_message_timestamp(self): + return self.json_data['meta']['date']['timestamp'] # TODO CREATE DEFAULT TIMESTAMP + # if self.json_data['meta'].get('date'): + # date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp']) + # date = date.strftime('%Y/%m/%d') + # else: + # date = datetime.date.today().strftime("%Y/%m/%d") + + def get_message_date_timestamp(self): + timestamp = self.get_message_timestamp() + date = datetime.datetime.fromtimestamp(timestamp) + date = date.strftime('%Y%m%d') + return date, timestamp + + def get_message_sender_id(self): + return self.json_data['meta']['sender']['id'] + + def get_message_reply(self): + return self.json_data['meta'].get('reply_to') # TODO change to reply ??? + + def get_message_reply_id(self): + return self.json_data['meta'].get('reply_to', None) + + def get_message_content(self): + decoded = base64.standard_b64decode(self.json_data['data']) + return _gunzip_bytes_obj(decoded) + + def get_obj(self): # TODO handle others objects -> images, pdf, ... + #### TIMESTAMP #### + timestamp = self.get_message_timestamp() + + #### Create Object ID #### + chat_id = str(self.json_data['meta']['chat']['id']) + message_id = str(self.json_data['meta']['id']) + # channel id + # thread id + + obj_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp) + self.obj = Messages.Message(obj_id) + return self.obj + + def process_chat(self, message, date, timestamp, reply_id=None): # TODO threads + meta = self.json_data['meta']['chat'] + chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid()) + chat.add(date) # TODO ### Dynamic subtype ??? + + if meta.get('name'): + chat.set_name(meta['name']) + + if meta.get('date'): # TODO check if already exists + chat.set_created_at(int(meta['date']['timestamp'])) + + if meta.get('username'): + username = Username(meta['username'], self.get_chat_protocol()) + chat.update_username_timeline(username.get_global_id(), timestamp) + + if meta.get('subchannel'): + subchannel = self.process_subchannel(message, date, timestamp, reply_id=reply_id) + chat.add_children(obj_global_id=subchannel.get_global_id()) + else: + chat.add_message(message.get_global_id(), message.id, timestamp, reply_id=reply_id) + + # if meta.get('subchannels'): # TODO Update icon + names + + return chat + + # def process_subchannels(self): + # pass + + def process_subchannel(self, message, date, timestamp, reply_id=None): # TODO CREATE DATE + meta = self.json_data['meta']['chat']['subchannel'] + subchannel = ChatSubChannels.ChatSubChannel(f'{self.get_chat_id()}/{meta["id"]}', self.get_chat_instance_uuid()) + subchannel.add(date) + + if meta.get('date'): # TODO check if already exists + subchannel.set_created_at(int(meta['date']['timestamp'])) + + if meta.get('name'): + subchannel.set_name(meta['name']) + # subchannel.update_name(meta['name'], timestamp) # TODO ################# + + subchannel.add_message(message.get_global_id(), message.id, timestamp, reply_id=reply_id) + return subchannel + + def process_sender(self, date, timestamp): + meta = self.json_data['meta']['sender'] + user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid()) + + if meta.get('username'): + username = Username(meta['username'], self.get_chat_protocol()) + # TODO timeline or/and correlation ???? + user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id) + user_account.update_username_timeline(username.get_global_id(), timestamp) + + # Username---Message + username.add(date) # TODO # correlation message ??? + + # ADDITIONAL METAS + if meta.get('firstname'): + user_account.set_first_name(meta['firstname']) + if meta.get('lastname'): + user_account.set_last_name(meta['lastname']) + if meta.get('phone'): + user_account.set_phone(meta['phone']) + + return user_account + + # Create abstract class: -> new API endpoint ??? => force field, check if already imported ? + # 1) Create/Get MessageInstance - # TODO uuidv5 + text like discord and telegram for default + # 2) Create/Get CHAT ID - Done + # 3) Create/Get Channel IF is in channel + # 4) Create/Get Thread IF is in thread + # 5) Create/Update Username and User-account - Done + def process_meta(self): # TODO CHECK MANDATORY FIELDS + """ + Process JSON meta filed. + """ + # meta = self.get_json_meta() + + date, timestamp = self.get_message_date_timestamp() + + # REPLY + reply_id = self.get_message_reply_id() + + # TODO Translation + + # Content + content = self.get_message_content() + + message = Messages.create(self.obj.id, content) # TODO translation + + # CHAT + chat = self.process_chat(message, date, timestamp, reply_id=reply_id) + + # SENDER # TODO HANDLE NULL SENDER + user_account = self.process_sender(date, timestamp) + + # UserAccount---Message + user_account.add(date, obj=message) + # UserAccount---Chat + user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id) + + # if chat: # TODO Chat---Username correlation ??? + # # Chat---Username + # chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id) + + + + + + + + + + + + + + + + + + + + + + + diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 5e15eabf..4dce4e63 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -13,6 +13,7 @@ from lib.ConfigLoader import ConfigLoader config_loader = ConfigLoader() r_serv_db = config_loader.get_db_conn("Kvrocks_DB") +r_object = config_loader.get_db_conn("Kvrocks_Objects") config_loader = None AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash', 'item', @@ -40,9 +41,11 @@ def get_all_objects(): def get_objects_with_subtypes(): return ['chat', 'cryptocurrency', 'pgp', 'username'] -def get_object_all_subtypes(obj_type): +def get_object_all_subtypes(obj_type): # TODO Dynamic subtype if obj_type == 'chat': - return ['discord', 'jabber', 'telegram'] + return r_object.smembers(f'all_chat:subtypes') + if obj_type == 'chat-subchannel': + return r_object.smembers(f'all_chat-subchannel:subtypes') if obj_type == 'cryptocurrency': return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash'] if obj_type == 'pgp': diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py new file mode 100755 index 00000000..8c777707 --- /dev/null +++ b/bin/lib/chats_viewer.py @@ -0,0 +1,305 @@ +#!/usr/bin/python3 + +""" +Chats Viewer +=================== + + +""" +import os +import sys +import time +import uuid + + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader +from lib.objects import Chats +from lib.objects import ChatSubChannels + +config_loader = ConfigLoader() +r_db = config_loader.get_db_conn("Kvrocks_DB") +r_crawler = config_loader.get_db_conn("Kvrocks_Crawler") +r_cache = config_loader.get_redis_conn("Redis_Cache") + +r_obj = config_loader.get_db_conn("Kvrocks_DB") # TEMP new DB ???? + +# # # # # # # # +# # +# COMMON # +# # +# # # # # # # # + +# TODO ChatDefaultPlatform + +# CHAT(type=chat, subtype=platform, id= chat_id) + +# Channel(type=channel, subtype=platform, id=channel_id) + +# Thread(type=thread, subtype=platform, id=thread_id) + +# Message(type=message, subtype=platform, id=message_id) + + +# Protocol/Platform + + +# class ChatProtocols: # TODO Remove Me +# +# def __init__(self): # name ???? subtype, id ???? +# # discord, mattermost, ... +# pass +# +# def get_chat_protocols(self): +# pass +# +# def get_chat_protocol(self, protocol): +# pass +# +# ################################################################ +# +# def get_instances(self): +# pass +# +# def get_chats(self): +# pass +# +# def get_chats_by_instance(self, instance): +# pass +# +# +# class ChatNetwork: # uuid or protocol +# def __init__(self, network='default'): +# self.id = network +# +# def get_addresses(self): +# pass +# +# +# class ChatServerAddress: # uuid or protocol + network +# def __init__(self, address='default'): +# self.id = address + +# map uuid -> type + field + +# TODO option last protocol/ imported messages/chat -> unread mode ???? + +# # # # # # # # # +# # +# PROTOCOLS # IRC, discord, mattermost, ... +# # +# # # # # # # # # TODO icon => UI explorer by protocol + network + instance + +def get_chat_protocols(): + return r_obj.smembers(f'chat:protocols') + +def get_chat_protocols_meta(): + metas = [] + for protocol_id in get_chat_protocols(): + protocol = ChatProtocol(protocol_id) + metas.append(protocol.get_meta(options={'icon'})) + return metas + +class ChatProtocol: # TODO first seen last seen ???? + nb by day ???? + def __init__(self, protocol): + self.id = protocol + + def exists(self): + return r_db.exists(f'chat:protocol:{self.id}') + + def get_networks(self): + return r_db.smembers(f'chat:protocol:{self.id}') + + def get_nb_networks(self): + return r_db.scard(f'chat:protocol:{self.id}') + + def get_icon(self): + if self.id == 'discord': + icon = {'style': 'fab', 'icon': 'fa-discord'} + elif self.id == 'telegram': + icon = {'style': 'fab', 'icon': 'fa-telegram'} + else: + icon = {} + return icon + + def get_meta(self, options=set()): + meta = {'id': self.id} + if 'icon' in options: + meta['icon'] = self.get_icon() + return meta + + # def get_addresses(self): + # pass + # + # def get_instances_uuids(self): + # pass + + +# # # # # # # # # # # # # # +# # +# ChatServiceInstance # +# # +# # # # # # # # # # # # # # + +# uuid -> protocol + network + server +class ChatServiceInstance: + def __init__(self, instance_uuid): + self.uuid = instance_uuid + + def exists(self): + return r_obj.exists(f'chatSerIns:{self.uuid}') + + def get_protocol(self): # return objects ???? + return r_obj.hget(f'chatSerIns:{self.uuid}', 'protocol') + + def get_network(self): # return objects ???? + network = r_obj.hget(f'chatSerIns:{self.uuid}', 'network') + if network: + return network + + def get_address(self): # return objects ???? + address = r_obj.hget(f'chatSerIns:{self.uuid}', 'address') + if address: + return address + + def get_meta(self, options=set()): + meta = {'uuid': self.uuid, + 'protocol': self.get_protocol(), + 'network': self.get_network(), + 'address': self.get_address()} + if 'chats' in options: + meta['chats'] = [] + for chat_id in self.get_chats(): + meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'nb_subchannels'})) + return meta + + def get_nb_chats(self): + return Chats.Chats().get_nb_ids_by_subtype(self.uuid) + + def get_chats(self): + return Chats.Chats().get_ids_by_subtype(self.uuid) + +def get_chat_service_instances(): + return r_obj.smembers(f'chatSerIns:all') + +def get_chat_service_instance_uuid(protocol, network, address): + if not network: + network = '' + if not address: + address = '' + return r_obj.hget(f'map:chatSerIns:{protocol}:{network}', address) + +def get_chat_service_instance(protocol, network, address): + instance_uuid = get_chat_service_instance_uuid(protocol, network, address) + if instance_uuid: + return ChatServiceInstance(instance_uuid) + +def create_chat_service_instance(protocol, network=None, address=None): + instance_uuid = get_chat_service_instance_uuid(protocol, network, address) + if instance_uuid: + return instance_uuid + else: + if not network: + network = '' + if not address: + address = '' + instance_uuid = str(uuid.uuid5(uuid.NAMESPACE_URL, f'{protocol}|{network}|{address}')) + r_obj.sadd(f'chatSerIns:all', instance_uuid) + + # map instance - uuid + r_obj.hset(f'map:chatSerIns:{protocol}:{network}', address, instance_uuid) + + r_obj.hset(f'chatSerIns:{instance_uuid}', 'protocol', protocol) + if network: + r_obj.hset(f'chatSerIns:{instance_uuid}', 'network', network) + if address: + r_obj.hset(f'chatSerIns:{instance_uuid}', 'address', address) + + # protocols + r_obj.sadd(f'chat:protocols', protocol) # TODO first seen / last seen + + # protocol -> network + r_obj.sadd(f'chat:protocol:networks:{protocol}', network) + + return instance_uuid + + + + + # INSTANCE ===> CHAT IDS + + + + + + # protocol -> instance_uuids => for protocol->networks -> protocol+network => HGETALL + # protocol+network -> instance_uuids => HGETALL + + # protocol -> networks ???default??? or '' + + # -------------------------------------------------------- + # protocol+network -> addresses => HKEYS + # protocol+network+addresse => HGET + + +# Chat -> subtype=uuid, id = chat id + + +# instance_uuid -> chat id + + +# protocol - uniq ID +# protocol + network -> uuid ???? +# protocol + network + address -> uuid + +####################################################################################### + +def get_subchannels_meta_from_global_id(subchannels): + meta = [] + for sub in subchannels: + _, instance_uuid, sub_id = sub.split(':', 2) + subchannel = ChatSubChannels.ChatSubChannel(sub_id, instance_uuid) + meta.append(subchannel.get_meta({'nb_messages'})) + return meta + +def api_get_chat_service_instance(chat_instance_uuid): + chat_instance = ChatServiceInstance(chat_instance_uuid) + if not chat_instance.exists(): + return {"status": "error", "reason": "Unknown uuid"}, 404 + return chat_instance.get_meta({'chats'}), 200 + +def api_get_chat(chat_id, chat_instance_uuid): + chat = Chats.Chat(chat_id, chat_instance_uuid) + if not chat.exists(): + return {"status": "error", "reason": "Unknown chat"}, 404 + meta = chat.get_meta({'img', 'subchannels', 'username'}) + if meta['subchannels']: + print(meta['subchannels']) + meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels']) + return meta, 200 + +def api_get_subchannel(chat_id, chat_instance_uuid): + subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid) + if not subchannel.exists(): + return {"status": "error", "reason": "Unknown chat"}, 404 + meta = subchannel.get_meta({'img', 'nb_messages'}) + meta['messages'], meta['tags_messages'] = subchannel.get_messages() + return meta, 200 + +# # # # # # # # # # LATER +# # +# ChatCategory # +# # +# # # # # # # # # # + + +if __name__ == '__main__': + r = get_chat_service_instances() + print(r) + r = ChatServiceInstance(r.pop()) + print(r.get_meta({'chats'})) + # r = get_chat_protocols() + # print(r) \ No newline at end of file diff --git a/bin/lib/objects/ChatSubChannels.py b/bin/lib/objects/ChatSubChannels.py new file mode 100755 index 00000000..15521196 --- /dev/null +++ b/bin/lib/objects/ChatSubChannels.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys + +from datetime import datetime + +from flask import url_for +# from pymisp import MISPObject + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import ail_core +from lib.ConfigLoader import ConfigLoader +from lib.objects.abstract_chat_object import AbstractChatObject, AbstractChatObjects + +from lib.data_retention_engine import update_obj_date +from lib.objects import ail_objects +from lib.timeline_engine import Timeline + +from lib.correlations_engine import get_correlation_by_correl_type + +config_loader = ConfigLoader() +baseurl = config_loader.get_config_str("Notifications", "ail_domain") +r_object = config_loader.get_db_conn("Kvrocks_Objects") +r_cache = config_loader.get_redis_conn("Redis_Cache") +config_loader = None + + +################################################################################ +################################################################################ +################################################################################ + +class ChatSubChannel(AbstractChatObject): + """ + AIL Chat Object. (strings) + """ + + # ID -> / subtype = chat_instance_uuid + def __init__(self, id, subtype): + super(ChatSubChannel, self).__init__('chat-subchannel', id, subtype) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True), + # 'compress': 'gzip'} + # return payload + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + # # TODO: + pass + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id) + else: + url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}' + return url + + def get_svg_icon(self): # TODO + # if self.subtype == 'telegram': + # style = 'fab' + # icon = '\uf2c6' + # elif self.subtype == 'discord': + # style = 'fab' + # icon = '\uf099' + # else: + # style = 'fas' + # icon = '\uf007' + style = 'fas' + icon = '\uf086' + return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5} + + # TODO TIME LAST MESSAGES + + def get_meta(self, options=set()): + meta = self._get_meta(options=options) + meta['tags'] = self.get_tags(r_list=True) + meta['name'] = self.get_name() + if 'img' in options: + meta['sub'] = self.get_img() + if 'nb_messages': + meta['nb_messages'] = self.get_nb_messages() + return meta + + def get_misp_object(self): + # obj_attrs = [] + # if self.subtype == 'telegram': + # obj = MISPObject('telegram-account', standalone=True) + # obj_attrs.append(obj.add_attribute('username', value=self.id)) + # + # elif self.subtype == 'twitter': + # obj = MISPObject('twitter-account', standalone=True) + # obj_attrs.append(obj.add_attribute('name', value=self.id)) + # + # else: + # obj = MISPObject('user-account', standalone=True) + # obj_attrs.append(obj.add_attribute('username', value=self.id)) + # + # first_seen = self.get_first_seen() + # last_seen = self.get_last_seen() + # if first_seen: + # obj.first_seen = first_seen + # if last_seen: + # obj.last_seen = last_seen + # if not first_seen or not last_seen: + # self.logger.warning( + # f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}') + # + # for obj_attr in obj_attrs: + # for tag in self.get_tags(): + # obj_attr.add_tag(tag) + # return obj + return + + ############################################################################ + ############################################################################ + + # others optional metas, ... -> # TODO ALL meta in hset + + def _get_timeline_name(self): + return Timeline(self.get_global_id(), 'username') + + def update_name(self, name, timestamp): + self._get_timeline_name().add_timestamp(timestamp, name) + + + # TODO # # # # # # # # # # # + def get_users(self): + pass + + #### Categories #### + + #### Threads #### + + #### Messages #### TODO set parents + + # def get_last_message_id(self): + # + # return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id') + + +class ChatSubChannels(AbstractChatObjects): + def __init__(self): + super().__init__('chat-subchannels') + +# if __name__ == '__main__': +# chat = Chat('test', 'telegram') +# r = chat.get_messages() +# print(r) diff --git a/bin/lib/objects/ChatThreads.py b/bin/lib/objects/ChatThreads.py new file mode 100755 index 00000000..ac78be2a --- /dev/null +++ b/bin/lib/objects/ChatThreads.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys + +from datetime import datetime + +from flask import url_for +# from pymisp import MISPObject + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import ail_core +from lib.ConfigLoader import ConfigLoader +from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id +from lib.data_retention_engine import update_obj_date +from lib.objects import ail_objects +from lib.timeline_engine import Timeline + +from lib.correlations_engine import get_correlation_by_correl_type + +config_loader = ConfigLoader() +baseurl = config_loader.get_config_str("Notifications", "ail_domain") +r_object = config_loader.get_db_conn("Kvrocks_Objects") +r_cache = config_loader.get_redis_conn("Redis_Cache") +config_loader = None + + +################################################################################ +################################################################################ +################################################################################ + +class Chat(AbstractSubtypeObject): # TODO # ID == username ????? + """ + AIL Chat Object. (strings) + """ + + def __init__(self, id, subtype): + super(Chat, self).__init__('chat-thread', id, subtype) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True), + # 'compress': 'gzip'} + # return payload + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + # # TODO: + pass + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id) + else: + url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}' + return url + + def get_svg_icon(self): # TODO + # if self.subtype == 'telegram': + # style = 'fab' + # icon = '\uf2c6' + # elif self.subtype == 'discord': + # style = 'fab' + # icon = '\uf099' + # else: + # style = 'fas' + # icon = '\uf007' + style = 'fas' + icon = '\uf086' + return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5} + + def get_meta(self, options=set()): + meta = self._get_meta(options=options) + meta['id'] = self.id + meta['subtype'] = self.subtype + meta['tags'] = self.get_tags(r_list=True) + if 'username': + meta['username'] = self.get_username() + return meta + + def get_misp_object(self): + return + + ############################################################################ + ############################################################################ + + # others optional metas, ... -> # TODO ALL meta in hset + + #### Messages #### TODO set parents + + # def get_last_message_id(self): + # + # return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id') + + + +if __name__ == '__main__': + chat = Chat('test', 'telegram') + r = chat.get_messages() + print(r) diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index bb27413d..bf73e95b 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -15,6 +15,9 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from lib import ail_core from lib.ConfigLoader import ConfigLoader +from lib.objects.abstract_chat_object import AbstractChatObject, AbstractChatObjects + + from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id from lib.data_retention_engine import update_obj_date from lib.objects import ail_objects @@ -33,19 +36,14 @@ config_loader = None ################################################################################ ################################################################################ -class Chat(AbstractSubtypeObject): # TODO # ID == username ????? +class Chat(AbstractChatObject): """ - AIL Chat Object. (strings) + AIL Chat Object. """ def __init__(self, id, subtype): super(Chat, self).__init__('chat', id, subtype) - # def get_ail_2_ail_payload(self): - # payload = {'raw': self.get_gzip_content(b64=True), - # 'compress': 'gzip'} - # return payload - # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ def delete(self): # # TODO: @@ -74,9 +72,16 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? def get_meta(self, options=set()): meta = self._get_meta(options=options) - meta['id'] = self.id - meta['subtype'] = self.subtype + meta['name'] = self.get_name() meta['tags'] = self.get_tags(r_list=True) + if 'img': + meta['icon'] = self.get_img() + if 'username' in options: + meta['username'] = self.get_username() + if 'subchannels' in options: + meta['subchannels'] = self.get_subchannels() + if 'nb_subchannels': + meta['nb_subchannels'] = self.get_nb_subchannels() return meta def get_misp_object(self): @@ -112,11 +117,6 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? ############################################################################ ############################################################################ - # others optional metas, ... -> # TODO ALL meta in hset - - def get_name(self): # get username ???? - pass - # users that send at least a message else participants/spectator # correlation created by messages def get_users(self): @@ -138,22 +138,22 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? def update_username_timeline(self, username_global_id, timestamp): self._get_timeline_username().add_timestamp(timestamp, username_global_id) + #### ChatSubChannels #### + + + #### Categories #### + + #### Threads #### + + #### Messages #### TODO set parents # def get_last_message_id(self): # # return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id') - def get_obj_message_id(self, obj_id): - if obj_id.endswith('.gz'): - obj_id = obj_id[:-3] - return int(obj_id.split('_')[-1]) - def _get_message_timestamp(self, obj_global_id): return r_object.zscore(f'messages:{self.type}:{self.subtype}:{self.id}', obj_global_id) - def _get_messages(self): - return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True) - def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None): obj = ail_objects.get_obj_from_global_id(obj_global_id) mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'}) @@ -179,26 +179,6 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S') return mess_dict - - def get_messages(self, start=0, page=1, nb=500): # TODO limit nb returned, # TODO add replies - start = 0 - stop = -1 - # r_object.delete(f'messages:{self.type}:{self.subtype}:{self.id}') - - # TODO chat without username ???? -> chat ID ???? - - messages = {} - curr_date = None - for message in self._get_messages(): - date = datetime.fromtimestamp(message[1]) - date_day = date.strftime('%Y/%m/%d') - if date_day != curr_date: - messages[date_day] = [] - curr_date = date_day - mess_dict = self.get_message_meta(message[0], parent=True, mess_datetime=date) - messages[date_day].append(mess_dict) - return messages - # Zset with ID ??? id -> item id ??? multiple id == media + text # id -> media id # How do we handle reply/thread ??? -> separate with new chats name/id ZSET ??? @@ -229,43 +209,12 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? # domain = get_item_domain(item_id) # self.add_correlation('domain', '', domain) - # TODO kvrocks exception if key don't exists - def get_obj_by_message_id(self, mess_id): - return r_object.hget(f'messages:ids:{self.type}:{self.subtype}:{self.id}', mess_id) - # importer -> use cache for previous reply SET to_add_id: previously_imported : expire SET key -> 30 mn - def add_message(self, obj_global_id, timestamp, mess_id, reply_id=None): - r_object.hset(f'messages:ids:{self.type}:{self.subtype}:{self.id}', mess_id, obj_global_id) - r_object.zadd(f'messages:{self.type}:{self.subtype}:{self.id}', {obj_global_id: timestamp}) - if reply_id: - reply_obj = self.get_obj_by_message_id(reply_id) - if reply_obj: - self.add_obj_children(reply_obj, obj_global_id) - else: - self.add_message_cached_reply(reply_id, mess_id) - - # ADD cached replies - for reply_obj in self.get_cached_message_reply(mess_id): - self.add_obj_children(obj_global_id, reply_obj) - - def _get_message_cached_reply(self, message_id): - return r_cache.smembers(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{message_id}') - - def get_cached_message_reply(self, message_id): - objs_global_id = [] - for mess_id in self._get_message_cached_reply(message_id): - obj_global_id = self.get_obj_by_message_id(mess_id) - if obj_global_id: - objs_global_id.append(obj_global_id) - return objs_global_id - - def add_message_cached_reply(self, reply_to_id, message_id): - r_cache.sadd(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_to_id}', message_id) - r_cache.expire(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_to_id}', 600) - - # TODO nb replies = nb son ???? what if it create a onion item ??? -> need source filtering +class Chats(AbstractChatObjects): + def __init__(self): + super().__init__('chat') # TODO factorize def get_all_subtypes(): @@ -280,28 +229,6 @@ def get_all(): def get_all_by_subtype(subtype): return get_all_id('chat', subtype) -# # TODO FILTER NAME + Key + mail -# def sanitize_username_name_to_search(name_to_search, subtype): # TODO FILTER NAME -# -# return name_to_search -# -# def search_usernames_by_name(name_to_search, subtype, r_pos=False): -# usernames = {} -# # for subtype in subtypes: -# r_name = sanitize_username_name_to_search(name_to_search, subtype) -# if not name_to_search or isinstance(r_name, dict): -# # break -# return usernames -# r_name = re.compile(r_name) -# for user_name in get_all_usernames_by_subtype(subtype): -# res = re.search(r_name, user_name) -# if res: -# usernames[user_name] = {} -# if r_pos: -# usernames[user_name]['hl-start'] = res.start() -# usernames[user_name]['hl-end'] = res.end() -# return usernames - if __name__ == '__main__': chat = Chat('test', 'telegram') diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 2f1ef5de..a1e42c10 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -43,6 +43,10 @@ config_loader = None # /!\ handle null chat and message id -> chat = uuid and message = timestamp ??? +# ID = /// => telegram without channels +# ID = //// +# ID = //// +# ID = ///// class Message(AbstractObject): """ AIL Message Object. (strings) @@ -86,7 +90,7 @@ class Message(AbstractObject): return dirs[-2] def get_message_id(self): # TODO optimize - message_id = self.get_basename().rsplit('_', 1)[1] + message_id = self.get_basename().rsplit('/', 1)[1] # if message_id.endswith('.gz'): # message_id = message_id[:-3] return message_id @@ -97,6 +101,10 @@ class Message(AbstractObject): # chat_id = chat_id[:-3] return chat_id + # TODO get Instance ID + # TODO get channel ID + # TODO get thread ID + def get_user_account(self): user_account = self.get_correlation('user-account') if user_account.get('user-account'): @@ -255,8 +263,16 @@ class Message(AbstractObject): def delete(self): pass -def create_obj_id(source, chat_id, message_id, timestamp): - return f'{source}/{timestamp}/{chat_id}_{message_id}' +def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None, thread_id=None): + timestamp = int(timestamp) + if channel_id and thread_id: + return f'{chat_instance}/{timestamp}/{chat_id}/{chat_id}/{message_id}' # TODO add thread ID ????? + elif channel_id: + return f'{chat_instance}/{timestamp}/{channel_id}/{chat_id}/{message_id}' + elif thread_id: + return f'{chat_instance}/{timestamp}/{chat_id}/{thread_id}/{message_id}' + else: + return f'{chat_instance}/{timestamp}/{chat_id}/{message_id}' # TODO Check if already exists # def create(source, chat_id, message_id, timestamp, content, tags=[]): diff --git a/bin/lib/objects/abstract_chat_object.py b/bin/lib/objects/abstract_chat_object.py new file mode 100755 index 00000000..be8c1397 --- /dev/null +++ b/bin/lib/objects/abstract_chat_object.py @@ -0,0 +1,241 @@ +# -*-coding:UTF-8 -* +""" +Base Class for AIL Objects +""" + +################################## +# Import External packages +################################## +import os +import sys +from abc import ABC + +from datetime import datetime +# from flask import url_for + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.objects.abstract_subtype_object import AbstractSubtypeObject +from lib.ail_core import get_object_all_subtypes, zscan_iter ################ +from lib.ConfigLoader import ConfigLoader +from lib.objects.Messages import Message +from lib.objects.UsersAccount import UserAccount +from lib.objects.Usernames import Username +from lib.data_retention_engine import update_obj_date + +from packages import Date + +# LOAD CONFIG +config_loader = ConfigLoader() +r_cache = config_loader.get_redis_conn("Redis_Cache") +r_object = config_loader.get_db_conn("Kvrocks_Objects") +config_loader = None + +# # FIXME: SAVE SUBTYPE NAMES ????? + +class AbstractChatObject(AbstractSubtypeObject, ABC): + """ + Abstract Subtype Object + """ + + def __init__(self, obj_type, id, subtype): + """ Abstract for all the AIL object + + :param obj_type: object type (item, ...) + :param id: Object ID + """ + super().__init__(obj_type, id, subtype) + + # get useraccount / username + # get users ? + # timeline name ???? + # info + # created + # last imported/updated + + # TODO get instance + # TODO get protocol + # TODO get network + # TODO get address + + def get_chat(self): # require ail object TODO ## + if self.type != 'chat': + parent = self.get_parent() + obj_type, _ = parent.split(':', 1) + if obj_type == 'chat': + return parent + + def get_subchannels(self): + subchannels = [] + if self.type == 'chat': # category ??? + print(self.get_childrens()) + for obj_global_id in self.get_childrens(): + print(obj_global_id) + obj_type, _ = obj_global_id.split(':', 1) + if obj_type == 'chat-subchannel': + subchannels.append(obj_global_id) + return subchannels + + def get_nb_subchannels(self): + nb = 0 + if self.type == 'chat': + for obj_global_id in self.get_childrens(): + obj_type, _ = obj_global_id.split(':', 1) + if obj_type == 'chat-subchannel': + nb += 1 + return nb + + def get_threads(self): + threads = [] + for obj_global_id in self.get_childrens(): + obj_type, _ = obj_global_id.split(':', 1) + if obj_type == 'chat-thread': + threads.append(obj_global_id) + return threads + + def get_created_at(self): + return self._get_field('created_at') + + def set_created_at(self, timestamp): + self._set_field('created_at', timestamp) + + def get_name(self): + name = self._get_field('name') + if not name: + name = '' + return name + + def set_name(self, name): + self._set_field('name', name) + + def get_img(self): + return self._get_field('img') + + def set_img(self, icon): + self._set_field('img', icon) + + def get_nb_messages(self): + return r_object.zcard(f'messages:{self.type}:{self.subtype}:{self.id}') + + def _get_messages(self): # TODO paginate + return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True) + + def get_message_meta(self, message, parent=True, mess_datetime=None): # TODO handle file message + obj = Message(message[9:]) + mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'}) + print(mess_dict) + if mess_dict.get('parent') and parent: + mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False) + if mess_dict.get('user-account'): + _, user_account_subtype, user_account_id = mess_dict['user-account'].split(':', 3) + print(mess_dict['user-account']) + user_account = UserAccount(user_account_id, user_account_subtype) + mess_dict['user-account'] = {} + mess_dict['user-account']['type'] = user_account.get_type() + mess_dict['user-account']['subtype'] = user_account.get_subtype(r_str=True) + mess_dict['user-account']['id'] = user_account.get_id() + username = user_account.get_username() + if username: + _, username_account_subtype, username_account_id = username.split(':', 3) + username = Username(username_account_id, username_account_subtype).get_default_meta(link=False) + mess_dict['user-account']['username'] = username # TODO get username at the given timestamp ??? + else: + mess_dict['user-account'] = {'id': 'UNKNOWN'} + + if not mess_datetime: + obj_mess_id = message.get_timestamp() + mess_datetime = datetime.fromtimestamp(obj_mess_id) + mess_dict['date'] = mess_datetime.isoformat(' ') + mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S') + return mess_dict + + def get_messages(self, start=0, page=1, nb=500, unread=False): # threads ???? + # TODO return message meta + tags = {} + messages = {} + curr_date = None + for message in self._get_messages(): + date = datetime.fromtimestamp(message[1]) + date_day = date.strftime('%Y/%m/%d') + if date_day != curr_date: + messages[date_day] = [] + curr_date = date_day + mess_dict = self.get_message_meta(message[0], parent=True, mess_datetime=date) # TODO use object + messages[date_day].append(mess_dict) + + if mess_dict.get('tags'): + for tag in mess_dict['tags']: + if tag not in tags: + tags[tag] = 0 + tags[tag] += 1 + return messages, tags + + # TODO REWRITE ADD OR ADD MESSAGE ???? + # add + # add message + + def get_obj_by_message_id(self, message_id): + return r_object.hget(f'messages:ids:{self.type}:{self.subtype}:{self.id}', message_id) + + def add_message_cached_reply(self, reply_id, message_id): + r_cache.sadd(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_id}', message_id) + r_cache.expire(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_id}', 600) + + def _get_message_cached_reply(self, message_id): + return r_cache.smembers(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{message_id}') + + def get_cached_message_reply(self, message_id): + objs_global_id = [] + for mess_id in self._get_message_cached_reply(message_id): + obj_global_id = self.get_obj_by_message_id(mess_id) # TODO CATCH EXCEPTION + if obj_global_id: + objs_global_id.append(obj_global_id) + return objs_global_id + + def add_message(self, obj_global_id, message_id, timestamp, reply_id=None): + r_object.hset(f'messages:ids:{self.type}:{self.subtype}:{self.id}', message_id, obj_global_id) + r_object.zadd(f'messages:{self.type}:{self.subtype}:{self.id}', {obj_global_id: float(timestamp)}) + + # MESSAGE REPLY + if reply_id: + reply_obj = self.get_obj_by_message_id(reply_id) # TODO CATCH EXCEPTION + if reply_obj: + self.add_obj_children(reply_obj, obj_global_id) + else: + self.add_message_cached_reply(reply_id, message_id) + + + # get_messages_meta ???? + +# TODO move me to abstract subtype +class AbstractChatObjects(ABC): + def __init__(self, type): + self.type = type + + def add_subtype(self, subtype): + r_object.sadd(f'all_{self.type}:subtypes', subtype) + + def get_subtypes(self): + return r_object.smembers(f'all_{self.type}:subtypes') + + def get_nb_ids_by_subtype(self, subtype): + return r_object.zcard(f'{self.type}_all:{subtype}') + + def get_ids_by_subtype(self, subtype): + print(subtype) + print(f'{self.type}_all:{subtype}') + return r_object.zrange(f'{self.type}_all:{subtype}', 0, -1) + + def get_all_id_iterator_iter(self, subtype): + return zscan_iter(r_object, f'{self.type}_all:{subtype}') + + def get_ids(self): + pass + + def search(self): + pass + + + diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index a3f25216..808e7547 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -289,23 +289,24 @@ class AbstractObject(ABC): def get_parent(self): return r_object.hget(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent') - def get_children(self): + def get_childrens(self): return r_object.smembers(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}') - def set_parent(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO ###################### + def set_parent(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO # REMOVE ITEM DUP if not obj_global_id: if obj_subtype is None: obj_subtype = '' obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}' r_object.hset(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent', obj_global_id) - def add_children(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO ###################### + def add_children(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO # REMOVE ITEM DUP if not obj_global_id: if obj_subtype is None: obj_subtype = '' obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}' r_object.sadd(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', obj_global_id) + ## others objects ## def add_obj_children(self, parent_global_id, son_global_id): r_object.sadd(f'child:{parent_global_id}', son_global_id) r_object.hset(f'meta:{son_global_id}', 'parent', parent_global_id) diff --git a/bin/lib/objects/abstract_subtype_object.py b/bin/lib/objects/abstract_subtype_object.py index 007f716b..fa0030da 100755 --- a/bin/lib/objects/abstract_subtype_object.py +++ b/bin/lib/objects/abstract_subtype_object.py @@ -88,7 +88,9 @@ class AbstractSubtypeObject(AbstractObject, ABC): def _get_meta(self, options=None): if options is None: options = set() - meta = {'first_seen': self.get_first_seen(), + meta = {'id': self.id, + 'subtype': self.subtype, + 'first_seen': self.get_first_seen(), 'last_seen': self.get_last_seen(), 'nb_seen': self.get_nb_seen()} if 'icon' in options: @@ -150,8 +152,11 @@ class AbstractSubtypeObject(AbstractObject, ABC): # => data Retention + efficient search # # + def _add_subtype(self): + r_object.sadd(f'all_{self.type}:subtypes', self.subtype) def add(self, date, obj=None): + self._add_subtype() self.update_daterange(date) update_obj_date(date, self.type, self.subtype) # daily diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index c330443b..97f6dde3 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -50,7 +50,7 @@ from blueprints.objects_title import objects_title from blueprints.objects_cookie_name import objects_cookie_name from blueprints.objects_etag import objects_etag from blueprints.objects_hhhash import objects_hhhash -from blueprints.objects_chat import objects_chat +from blueprints.chats_explorer import chats_explorer Flask_dir = os.environ['AIL_FLASK'] @@ -108,7 +108,7 @@ app.register_blueprint(objects_title, url_prefix=baseUrl) app.register_blueprint(objects_cookie_name, url_prefix=baseUrl) app.register_blueprint(objects_etag, url_prefix=baseUrl) app.register_blueprint(objects_hhhash, url_prefix=baseUrl) -app.register_blueprint(objects_chat, url_prefix=baseUrl) +app.register_blueprint(chats_explorer, url_prefix=baseUrl) # ========= =========# diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py new file mode 100644 index 00000000..ff180a32 --- /dev/null +++ b/var/www/blueprints/chats_explorer.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... +''' + +import os +import sys +import json + +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file +from flask_login import login_required, current_user + +# Import Role_Manager +from Role_Manager import login_admin, login_analyst, login_read_only + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import chats_viewer + + + +############################################ + +from lib import ail_core +from lib.objects import ail_objects +from lib import chats_viewer +from lib.objects import Chats +from lib.objects import ChatSubChannels + +# ============ BLUEPRINT ============ +chats_explorer = Blueprint('chats_explorer', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/chats_explorer')) + +# ============ VARIABLES ============ +bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] + +def create_json_response(data, status_code): + return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code + +# ============ FUNCTIONS ============ + +# ============= ROUTES ============== + +@chats_explorer.route("/chats/explorer", methods=['GET']) +@login_required +@login_read_only +def chats_explorer_dashboard(): + return + +@chats_explorer.route("chats/explorer/protocols", methods=['GET']) +@login_required +@login_read_only +def chats_explorer_protocols(): + protocols = chats_viewer.get_chat_protocols_meta() + return render_template('chats_protocols.html', protocols=protocols) + +@chats_explorer.route("chats/explorer/instance", methods=['GET']) +@login_required +@login_read_only +def chats_explorer_instance(): + intance_uuid = request.args.get('uuid') + chat_instance = chats_viewer.api_get_chat_service_instance(intance_uuid) + if chat_instance[1] != 200: + return create_json_response(chat_instance[0], chat_instance[1]) + else: + chat_instance = chat_instance[0] + return render_template('chat_instance.html', chat_instance=chat_instance) + +@chats_explorer.route("chats/explorer/chat", methods=['GET']) +@login_required +@login_read_only +def chats_explorer_chat(): + chat_id = request.args.get('id') + instance_uuid = request.args.get('uuid') + chat = chats_viewer.api_get_chat(chat_id, instance_uuid) + if chat[1] != 200: + return create_json_response(chat[0], chat[1]) + else: + chat = chat[0] + return render_template('chat_viewer.html', chat=chat) + +@chats_explorer.route("/chats/explorer/subchannel", methods=['GET']) +@login_required +@login_read_only +def objects_subchannel_messages(): + subchannel_id = request.args.get('id') + instance_uuid = request.args.get('uuid') + subchannel = chats_viewer.api_get_subchannel(subchannel_id, instance_uuid) + if subchannel[1] != 200: + return create_json_response(subchannel[0], subchannel[1]) + else: + subchannel = subchannel[0] + return render_template('SubChannelMessages.html', subchannel=subchannel) + +@chats_explorer.route("/chats/explorer/subchannel", methods=['GET']) +@login_required +@login_read_only +def objects_message(): + message_id = request.args.get('id') + message = chats_viewer.api_get_message(message_id) + if message[1] != 200: + return create_json_response(message[0], message[1]) + else: + message = message[0] + return render_template('ChatMessage.html', message=message) + +############################################################################################# +############################################################################################# +############################################################################################# + + +@chats_explorer.route("/objects/chat/messages", methods=['GET']) +@login_required +@login_read_only +def objects_dashboard_chat(): + chat = request.args.get('id') + subtype = request.args.get('subtype') + chat = Chats.Chat(chat, subtype) + if chat.exists(): + messages, mess_tags = chat.get_messages() + print(messages) + print(chat.get_subchannels()) + meta = chat.get_meta({'icon', 'username'}) + if meta.get('username'): + meta['username'] = ail_objects.get_obj_from_global_id(meta['username']).get_meta() + print(meta) + return render_template('ChatMessages.html', meta=meta, messages=messages, mess_tags=mess_tags, bootstrap_label=bootstrap_label) + else: + return abort(404) diff --git a/var/www/blueprints/objects_chat.py b/var/www/blueprints/objects_chat.py deleted file mode 100644 index 8a1db11f..00000000 --- a/var/www/blueprints/objects_chat.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -''' - Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... -''' - -import os -import sys -import json - -from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file -from flask_login import login_required, current_user - -# Import Role_Manager -from Role_Manager import login_admin, login_analyst, login_read_only - -sys.path.append(os.environ['AIL_BIN']) -################################## -# Import Project packages -################################## -from lib import ail_core -from lib.objects import abstract_subtype_object -from lib.objects import ail_objects -from lib.objects import Chats -from packages import Date - -# ============ BLUEPRINT ============ -objects_chat = Blueprint('objects_chat', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/chat')) - -# ============ VARIABLES ============ -bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] - -def create_json_response(data, status_code): - return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code - -# ============ FUNCTIONS ============ - -# ============= ROUTES ============== - - -@objects_chat.route("/objects/chat/messages", methods=['GET']) -@login_required -@login_read_only -def objects_dashboard_chat(): - chat = request.args.get('id') - subtype = request.args.get('subtype') - chat = Chats.Chat(chat, subtype) - if chat.exists(): - messages = chat.get_messages() - meta = chat.get_meta({'icon'}) - print(meta) - return render_template('ChatMessages.html', meta=meta, messages=messages, bootstrap_label=bootstrap_label) - else: - return abort(404) - - - diff --git a/var/www/templates/chats_explorer/ChatMessage.html b/var/www/templates/chats_explorer/ChatMessage.html new file mode 100644 index 00000000..32aae48e --- /dev/null +++ b/var/www/templates/chats_explorer/ChatMessage.html @@ -0,0 +1,209 @@ + + + + + Chat Messages - AIL + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + +
    +
    +

    {{ meta["id"] }} :

    +
      +
    • +
      +
      + + + + + + + + + + + + + + + + + + + +
      Object subtypeFirst seenLast seenUsernameNb seen
      + + + + {{ meta["icon"]["icon"] }} + + + {{ meta["subtype"] }} + {{ meta['first_seen'] }}{{ meta['last_seen'] }} + {% if 'username' in meta %} + {{ meta['username']['id'] }} + {% endif %} + {{ meta['nb_seen'] }}
      +
      +
      +
      +
      +
      +
      • +
    • +
      +
      + Tags: + {% for tag in meta['tags'] %} + + {% endfor %} + +
      +
      • +
    + + {% with obj_type='chat', obj_id=meta['id'], obj_subtype=meta['subtype'] %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + +
    +
    + + {% for tag in mess_tags %} + {{ tag }} {{ mess_tags[tag] }} + {% endfor %} + +
    +
    + {% for date in messages %} + {{ date }} + {% endfor %} +
    +
    + +
    +
    + +

    {{ date }}

    + +
    +
    + {{ mess['user-account']['id'] }} +
    {{ mess['hour'] }}
    +
    +
    +
    + {% if mess['user-account']['username'] %} + {{ mess['user-account']['username']['id'] }} + {% else %} + {{ mess['user-account']['id'] }} + {% endif %} +
    + {% if mess['reply_to'] %} +
    +
    + {% if mess['reply_to']['user-account']['username'] %} + {{ mess['reply_to']['user-account']['username']['id'] }} + {% else %} + {{ mess['reply_to']['user-account']['id'] }} + {% endif %} +
    + 
    {{ mess['reply_to']['content'] }}
    + {% for tag in mess['reply_to']['tags'] %} + {{ tag }} + {% endfor %} +
    {{ mess['reply_to']['date'] }}
    +{#
    #} +{# #} +{# #} +{#
    #} +
    + {% endif %} + 
    {{ mess['content'] }}
    + {% for tag in mess['tags'] %} + {{ tag }} + {% endfor %} +
    + + +
    +
    +
    + +
    +
    + +
    + +
    +
    + + + + + + diff --git a/var/www/templates/objects/chat/ChatMessages.html b/var/www/templates/chats_explorer/ChatMessages.html similarity index 90% rename from var/www/templates/objects/chat/ChatMessages.html rename to var/www/templates/chats_explorer/ChatMessages.html index dd99b250..729a132c 100644 --- a/var/www/templates/objects/chat/ChatMessages.html +++ b/var/www/templates/chats_explorer/ChatMessages.html @@ -58,6 +58,7 @@ Object subtype First seen Last seen + Username Nb seen @@ -74,6 +75,11 @@ {{ meta['first_seen'] }} {{ meta['last_seen'] }} + + {% if 'username' in meta %} + {{ meta['username']['id'] }} + {% endif %} + {{ meta['nb_seen'] }} @@ -111,11 +117,23 @@
    + {% for tag in mess_tags %} + {{ tag }} {{ mess_tags[tag] }} + {% endfor %} + +
    +
    + {% for date in messages %} + {{ date }} + {% endfor %} +
    +
    +
    {% for date in messages %} -

    {{ date }}

    +

    {{ date }}

    {% for mess in messages[date] %}
    diff --git a/var/www/templates/chats_explorer/SubChannelMessages.html b/var/www/templates/chats_explorer/SubChannelMessages.html new file mode 100644 index 00000000..846e9cbd --- /dev/null +++ b/var/www/templates/chats_explorer/SubChannelMessages.html @@ -0,0 +1,211 @@ + + + + + Sub-Channel Messages - AIL + + + + + + +{# #} + + + + + + + +{# + #} + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + +
    +
    +

    {{ subchannel["id"] }} :

    +
      +
    • +
      +
      + + + + + + + + + + + + + + + + + + + +
      Chat InstanceFirst seenLast seenUsernameNb Messages
      + {{ subchannel["subtype"] }} + {{ subchannel['first_seen'] }}{{ subchannel['last_seen'] }} + {% if 'username' in subchannel %} + {{ subchannel['username'] }} + {% endif %} + {{ subchannel['nb_messages'] }}
      +
      +
      +
      • +
    • +
      +
      + Tags: + {% for tag in subchannel['tags'] %} + + {% endfor %} + +
      +
      • +
    + + {% with obj_type='chat', obj_id=subchannel['id'], obj_subtype=subchannel['subtype'] %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + +
    +
    + + {% for tag in mess_tags %} + {{ tag }} {{ mess_tags[tag] }} + {% endfor %} + +
    +
    + {% for date in messages %} + {{ date }} + {% endfor %} +
    +
    + +
    +
    + + {% for date in subchannel['messages'] %} +

    {{ date }}

    + {% for mess in subchannel['messages'][date] %} + +
    +
    + {{ mess['user-account']['id'] }} +
    {{ mess['hour'] }}
    +
    +
    +
    + {% if mess['user-account']['username'] %} + {{ mess['user-account']['username']['id'] }} + {% else %} + {{ mess['user-account']['id'] }} + {% endif %} +
    + {% if mess['reply_to'] %} +
    +
    + {% if mess['reply_to']['user-account']['username'] %} + {{ mess['reply_to']['user-account']['username']['id'] }} + {% else %} + {{ mess['reply_to']['user-account']['id'] }} + {% endif %} +
    + 
    {{ mess['reply_to']['content'] }}
    + {% for tag in mess['reply_to']['tags'] %} + {{ tag }} + {% endfor %} +
    {{ mess['reply_to']['date'] }}
    +{#
    #} +{# #} +{# #} +{#
    #} +
    + {% endif %} + 
    {{ mess['content'] }}
    + {% for tag in mess['tags'] %} + {{ tag }} + {% endfor %} +
    + + +
    +
    +
    + + {% endfor %} +
    + {% endfor %} + +
    +
    + +
    + +
    +
    + + + + + + diff --git a/var/www/templates/chats_explorer/chat_instance.html b/var/www/templates/chats_explorer/chat_instance.html new file mode 100644 index 00000000..5ddbc136 --- /dev/null +++ b/var/www/templates/chats_explorer/chat_instance.html @@ -0,0 +1,123 @@ + + + + + Chats Protocols - AIL + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + +
    + +
    +

    {{ chat_instance["protocol"] }} :

    +
      +
    • + + + + + + + + + + + + + + + + + +
      Chat InstanceNetworkAddressNb Chats
      + {{ chat_instance["uuid"] }} + {% if chat_instance["network"] %}{{ chat_instance["network"]}}{% endif %}{% if chat_instance["address"] %}{{ chat_instance["address"] }}{% endif %}{{ chat_instance["chats"] | length }}
      +
      • +
    + +
    +
    + + + + + + + + + + + + + + {% for chat in chat_instance["chats"] %} + + + + + + + + + {% endfor %} + +
    IconNameIDFirst SeenLast SeenNB Chats
    {{ chat['name'] }}{{ chat['id'] }}{{ chat['first_seen'] }}{{ chat['last_seen'] }}{{ chat['nb_subchannels'] }}
    + +
    + +
    +
    + + + + + + diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html new file mode 100644 index 00000000..97059d95 --- /dev/null +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -0,0 +1,133 @@ + + + + + Chats Protocols - AIL + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + +
    + +
    +

    {{ chat["username"] }} {{ chat["id"] }} :

    +
      +
    • + + + + + + + + + + + + + + + + + + + + + +
      IconNameIDFirst SeenLast SeenNB Sub-Channels
      {{ chat['name'] }}{{ chat['id'] }}{{ chat['first_seen'] }}{{ chat['last_seen'] }}{{ chat['nb_subchannels'] }}
      +
      • +
    + +
    +
    + + {% if chat['subchannels'] %} +

    Sub-Channels:

    + + + + + + + + + + + + + {% for meta in chat["subchannels"] %} + + + + + + + + + {% endfor %} + +
    IconNameIDFirst SeenLast SeenNB Messages
    + {{ meta['id'] }} + {{ meta['name'] }}{{ meta['id'] }}{{ meta['first_seen'] }}{{ meta['last_seen'] }}{{ meta['nb_messages'] }}
    + + {% endif %} + +
    + +
    +
    + + + + + + diff --git a/var/www/templates/chats_explorer/chats_instance.html b/var/www/templates/chats_explorer/chats_instance.html new file mode 100644 index 00000000..f7b56e39 --- /dev/null +++ b/var/www/templates/chats_explorer/chats_instance.html @@ -0,0 +1,80 @@ + + + + + Chats Protocols - AIL + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + + {% if protocols %} +
    + {% for meta in protocols %} + +
    +

    + {% if meta['icon'] %} + + {% endif %} + {{ meta['id'] }} +

    +
    +     + {% endfor %} +
    + + {% else %} +

    No Protocol/Chats Imported

    + {% endif %} + +
    + +
    +
    + + + + + + diff --git a/var/www/templates/chats_explorer/chats_protocols.html b/var/www/templates/chats_explorer/chats_protocols.html new file mode 100644 index 00000000..f7b56e39 --- /dev/null +++ b/var/www/templates/chats_explorer/chats_protocols.html @@ -0,0 +1,80 @@ + + + + + Chats Protocols - AIL + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + + {% if protocols %} +
    + {% for meta in protocols %} + +
    +

    + {% if meta['icon'] %} + + {% endif %} + {{ meta['id'] }} +

    +
    +     + {% endfor %} +
    + + {% else %} +

    No Protocol/Chats Imported

    + {% endif %} + +
    + +
    +
    + + + + + + From 789210bcba5cd8ccfac9c3fe60ab41c8304bf88e Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 6 Nov 2023 14:08:23 +0100 Subject: [PATCH 117/238] chg: [chats] improve UI + fix importer --- bin/lib/chats_viewer.py | 12 +- bin/lib/objects/ChatSubChannels.py | 4 +- bin/lib/objects/abstract_chat_object.py | 14 +-- bin/lib/objects/abstract_object.py | 2 + var/www/blueprints/chats_explorer.py | 14 +-- .../chats_explorer/SubChannelMessages.html | 41 ++++-- .../chats_explorer/chat_instance.html | 16 ++- .../templates/chats_explorer/chat_viewer.html | 119 +++++++++++++++++- 8 files changed, 179 insertions(+), 43 deletions(-) diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 8c777707..ef02e0c8 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -265,6 +265,11 @@ def get_subchannels_meta_from_global_id(subchannels): meta.append(subchannel.get_meta({'nb_messages'})) return meta +def get_chat_meta_from_global_id(chat_global_id): + _, instance_uuid, chat_id = chat_global_id.split(':', 2) + chat = Chats.Chat(chat_id, instance_uuid) + return chat.get_meta() + def api_get_chat_service_instance(chat_instance_uuid): chat_instance = ChatServiceInstance(chat_instance_uuid) if not chat_instance.exists(): @@ -277,15 +282,18 @@ def api_get_chat(chat_id, chat_instance_uuid): return {"status": "error", "reason": "Unknown chat"}, 404 meta = chat.get_meta({'img', 'subchannels', 'username'}) if meta['subchannels']: - print(meta['subchannels']) meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels']) + else: + meta['messages'], meta['tags_messages'] = chat.get_messages() return meta, 200 def api_get_subchannel(chat_id, chat_instance_uuid): subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid) if not subchannel.exists(): return {"status": "error", "reason": "Unknown chat"}, 404 - meta = subchannel.get_meta({'img', 'nb_messages'}) + meta = subchannel.get_meta({'chat', 'img', 'nb_messages'}) + if meta['chat']: + meta['chat'] = get_chat_meta_from_global_id(meta['chat']) meta['messages'], meta['tags_messages'] = subchannel.get_messages() return meta, 200 diff --git a/bin/lib/objects/ChatSubChannels.py b/bin/lib/objects/ChatSubChannels.py index 15521196..8d73524a 100755 --- a/bin/lib/objects/ChatSubChannels.py +++ b/bin/lib/objects/ChatSubChannels.py @@ -80,8 +80,10 @@ class ChatSubChannel(AbstractChatObject): meta = self._get_meta(options=options) meta['tags'] = self.get_tags(r_list=True) meta['name'] = self.get_name() + if 'chat' in options: + meta['chat'] = self.get_chat() if 'img' in options: - meta['sub'] = self.get_img() + meta['img'] = self.get_img() if 'nb_messages': meta['nb_messages'] = self.get_nb_messages() return meta diff --git a/bin/lib/objects/abstract_chat_object.py b/bin/lib/objects/abstract_chat_object.py index be8c1397..465bb9e4 100755 --- a/bin/lib/objects/abstract_chat_object.py +++ b/bin/lib/objects/abstract_chat_object.py @@ -63,16 +63,15 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): def get_chat(self): # require ail object TODO ## if self.type != 'chat': parent = self.get_parent() - obj_type, _ = parent.split(':', 1) - if obj_type == 'chat': - return parent + if parent: + obj_type, _ = parent.split(':', 1) + if obj_type == 'chat': + return parent def get_subchannels(self): subchannels = [] if self.type == 'chat': # category ??? - print(self.get_childrens()) for obj_global_id in self.get_childrens(): - print(obj_global_id) obj_type, _ = obj_global_id.split(':', 1) if obj_type == 'chat-subchannel': subchannels.append(obj_global_id) @@ -125,12 +124,11 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): def get_message_meta(self, message, parent=True, mess_datetime=None): # TODO handle file message obj = Message(message[9:]) mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'}) - print(mess_dict) + # print(mess_dict) if mess_dict.get('parent') and parent: mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False) if mess_dict.get('user-account'): _, user_account_subtype, user_account_id = mess_dict['user-account'].split(':', 3) - print(mess_dict['user-account']) user_account = UserAccount(user_account_id, user_account_subtype) mess_dict['user-account'] = {} mess_dict['user-account']['type'] = user_account.get_type() @@ -224,8 +222,6 @@ class AbstractChatObjects(ABC): return r_object.zcard(f'{self.type}_all:{subtype}') def get_ids_by_subtype(self, subtype): - print(subtype) - print(f'{self.type}_all:{subtype}') return r_object.zrange(f'{self.type}_all:{subtype}', 0, -1) def get_all_id_iterator_iter(self, subtype): diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index 808e7547..cac9d58c 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -298,6 +298,7 @@ class AbstractObject(ABC): obj_subtype = '' obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}' r_object.hset(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent', obj_global_id) + r_object.sadd(f'child:{obj_global_id}', self.get_global_id()) def add_children(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO # REMOVE ITEM DUP if not obj_global_id: @@ -305,6 +306,7 @@ class AbstractObject(ABC): obj_subtype = '' obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}' r_object.sadd(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', obj_global_id) + r_object.hset(f'meta:{obj_global_id}', 'parent', self.get_global_id()) ## others objects ## def add_obj_children(self, parent_global_id, son_global_id): diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index ff180a32..2f122a77 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -80,7 +80,7 @@ def chats_explorer_chat(): return create_json_response(chat[0], chat[1]) else: chat = chat[0] - return render_template('chat_viewer.html', chat=chat) + return render_template('chat_viewer.html', chat=chat, bootstrap_label=bootstrap_label) @chats_explorer.route("/chats/explorer/subchannel", methods=['GET']) @login_required @@ -95,18 +95,6 @@ def objects_subchannel_messages(): subchannel = subchannel[0] return render_template('SubChannelMessages.html', subchannel=subchannel) -@chats_explorer.route("/chats/explorer/subchannel", methods=['GET']) -@login_required -@login_read_only -def objects_message(): - message_id = request.args.get('id') - message = chats_viewer.api_get_message(message_id) - if message[1] != 200: - return create_json_response(message[0], message[1]) - else: - message = message[0] - return render_template('ChatMessage.html', message=message) - ############################################################################################# ############################################################################################# ############################################################################################# diff --git a/var/www/templates/chats_explorer/SubChannelMessages.html b/var/www/templates/chats_explorer/SubChannelMessages.html index 846e9cbd..3d7bf5fb 100644 --- a/var/www/templates/chats_explorer/SubChannelMessages.html +++ b/var/www/templates/chats_explorer/SubChannelMessages.html @@ -32,6 +32,13 @@ flex-direction: row-reverse; margin-left: auto } + .divider:after, + .divider:before { + content: ""; + flex: 1; + height: 2px; + background: #eee; + } @@ -47,7 +54,7 @@
    -

    {{ subchannel["id"] }} :

    +

    {% if subchannel['chat']['name'] %}{{ subchannel['chat']['name'] }} {% else %} {{ subchannel['chat']['id'] }}{% endif %} - {% if subchannel['username'] %}{{ subchannel["username"] }} {% else %} {{ subchannel['name'] }}{% endif %} :

    {{ subchannel["id"] }}
    • @@ -55,7 +62,8 @@ - + +{# #} @@ -65,10 +73,19 @@ + + - - {% for chat in chat_instance["chats"] %} - + - - + + {% endfor %} diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html index 97059d95..cde2e9bb 100644 --- a/var/www/templates/chats_explorer/chat_viewer.html +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -17,6 +17,25 @@ + + @@ -32,11 +51,11 @@
      -

      {{ chat["username"] }} {{ chat["id"] }} :

      +

      {% if chat['username'] %}{{ chat["username"] }} {% else %} {{ chat['name'] }}{% endif %} :

      {{ chat["id"] }}
      Chat InstanceNameChat InstanceFirst seen Last seen Username
      - {{ subchannel["subtype"] }} +{# {{ subchannel["subtype"] }}#} + {{ subchannel['name'] }} + + {% if subchannel['first_seen'] %} + {{ subchannel['first_seen'][0:4] }}-{{ subchannel['first_seen'][4:6] }}-{{ subchannel['first_seen'][6:8] }} + {% endif %} + + {% if subchannel['last_seen'] %} + {{ subchannel['last_seen'][0:4] }}-{{ subchannel['last_seen'][4:6] }}-{{ subchannel['last_seen'][6:8] }} + {% endif %} {{ subchannel['first_seen'] }}{{ subchannel['last_seen'] }} {% if 'username' in subchannel %} {{ subchannel['username'] }} @@ -108,8 +125,8 @@ - {% for tag in mess_tags %} - {{ tag }} {{ mess_tags[tag] }} + {% for tag in subchannel['tags_messages'] %} + {{ tag }} {{ subchannel['tags_messages'][tag] }} {% endfor %}
      @@ -121,10 +138,16 @@
      -
      +
      {% for date in subchannel['messages'] %} -

      {{ date }}

      + +
      +

      + {{ date }} +

      +
      + {% for mess in subchannel['messages'][date] %}
      diff --git a/var/www/templates/chats_explorer/chat_instance.html b/var/www/templates/chats_explorer/chat_instance.html index 5ddbc136..9a32d71d 100644 --- a/var/www/templates/chats_explorer/chat_instance.html +++ b/var/www/templates/chats_explorer/chat_instance.html @@ -75,11 +75,21 @@
      + {{ chat['id'] }} + {{ chat['name'] }} {{ chat['id'] }}{{ chat['first_seen'] }}{{ chat['last_seen'] }} + {% if chat['first_seen'] %} + {{ chat['first_seen'][0:4] }}-{{ chat['first_seen'][4:6] }}-{{ chat['first_seen'][6:8] }} + {% endif %} + + {% if chat['last_seen'] %} + {{ chat['last_seen'][0:4] }}-{{ chat['last_seen'][4:6] }}-{{ chat['last_seen'][6:8] }} + {% endif %} + {{ chat['nb_subchannels'] }}
      - + @@ -51,8 +70,16 @@ - - + + @@ -63,6 +90,10 @@ + {% for tag in chat['tags_messages'] %} + {{ tag }} {{ chat['tags_messages'][tag] }} + {% endfor %} + {% if chat['subchannels'] %}

      Sub-Channels:

      Icon Name {{ chat['name'] }} {{ chat['id'] }}{{ chat['first_seen'] }}{{ chat['last_seen'] }} + {% if chat['first_seen'] %} + {{ chat['first_seen'][0:4] }}-{{ chat['first_seen'][4:6] }}-{{ chat['first_seen'][6:8] }} + {% endif %} + + {% if chat['last_seen'] %} + {{ chat['last_seen'][0:4] }}-{{ chat['last_seen'][4:6] }}-{{ chat['last_seen'][6:8] }} + {% endif %} + {{ chat['nb_subchannels'] }}
      @@ -84,8 +115,16 @@ - - + + {% endfor %} @@ -94,6 +133,74 @@ {% endif %} + {% if chat['messages'] %} + +
      +
      + + {% for date in chat['messages'] %} + +
      +

      + {{ date }} +

      +
      + + {% for mess in chat['messages'][date] %} + +
      +
      + {{ mess['user-account']['id'] }} +
      {{ mess['hour'] }}
      +
      +
      +
      + {% if mess['user-account']['username'] %} + {{ mess['user-account']['username']['id'] }} + {% else %} + {{ mess['user-account']['id'] }} + {% endif %} +
      + {% if mess['reply_to'] %} +
      +
      + {% if mess['reply_to']['user-account']['username'] %} + {{ mess['reply_to']['user-account']['username']['id'] }} + {% else %} + {{ mess['reply_to']['user-account']['id'] }} + {% endif %} +
      + 
      {{ mess['reply_to']['content'] }}
      + {% for tag in mess['reply_to']['tags'] %} + {{ tag }} + {% endfor %} +
      {{ mess['reply_to']['date'] }}
      + {#
      #} + {# #} + {# #} + {#
      #} +
      + {% endif %} + 
      {{ mess['content'] }}
      + {% for tag in mess['tags'] %} + {{ tag }} + {% endfor %} +
      + + +
      +
      +
      + + {% endfor %} +
      + {% endfor %} + +
      +
      + + {% endif %} + From b1d5399607057c098b88baa6f7c5760dc243ece8 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 6 Nov 2023 16:38:31 +0100 Subject: [PATCH 118/238] chg: [chats] add UI shortcut + networks list + show chats/subchannels info --- bin/importer/feeders/abstract_chats_feeder.py | 6 ++ bin/lib/chats_viewer.py | 19 ++++- bin/lib/objects/Chats.py | 2 + bin/lib/objects/abstract_chat_object.py | 6 ++ var/www/blueprints/chats_explorer.py | 15 +++- .../chats_explorer/SubChannelMessages.html | 5 ++ .../templates/chats_explorer/chat_viewer.html | 5 ++ .../chats_explorer/chats_networks.html | 78 +++++++++++++++++++ .../chats_explorer/chats_protocols.html | 4 +- .../templates/sidebars/sidebar_objects.html | 18 +++++ 10 files changed, 155 insertions(+), 3 deletions(-) create mode 100644 var/www/templates/chats_explorer/chats_networks.html diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py index dfc559b7..21f12d0c 100755 --- a/bin/importer/feeders/abstract_chats_feeder.py +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -128,6 +128,9 @@ class AbstractChatFeeder(DefaultFeeder, ABC): if meta.get('name'): chat.set_name(meta['name']) + if meta.get('info'): + chat.set_info(meta['info']) + if meta.get('date'): # TODO check if already exists chat.set_created_at(int(meta['date']['timestamp'])) @@ -160,6 +163,9 @@ class AbstractChatFeeder(DefaultFeeder, ABC): subchannel.set_name(meta['name']) # subchannel.update_name(meta['name'], timestamp) # TODO ################# + if meta.get('info'): + subchannel.set_info(meta['info']) + subchannel.add_message(message.get_global_id(), message.id, timestamp, reply_id=reply_id) return subchannel diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index ef02e0c8..69b9c4af 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -185,6 +185,15 @@ class ChatServiceInstance: def get_chat_service_instances(): return r_obj.smembers(f'chatSerIns:all') +def get_chat_service_instances_by_protocol(protocol): + instance_uuids = {} + for network in r_obj.smembers(f'chat:protocol:networks:{protocol}'): + inst_uuids = r_obj.hvals(f'map:chatSerIns:{protocol}:{network}') + if not network: + network = 'default' + instance_uuids[network] = inst_uuids + return instance_uuids + def get_chat_service_instance_uuid(protocol, network, address): if not network: network = '' @@ -192,6 +201,14 @@ def get_chat_service_instance_uuid(protocol, network, address): address = '' return r_obj.hget(f'map:chatSerIns:{protocol}:{network}', address) +def get_chat_service_instance_uuid_meta_from_network_dict(instance_uuids): + for network in instance_uuids: + metas = [] + for instance_uuid in instance_uuids[network]: + metas.append(ChatServiceInstance(instance_uuid).get_meta()) + instance_uuids[network] = metas + return instance_uuids + def get_chat_service_instance(protocol, network, address): instance_uuid = get_chat_service_instance_uuid(protocol, network, address) if instance_uuid: @@ -280,7 +297,7 @@ def api_get_chat(chat_id, chat_instance_uuid): chat = Chats.Chat(chat_id, chat_instance_uuid) if not chat.exists(): return {"status": "error", "reason": "Unknown chat"}, 404 - meta = chat.get_meta({'img', 'subchannels', 'username'}) + meta = chat.get_meta({'img', 'info', 'subchannels', 'username'}) if meta['subchannels']: meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels']) else: diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index bf73e95b..f4e3bd72 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -76,6 +76,8 @@ class Chat(AbstractChatObject): meta['tags'] = self.get_tags(r_list=True) if 'img': meta['icon'] = self.get_img() + if 'info': + meta['info'] = self.get_info() if 'username' in options: meta['username'] = self.get_username() if 'subchannels' in options: diff --git a/bin/lib/objects/abstract_chat_object.py b/bin/lib/objects/abstract_chat_object.py index 465bb9e4..a8f4c960 100755 --- a/bin/lib/objects/abstract_chat_object.py +++ b/bin/lib/objects/abstract_chat_object.py @@ -115,6 +115,12 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): def set_img(self, icon): self._set_field('img', icon) + def get_info(self): + return self._get_field('info') + + def set_info(self, info): + self._set_field('info', info) + def get_nb_messages(self): return r_object.zcard(f'messages:{self.type}:{self.subtype}:{self.id}') diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index 2f122a77..132e5a94 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -57,6 +57,19 @@ def chats_explorer_protocols(): protocols = chats_viewer.get_chat_protocols_meta() return render_template('chats_protocols.html', protocols=protocols) +@chats_explorer.route("chats/explorer/networks", methods=['GET']) +@login_required +@login_read_only +def chats_explorer_networks(): + protocol = request.args.get('protocol') + networks = chats_viewer.get_chat_service_instances_by_protocol(protocol) + if len(networks) == 1: + instance_uuid = list(networks.values())[0] + return redirect(url_for('chats_explorer.chats_explorer_instance', uuid=instance_uuid)) + else: + return render_template('chats_networks.html', protocol=protocol, networks=networks) + + @chats_explorer.route("chats/explorer/instance", methods=['GET']) @login_required @login_read_only @@ -93,7 +106,7 @@ def objects_subchannel_messages(): return create_json_response(subchannel[0], subchannel[1]) else: subchannel = subchannel[0] - return render_template('SubChannelMessages.html', subchannel=subchannel) + return render_template('SubChannelMessages.html', subchannel=subchannel, bootstrap_label=bootstrap_label) ############################################################################################# ############################################################################################# diff --git a/var/www/templates/chats_explorer/SubChannelMessages.html b/var/www/templates/chats_explorer/SubChannelMessages.html index 3d7bf5fb..a21c9065 100644 --- a/var/www/templates/chats_explorer/SubChannelMessages.html +++ b/var/www/templates/chats_explorer/SubChannelMessages.html @@ -98,6 +98,11 @@ + {% if subchannel['info'] %} +
    • + 
      {{ subchannel['info'] }}
      +
      • + {% endif %}

    • diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html index cde2e9bb..e7756137 100644 --- a/var/www/templates/chats_explorer/chat_viewer.html +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -84,6 +84,11 @@
      • {{ meta['name'] }} {{ meta['id'] }}{{ meta['first_seen'] }}{{ meta['last_seen'] }} + {% if meta['first_seen'] %} + {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }} + {% endif %} + + {% if meta['last_seen'] %} + {{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }} + {% endif %} + {{ meta['nb_messages'] }}
      + {% if chat['info'] %} +
    • + 
      {{ chat['info'] }}
      +
      • + {% endif %}
    diff --git a/var/www/templates/chats_explorer/chats_networks.html b/var/www/templates/chats_explorer/chats_networks.html new file mode 100644 index 00000000..9754e2b0 --- /dev/null +++ b/var/www/templates/chats_explorer/chats_networks.html @@ -0,0 +1,78 @@ + + + + + Chats Protocols - AIL + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + +

    {{ protocol }} Networks:

    + + {% for network in networks %} + + {% endfor %} + +
    + +
    +
    + + + + + + diff --git a/var/www/templates/chats_explorer/chats_protocols.html b/var/www/templates/chats_explorer/chats_protocols.html index f7b56e39..401d2624 100644 --- a/var/www/templates/chats_explorer/chats_protocols.html +++ b/var/www/templates/chats_explorer/chats_protocols.html @@ -29,10 +29,12 @@
    +

    Chats Protocols:

    + {% if protocols %}
    {% for meta in protocols %} - +

    {% if meta['icon'] %} diff --git a/var/www/templates/sidebars/sidebar_objects.html b/var/www/templates/sidebars/sidebar_objects.html index 12b5abc0..69687677 100644 --- a/var/www/templates/sidebars/sidebar_objects.html +++ b/var/www/templates/sidebars/sidebar_objects.html @@ -24,10 +24,28 @@ +

    + Explorers +
    +     +
    Objects
    {% if mess['reply_to'] %} -
    +
    {% if mess['reply_to']['user-account']['username'] %} {{ mess['reply_to']['user-account']['username']['id'] }} diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html index e7756137..1d6d961a 100644 --- a/var/www/templates/chats_explorer/chat_viewer.html +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -167,7 +167,7 @@ {% endif %}
    {% if mess['reply_to'] %} -
    +
    {% if mess['reply_to']['user-account']['username'] %} {{ mess['reply_to']['user-account']['username']['id'] }} From 4cc9608a3f3835a7269e11f02f8c5095c353760c Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 7 Nov 2023 15:24:40 +0100 Subject: [PATCH 120/238] chg: [chats explorer] show chats/subchannels creation date --- bin/lib/chats_viewer.py | 6 +++--- bin/lib/objects/ChatSubChannels.py | 2 ++ bin/lib/objects/Chats.py | 2 ++ bin/lib/objects/abstract_chat_object.py | 8 ++++++-- var/www/templates/chats_explorer/SubChannelMessages.html | 4 ++-- var/www/templates/chats_explorer/chat_instance.html | 8 +++++--- var/www/templates/chats_explorer/chat_viewer.html | 6 +++++- 7 files changed, 25 insertions(+), 11 deletions(-) diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 69b9c4af..c2ced4d8 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -173,7 +173,7 @@ class ChatServiceInstance: if 'chats' in options: meta['chats'] = [] for chat_id in self.get_chats(): - meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'nb_subchannels'})) + meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'created_at', 'nb_subchannels'})) return meta def get_nb_chats(self): @@ -297,7 +297,7 @@ def api_get_chat(chat_id, chat_instance_uuid): chat = Chats.Chat(chat_id, chat_instance_uuid) if not chat.exists(): return {"status": "error", "reason": "Unknown chat"}, 404 - meta = chat.get_meta({'img', 'info', 'subchannels', 'username'}) + meta = chat.get_meta({'created_at', 'img', 'info', 'subchannels', 'username'}) if meta['subchannels']: meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels']) else: @@ -308,7 +308,7 @@ def api_get_subchannel(chat_id, chat_instance_uuid): subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid) if not subchannel.exists(): return {"status": "error", "reason": "Unknown chat"}, 404 - meta = subchannel.get_meta({'chat', 'img', 'nb_messages'}) + meta = subchannel.get_meta({'chat', 'created_at', 'img', 'nb_messages'}) if meta['chat']: meta['chat'] = get_chat_meta_from_global_id(meta['chat']) meta['messages'], meta['tags_messages'] = subchannel.get_messages() diff --git a/bin/lib/objects/ChatSubChannels.py b/bin/lib/objects/ChatSubChannels.py index 8d73524a..e3601ce1 100755 --- a/bin/lib/objects/ChatSubChannels.py +++ b/bin/lib/objects/ChatSubChannels.py @@ -86,6 +86,8 @@ class ChatSubChannel(AbstractChatObject): meta['img'] = self.get_img() if 'nb_messages': meta['nb_messages'] = self.get_nb_messages() + if 'created_at': + meta['created_at'] = self.get_created_at(date=True) return meta def get_misp_object(self): diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index f4e3bd72..a0e6dea1 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -84,6 +84,8 @@ class Chat(AbstractChatObject): meta['subchannels'] = self.get_subchannels() if 'nb_subchannels': meta['nb_subchannels'] = self.get_nb_subchannels() + if 'created_at': + meta['created_at'] = self.get_created_at(date=True) return meta def get_misp_object(self): diff --git a/bin/lib/objects/abstract_chat_object.py b/bin/lib/objects/abstract_chat_object.py index 2025f23c..e7e67e92 100755 --- a/bin/lib/objects/abstract_chat_object.py +++ b/bin/lib/objects/abstract_chat_object.py @@ -94,8 +94,12 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): threads.append(obj_global_id) return threads - def get_created_at(self): - return self._get_field('created_at') + def get_created_at(self, date=False): + created_at = self._get_field('created_at') + if date and created_at: + created_at = datetime.fromtimestamp(float(created_at)) + created_at = created_at.isoformat(' ') + return created_at def set_created_at(self, timestamp): self._set_field('created_at', timestamp) diff --git a/var/www/templates/chats_explorer/SubChannelMessages.html b/var/www/templates/chats_explorer/SubChannelMessages.html index 9c173342..d3ccf2d4 100644 --- a/var/www/templates/chats_explorer/SubChannelMessages.html +++ b/var/www/templates/chats_explorer/SubChannelMessages.html @@ -63,7 +63,7 @@ Name -{# Chat Instance#} + Created at First seen Last seen Username @@ -73,9 +73,9 @@ -{# {{ subchannel["subtype"] }}#} {{ subchannel['name'] }} + {{ subchannel["created_at"] }} {% if subchannel['first_seen'] %} {{ subchannel['first_seen'][0:4] }}-{{ subchannel['first_seen'][4:6] }}-{{ subchannel['first_seen'][6:8] }} diff --git a/var/www/templates/chats_explorer/chat_instance.html b/var/www/templates/chats_explorer/chat_instance.html index 9a32d71d..419b8116 100644 --- a/var/www/templates/chats_explorer/chat_instance.html +++ b/var/www/templates/chats_explorer/chat_instance.html @@ -67,9 +67,10 @@ Icon Name ID + Created at First Seen Last Seen - NB Chats + NB SubChannels @@ -78,8 +79,9 @@ {{ chat['id'] }} - {{ chat['name'] }} + {{ chat['name'] }} {{ chat['id'] }} + {{ chat['created_at'] }} {% if chat['first_seen'] %} {{ chat['first_seen'][0:4] }}-{{ chat['first_seen'][4:6] }}-{{ chat['first_seen'][6:8] }} @@ -109,7 +111,7 @@ $('#tablechats').DataTable({ "aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]], "iDisplayLength": 10, - "order": [[ 4, "desc" ]] + "order": [[ 5, "desc" ]] }); }); diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html index 1d6d961a..1b93b322 100644 --- a/var/www/templates/chats_explorer/chat_viewer.html +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -60,6 +60,7 @@ Icon Name ID + Created at First Seen Last Seen NB Sub-Channels @@ -70,6 +71,7 @@ {{ chat['name'] }} {{ chat['id'] }} + {{ chat['created_at'] }} {% if chat['first_seen'] %} {{ chat['first_seen'][0:4] }}-{{ chat['first_seen'][4:6] }}-{{ chat['first_seen'][6:8] }} @@ -107,6 +109,7 @@ Icon Name ID + Created at First Seen Last Seen NB Messages @@ -120,6 +123,7 @@ {{ meta['name'] }} {{ meta['id'] }} + {{ meta['created_at'] }} {% if meta['first_seen'] %} {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }} @@ -220,7 +224,7 @@ $('#tablesubchannels').DataTable({ "aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]], "iDisplayLength": 10, - "order": [[ 4, "desc" ]] + "order": [[ 5, "desc" ]] }); {% endif %} }); From e7f060c23d25a653684a708fb6119f8148737d52 Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 8 Nov 2023 10:31:51 +0100 Subject: [PATCH 121/238] chg: [messages] refactor get_messages_meta + add basic message template --- bin/lib/chats_viewer.py | 7 +++ bin/lib/objects/Chats.py | 37 -------------- bin/lib/objects/Messages.py | 36 +++++++++++-- bin/lib/objects/UsersAccount.py | 7 ++- bin/lib/objects/abstract_chat_object.py | 48 ++++-------------- var/www/blueprints/chats_explorer.py | 25 +++------- .../templates/chats_explorer/ChatMessage.html | 50 +++++++------------ .../chats_explorer/SubChannelMessages.html | 2 +- .../templates/chats_explorer/chat_viewer.html | 2 +- 9 files changed, 82 insertions(+), 132 deletions(-) diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index c2ced4d8..6e33cc3e 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -19,6 +19,7 @@ sys.path.append(os.environ['AIL_BIN']) from lib.ConfigLoader import ConfigLoader from lib.objects import Chats from lib.objects import ChatSubChannels +from lib.objects import Messages config_loader = ConfigLoader() r_db = config_loader.get_db_conn("Kvrocks_DB") @@ -314,6 +315,12 @@ def api_get_subchannel(chat_id, chat_instance_uuid): meta['messages'], meta['tags_messages'] = subchannel.get_messages() return meta, 200 +def api_get_message(message_id): + message = Messages.Message(message_id) + if not message.exists(): + return {"status": "error", "reason": "Unknown uuid"}, 404 + return message.get_meta({'content', 'icon', 'link', 'parent', 'parent_meta', 'user-account'}), 200 + # # # # # # # # # # LATER # # # ChatCategory # diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index a0e6dea1..8c95488d 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -155,43 +155,6 @@ class Chat(AbstractChatObject): # # return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id') - def _get_message_timestamp(self, obj_global_id): - return r_object.zscore(f'messages:{self.type}:{self.subtype}:{self.id}', obj_global_id) - - def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None): - obj = ail_objects.get_obj_from_global_id(obj_global_id) - mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'}) - if mess_dict.get('parent') and parent: - mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False) - if mess_dict.get('user-account'): - user_account = ail_objects.get_obj_from_global_id(mess_dict['user-account']) - mess_dict['user-account'] = {} - mess_dict['user-account']['type'] = user_account.get_type() - mess_dict['user-account']['subtype'] = user_account.get_subtype(r_str=True) - mess_dict['user-account']['id'] = user_account.get_id() - username = user_account.get_username() - if username: - username = ail_objects.get_obj_from_global_id(username).get_default_meta(link=False) - mess_dict['user-account']['username'] = username # TODO get username at the given timestamp ??? - else: - mess_dict['user-account']['id'] = 'UNKNOWN' - - if not mess_datetime: - obj_mess_id = self._get_message_timestamp(obj_global_id) - mess_datetime = datetime.fromtimestamp(obj_mess_id) - mess_dict['date'] = mess_datetime.isoformat(' ') - mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S') - return mess_dict - - # Zset with ID ??? id -> item id ??? multiple id == media + text - # id -> media id - # How do we handle reply/thread ??? -> separate with new chats name/id ZSET ??? - # Handle media ??? - - # list of message id -> obj_id - # list of obj_id -> - # abuse parent children ??? - # def add(self, timestamp, obj_id, mess_id=0, username=None, user_id=None): # date = # TODO get date from object # self.update_daterange(date) diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index a1e42c10..b0ffecc6 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -18,6 +18,7 @@ sys.path.append(os.environ['AIL_BIN']) from lib.ail_core import get_ail_uuid from lib.objects.abstract_object import AbstractObject from lib.ConfigLoader import ConfigLoader +from lib.objects import UsersAccount from lib.data_retention_engine import update_obj_date, get_obj_date_first # TODO Set all messages ??? @@ -105,10 +106,14 @@ class Message(AbstractObject): # TODO get channel ID # TODO get thread ID - def get_user_account(self): + def get_user_account(self, meta=False): user_account = self.get_correlation('user-account') if user_account.get('user-account'): - return f'user-account:{user_account["user-account"].pop()}' + user_account = f'user-account:{user_account["user-account"].pop()}' + if meta: + _, user_account_subtype, user_account_id = user_account.split(':', 3) + user_account = UsersAccount.UserAccount(user_account_id, user_account_subtype).get_meta(options={'username', 'username_meta'}) + return user_account # Update value on import # reply to -> parent ? @@ -176,26 +181,47 @@ class Message(AbstractObject): # return r_object.hget(f'meta:item::{self.id}', 'url') # options: set of optional meta fields - def get_meta(self, options=None): + def get_meta(self, options=None, timestamp=None): """ :type options: set + :type timestamp: float """ if options is None: options = set() meta = self.get_default_meta(tags=True) - meta['date'] = self.get_date() + + # timestamp + if not timestamp: + timestamp = self.get_timestamp() + else: + timestamp = float(timestamp) + timestamp = datetime.fromtimestamp(float(timestamp)) + meta['date'] = timestamp.strftime('%Y%m%d') + meta['hour'] = timestamp.strftime('%H:%M:%S') + meta['full_date'] = timestamp.isoformat(' ') + meta['source'] = self.get_source() # optional meta fields if 'content' in options: meta['content'] = self.get_content() if 'parent' in options: meta['parent'] = self.get_parent() + if meta['parent'] and 'parent_meta' in options: + options.remove('parent') + parent_type, _, parent_id = meta['parent'].split(':', 3) + if parent_type == 'message': + message = Message(parent_id) + meta['reply_to'] = message.get_meta(options=options) if 'investigations' in options: meta['investigations'] = self.get_investigations() if 'link' in options: meta['link'] = self.get_link(flask_context=True) + if 'icon' in options: + meta['icon'] = self.get_svg_icon() if 'user-account' in options: - meta['user-account'] = self.get_user_account() + meta['user-account'] = self.get_user_account(meta=True) + if not meta['user-account']: + meta['user-account'] = {'id': 'UNKNOWN'} # meta['encoding'] = None return meta diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py index 5bc94a9c..36df238b 100755 --- a/bin/lib/objects/UsersAccount.py +++ b/bin/lib/objects/UsersAccount.py @@ -16,6 +16,8 @@ from lib import ail_core from lib.ConfigLoader import ConfigLoader from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id from lib.timeline_engine import Timeline +from lib.objects import Usernames + config_loader = ConfigLoader() baseurl = config_loader.get_config_str("Notifications", "ail_domain") @@ -97,9 +99,12 @@ class UserAccount(AbstractSubtypeObject): meta = self._get_meta(options=options) meta['id'] = self.id meta['subtype'] = self.subtype - meta['tags'] = self.get_tags(r_list=True) + meta['tags'] = self.get_tags(r_list=True) # TODO add in options ???? if 'username' in options: meta['username'] = self.get_username() + if meta['username'] and 'username_meta' in options: + _, username_account_subtype, username_account_id = meta['username'].split(':', 3) + meta['username'] = Usernames.Username(username_account_id, username_account_subtype).get_meta() if 'usernames' in options: meta['usernames'] = self.get_usernames() return meta diff --git a/bin/lib/objects/abstract_chat_object.py b/bin/lib/objects/abstract_chat_object.py index e7e67e92..188f1c5a 100755 --- a/bin/lib/objects/abstract_chat_object.py +++ b/bin/lib/objects/abstract_chat_object.py @@ -21,11 +21,9 @@ from lib.objects.abstract_subtype_object import AbstractSubtypeObject from lib.ail_core import get_object_all_subtypes, zscan_iter ################ from lib.ConfigLoader import ConfigLoader from lib.objects import Messages -from lib.objects.UsersAccount import UserAccount -from lib.objects.Usernames import Username -from lib.data_retention_engine import update_obj_date -from packages import Date +# from lib.data_retention_engine import update_obj_date + # LOAD CONFIG config_loader = ConfigLoader() @@ -143,46 +141,23 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): def get_last_message(self): return r_object.zrevrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, 0) - def get_message_meta(self, message, parent=True, mess_datetime=None): # TODO handle file message - obj = Messages.Message(message[9:]) - mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'}) - # print(mess_dict) - if mess_dict.get('parent') and parent: - mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False) - if mess_dict.get('user-account'): - _, user_account_subtype, user_account_id = mess_dict['user-account'].split(':', 3) - user_account = UserAccount(user_account_id, user_account_subtype) - mess_dict['user-account'] = {} - mess_dict['user-account']['type'] = user_account.get_type() - mess_dict['user-account']['subtype'] = user_account.get_subtype(r_str=True) - mess_dict['user-account']['id'] = user_account.get_id() - username = user_account.get_username() - if username: - _, username_account_subtype, username_account_id = username.split(':', 3) - username = Username(username_account_id, username_account_subtype).get_default_meta(link=False) - mess_dict['user-account']['username'] = username # TODO get username at the given timestamp ??? - else: - mess_dict['user-account'] = {'id': 'UNKNOWN'} + def get_message_meta(self, message, timestamp=None): # TODO handle file message + message = Messages.Message(message[9:]) + meta = message.get_meta(options={'content', 'link', 'parent', 'parent_meta', 'user-account'}, timestamp=timestamp) + return meta - if not mess_datetime: - obj_mess_id = obj.get_timestamp() - mess_datetime = datetime.fromtimestamp(float(obj_mess_id)) - mess_dict['date'] = mess_datetime.isoformat(' ') - mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S') - return mess_dict - - def get_messages(self, start=0, page=1, nb=500, unread=False): # threads ???? + def get_messages(self, start=0, page=1, nb=500, unread=False): # threads ???? # TODO return message meta tags = {} messages = {} curr_date = None for message in self._get_messages(): - date = datetime.fromtimestamp(message[1]) - date_day = date.strftime('%Y/%m/%d') + timestamp = message[1] + date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d') if date_day != curr_date: messages[date_day] = [] curr_date = date_day - mess_dict = self.get_message_meta(message[0], parent=True, mess_datetime=date) # TODO use object + mess_dict = self.get_message_meta(message[0], timestamp=timestamp) messages[date_day].append(mess_dict) if mess_dict.get('tags'): @@ -257,6 +232,3 @@ class AbstractChatObjects(ABC): def search(self): pass - - - diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index 132e5a94..0094629c 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -108,26 +108,15 @@ def objects_subchannel_messages(): subchannel = subchannel[0] return render_template('SubChannelMessages.html', subchannel=subchannel, bootstrap_label=bootstrap_label) -############################################################################################# -############################################################################################# -############################################################################################# - -@chats_explorer.route("/objects/chat/messages", methods=['GET']) +@chats_explorer.route("/objects/message", methods=['GET']) @login_required @login_read_only def objects_dashboard_chat(): - chat = request.args.get('id') - subtype = request.args.get('subtype') - chat = Chats.Chat(chat, subtype) - if chat.exists(): - messages, mess_tags = chat.get_messages() - print(messages) - print(chat.get_subchannels()) - meta = chat.get_meta({'icon', 'username'}) - if meta.get('username'): - meta['username'] = ail_objects.get_obj_from_global_id(meta['username']).get_meta() - print(meta) - return render_template('ChatMessages.html', meta=meta, messages=messages, mess_tags=mess_tags, bootstrap_label=bootstrap_label) + message_id = request.args.get('id') + message = chats_viewer.api_get_message(message_id) + if message[1] != 200: + return create_json_response(message[0], message[1]) else: - return abort(404) + message = message[0] + return render_template('ChatMessage.html', meta=message, bootstrap_label=bootstrap_label) \ No newline at end of file diff --git a/var/www/templates/chats_explorer/ChatMessage.html b/var/www/templates/chats_explorer/ChatMessage.html index 32aae48e..db7a80c3 100644 --- a/var/www/templates/chats_explorer/ChatMessage.html +++ b/var/www/templates/chats_explorer/ChatMessage.html @@ -2,7 +2,7 @@ - Chat Messages - AIL + Chat Message - AIL @@ -112,63 +112,51 @@
    - {% for tag in mess_tags %} - {{ tag }} {{ mess_tags[tag] }} - {% endfor %} - -
    -
    - {% for date in messages %} - {{ date }} - {% endfor %} -
    -
    -
    +
    -

    {{ date }}

    + {{ meta['date'] }}
    - {{ mess['user-account']['id'] }} -
    {{ mess['hour'] }}
    + {{ meta['user-account']['id'] }} +
    {{ meta['hour'] }}
    - {% if mess['user-account']['username'] %} - {{ mess['user-account']['username']['id'] }} + {% if meta['user-account']['username'] %} + {{ meta['user-account']['username']['id'] }} {% else %} - {{ mess['user-account']['id'] }} + {{ meta['user-account']['id'] }} {% endif %}
    - {% if mess['reply_to'] %} -
    + {% if meta['reply_to'] %} +
    - {% if mess['reply_to']['user-account']['username'] %} - {{ mess['reply_to']['user-account']['username']['id'] }} + {% if meta['reply_to']['user-account']['username'] %} + {{ meta['reply_to']['user-account']['username']['id'] }} {% else %} - {{ mess['reply_to']['user-account']['id'] }} + {{ meta['reply_to']['user-account']['id'] }} {% endif %}
    - 
    {{ mess['reply_to']['content'] }}
    - {% for tag in mess['reply_to']['tags'] %} + 
    {{ meta['reply_to']['content'] }}
    + {% for tag in meta['reply_to']['tags'] %} {{ tag }} {% endfor %} -
    {{ mess['reply_to']['date'] }}
    +
    {{ meta['reply_to']['full_date'] }}
    {#
    #} {# #} {# #} {#
    #}
    {% endif %} - 
    {{ mess['content'] }}
    - {% for tag in mess['tags'] %} + 
    {{ meta['content'] }}
    + {% for tag in meta['tags'] %} {{ tag }} {% endfor %}
    - - +
    diff --git a/var/www/templates/chats_explorer/SubChannelMessages.html b/var/www/templates/chats_explorer/SubChannelMessages.html index d3ccf2d4..53a3deef 100644 --- a/var/www/templates/chats_explorer/SubChannelMessages.html +++ b/var/www/templates/chats_explorer/SubChannelMessages.html @@ -181,7 +181,7 @@ {% for tag in mess['reply_to']['tags'] %} {{ tag }} {% endfor %} -
    {{ mess['reply_to']['date'] }}
    +
    {{ mess['reply_to']['full_date'] }}
    {#
    #} {# #} {# #} diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html index 1b93b322..62828b48 100644 --- a/var/www/templates/chats_explorer/chat_viewer.html +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -183,7 +183,7 @@ {% for tag in mess['reply_to']['tags'] %} {{ tag }} {% endfor %} -
    {{ mess['reply_to']['date'] }}
    +
    {{ mess['reply_to']['full_date'] }}
    {#
    #} {# #} {# #} From acef57bb36da69d52be0eb993e6a07d6a7503fdf Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 8 Nov 2023 10:41:32 +0100 Subject: [PATCH 122/238] fix: [tags] fix galaxies synonyms --- bin/lib/Tag.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index 64850b3c..b9c86503 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -387,8 +387,12 @@ def get_cluster_tags(cluster_type, enabled=False): meta_tag = {'tag': tag, 'description': cluster_val.description} if enabled: meta_tag['enabled'] = is_galaxy_tag_enabled(cluster_type, tag) - synonyms = cluster_val.meta.synonyms - if not synonyms: + cluster_val_meta = cluster_val.meta + if cluster_val_meta: + synonyms = cluster_val_meta.synonyms + if not synonyms: + synonyms = [] + else: synonyms = [] meta_tag['synonyms'] = synonyms tags.append(meta_tag) From 6c77ca51365de6a0fc978dc3c644a00aec73225f Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 8 Nov 2023 11:25:30 +0100 Subject: [PATCH 123/238] fix: [chats] fix chat username --- bin/lib/chats_viewer.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 6e33cc3e..3845a493 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -20,6 +20,7 @@ from lib.ConfigLoader import ConfigLoader from lib.objects import Chats from lib.objects import ChatSubChannels from lib.objects import Messages +from lib.objects import Usernames config_loader = ConfigLoader() r_db = config_loader.get_db_conn("Kvrocks_DB") @@ -288,6 +289,11 @@ def get_chat_meta_from_global_id(chat_global_id): chat = Chats.Chat(chat_id, instance_uuid) return chat.get_meta() +def get_username_meta_from_global_id(username_global_id): + _, instance_uuid, username_id = username_global_id.split(':', 2) + username = Usernames.Username(username_id, instance_uuid) + return username.get_meta() + def api_get_chat_service_instance(chat_instance_uuid): chat_instance = ChatServiceInstance(chat_instance_uuid) if not chat_instance.exists(): @@ -299,6 +305,9 @@ def api_get_chat(chat_id, chat_instance_uuid): if not chat.exists(): return {"status": "error", "reason": "Unknown chat"}, 404 meta = chat.get_meta({'created_at', 'img', 'info', 'subchannels', 'username'}) + if meta['username']: + meta['username'] = get_username_meta_from_global_id(meta['username']) + print() if meta['subchannels']: meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels']) else: @@ -312,6 +321,8 @@ def api_get_subchannel(chat_id, chat_instance_uuid): meta = subchannel.get_meta({'chat', 'created_at', 'img', 'nb_messages'}) if meta['chat']: meta['chat'] = get_chat_meta_from_global_id(meta['chat']) + if meta['username']: + meta['username']: meta['messages'], meta['tags_messages'] = subchannel.get_messages() return meta, 200 From bcaf7040b383ada9ffd531e6758ef8bc76817f8f Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 8 Nov 2023 11:37:42 +0100 Subject: [PATCH 124/238] chg: [submodules] bump --- files/misp-galaxy | 2 +- files/misp-taxonomies | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/files/misp-galaxy b/files/misp-galaxy index de12f46b..89e39ddb 160000 --- a/files/misp-galaxy +++ b/files/misp-galaxy @@ -1 +1 @@ -Subproject commit de12f46ba6305d457b1e248cfeeec89827ec93c9 +Subproject commit 89e39ddb3f4414885960aeb67e7023e288a43ee2 diff --git a/files/misp-taxonomies b/files/misp-taxonomies index 7aeaa0b8..e8892b6c 160000 --- a/files/misp-taxonomies +++ b/files/misp-taxonomies @@ -1 +1 @@ -Subproject commit 7aeaa0b890e16f6d9b349f6db3577bf25a9a40ad +Subproject commit e8892b6cf91551d93acf94ce52a36a7112e756cc From 207a6524d7e93d7c4b89426189fa4ac2a5eda65d Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 8 Nov 2023 11:59:06 +0100 Subject: [PATCH 125/238] fix: [languages] fix language module --- bin/modules/Languages.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/modules/Languages.py b/bin/modules/Languages.py index 53c3c999..69e490a2 100755 --- a/bin/modules/Languages.py +++ b/bin/modules/Languages.py @@ -28,9 +28,9 @@ class Languages(AbstractModule): obj = self.get_obj() if obj.type == 'item': - if item.is_crawled(): - domain = Domain(item.get_domain()) - for lang in item.get_languages(min_probability=0.8): + if obj.is_crawled(): + domain = Domain(obj.get_domain()) + for lang in obj.get_languages(min_probability=0.8): domain.add_language(lang.language) From ce989adbd356cb830b06845a03f891d1d5122c3c Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 8 Nov 2023 13:07:00 +0100 Subject: [PATCH 126/238] fix: [chats] fix username meta --- bin/lib/chats_viewer.py | 5 ++--- var/www/templates/chats_explorer/SubChannelMessages.html | 6 ++++-- var/www/templates/chats_explorer/chat_viewer.html | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 3845a493..4e91f47c 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -307,7 +307,6 @@ def api_get_chat(chat_id, chat_instance_uuid): meta = chat.get_meta({'created_at', 'img', 'info', 'subchannels', 'username'}) if meta['username']: meta['username'] = get_username_meta_from_global_id(meta['username']) - print() if meta['subchannels']: meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels']) else: @@ -321,8 +320,8 @@ def api_get_subchannel(chat_id, chat_instance_uuid): meta = subchannel.get_meta({'chat', 'created_at', 'img', 'nb_messages'}) if meta['chat']: meta['chat'] = get_chat_meta_from_global_id(meta['chat']) - if meta['username']: - meta['username']: + if meta.get('username'): + meta['username'] = get_username_meta_from_global_id(meta['username']) meta['messages'], meta['tags_messages'] = subchannel.get_messages() return meta, 200 diff --git a/var/www/templates/chats_explorer/SubChannelMessages.html b/var/www/templates/chats_explorer/SubChannelMessages.html index 53a3deef..198ad504 100644 --- a/var/www/templates/chats_explorer/SubChannelMessages.html +++ b/var/www/templates/chats_explorer/SubChannelMessages.html @@ -54,7 +54,7 @@
    -

    {% if subchannel['chat']['name'] %}{{ subchannel['chat']['name'] }} {% else %} {{ subchannel['chat']['id'] }}{% endif %} - {% if subchannel['username'] %}{{ subchannel["username"] }} {% else %} {{ subchannel['name'] }}{% endif %} :

    {{ subchannel["id"] }} +

    {% if subchannel['chat']['name'] %}{{ subchannel['chat']['name'] }} {% else %} {{ subchannel['chat']['id'] }}{% endif %} - {% if subchannel['username'] %}{{ subchannel["username"]["id"] }} {% else %} {{ subchannel['name'] }}{% endif %} :

    {{ subchannel["id"] }}
    • @@ -88,7 +88,9 @@ {% if 'username' in subchannel %} - {{ subchannel['username'] }} + {% if subchannel['username'] %} + {{ subchannel['username']['id'] }} + {% endif %} {% endif %} {{ subchannel['nb_messages'] }} diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html index 62828b48..8010b508 100644 --- a/var/www/templates/chats_explorer/chat_viewer.html +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -51,7 +51,7 @@
      -

      {% if chat['username'] %}{{ chat["username"] }} {% else %} {{ chat['name'] }}{% endif %} :

      {{ chat["id"] }} +

      {% if chat['username'] %}{{ chat["username"]["id"] }} {% else %} {{ chat['name'] }}{% endif %} :

      {{ chat["id"] }}
      • From e0f70c5072e878424a66a5d67b1a34e70f945f85 Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 8 Nov 2023 13:35:13 +0100 Subject: [PATCH 127/238] fix: [investigations] delete obj --- bin/lib/objects/abstract_subtype_object.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/lib/objects/abstract_subtype_object.py b/bin/lib/objects/abstract_subtype_object.py index fa0030da..9a8aaafc 100755 --- a/bin/lib/objects/abstract_subtype_object.py +++ b/bin/lib/objects/abstract_subtype_object.py @@ -89,6 +89,7 @@ class AbstractSubtypeObject(AbstractObject, ABC): if options is None: options = set() meta = {'id': self.id, + 'type': self.type, 'subtype': self.subtype, 'first_seen': self.get_first_seen(), 'last_seen': self.get_last_seen(), From 54c57ea35b539366f1e91ee14d862982d19733ac Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 8 Nov 2023 15:46:05 +0100 Subject: [PATCH 128/238] chg: [chats] message object template --- bin/lib/ail_core.py | 2 +- bin/lib/chats_viewer.py | 6 +- bin/lib/objects/Messages.py | 10 +-- var/www/blueprints/chats_explorer.py | 21 ++---- .../templates/chats_explorer/ChatMessage.html | 73 +++++++++++++------ 5 files changed, 67 insertions(+), 45 deletions(-) diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 4dce4e63..e963ba73 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -17,7 +17,7 @@ r_object = config_loader.get_db_conn("Kvrocks_Objects") config_loader = None AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash', 'item', - 'pgp', 'screenshot', 'title', 'user-account', 'username'}) + 'message', 'pgp', 'screenshot', 'title', 'user-account', 'username'}) def get_ail_uuid(): ail_uuid = r_serv_db.get('ail:uuid') diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 4e91f47c..a3ad11c6 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -329,7 +329,11 @@ def api_get_message(message_id): message = Messages.Message(message_id) if not message.exists(): return {"status": "error", "reason": "Unknown uuid"}, 404 - return message.get_meta({'content', 'icon', 'link', 'parent', 'parent_meta', 'user-account'}), 200 + meta = message.get_meta({'chat', 'content', 'icon', 'link', 'parent', 'parent_meta', 'user-account'}) + # if meta['chat']: + # print(meta['chat']) + # # meta['chat'] = + return meta, 200 # # # # # # # # # # LATER # # diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index b0ffecc6..8f3eba6b 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -98,8 +98,6 @@ class Message(AbstractObject): def get_chat_id(self): # TODO optimize -> use me to tag Chat chat_id = self.get_basename().rsplit('_', 1)[0] - # if chat_id.endswith('.gz'): - # chat_id = chat_id[:-3] return chat_id # TODO get Instance ID @@ -151,9 +149,9 @@ class Message(AbstractObject): def get_link(self, flask_context=False): if flask_context: - url = url_for('correlation.show_correlation', type=self.type, id=self.id) + url = url_for('chats_explorer.objects_message', type=self.type, id=self.id) else: - url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}' + url = f'{baseurl}/objects/message?id={self.id}' return url def get_svg_icon(self): @@ -196,7 +194,7 @@ class Message(AbstractObject): else: timestamp = float(timestamp) timestamp = datetime.fromtimestamp(float(timestamp)) - meta['date'] = timestamp.strftime('%Y%m%d') + meta['date'] = timestamp.strftime('%Y%/m/%d') meta['hour'] = timestamp.strftime('%H:%M:%S') meta['full_date'] = timestamp.isoformat(' ') @@ -222,6 +220,8 @@ class Message(AbstractObject): meta['user-account'] = self.get_user_account(meta=True) if not meta['user-account']: meta['user-account'] = {'id': 'UNKNOWN'} + if 'chat' in options: + meta['chat'] = self.get_chat_id() # meta['encoding'] = None return meta diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index 0094629c..55cba38c 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -1,15 +1,15 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* -''' +""" Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... -''' +""" import os import sys import json -from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort from flask_login import login_required, current_user # Import Role_Manager @@ -19,17 +19,9 @@ sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## -from lib import chats_viewer - - - -############################################ - from lib import ail_core -from lib.objects import ail_objects from lib import chats_viewer -from lib.objects import Chats -from lib.objects import ChatSubChannels +from lib import Tag # ============ BLUEPRINT ============ chats_explorer = Blueprint('chats_explorer', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/chats_explorer')) @@ -112,11 +104,12 @@ def objects_subchannel_messages(): @chats_explorer.route("/objects/message", methods=['GET']) @login_required @login_read_only -def objects_dashboard_chat(): +def objects_message(): message_id = request.args.get('id') message = chats_viewer.api_get_message(message_id) if message[1] != 200: return create_json_response(message[0], message[1]) else: message = message[0] - return render_template('ChatMessage.html', meta=message, bootstrap_label=bootstrap_label) \ No newline at end of file + return render_template('ChatMessage.html', meta=message, bootstrap_label=bootstrap_label, + modal_add_tags=Tag.get_modal_add_tags(message['id'], object_type='message')) diff --git a/var/www/templates/chats_explorer/ChatMessage.html b/var/www/templates/chats_explorer/ChatMessage.html index db7a80c3..0e96015c 100644 --- a/var/www/templates/chats_explorer/ChatMessage.html +++ b/var/www/templates/chats_explorer/ChatMessage.html @@ -9,6 +9,8 @@ + + @@ -16,6 +18,7 @@ + @@ -50,11 +60,9 @@
        - - - + + - @@ -68,14 +76,12 @@ {{ meta["subtype"] }} - - + -
        Object subtypeFirst seenLast seenDate UsernameNb seen
        {{ meta['first_seen'] }}{{ meta['last_seen'] }}{{ meta['full_date'] }} - {% if 'username' in meta %} - {{ meta['username']['id'] }} + {% if 'username' in meta['user-account'] %} + {{ meta['user-account']['username']['id'] }} {% endif %} {{ meta['nb_seen'] }}
        @@ -85,29 +91,44 @@
      • -
    • -
      -
      - Tags: +
    + +
    +
    + + {% include 'modals/edit_tag.html' %} + {% for tag in meta['tags'] %} {% endfor %} + {% include 'modals/add_tags.html' %} -
    - - + + +
    - {% with obj_type='chat', obj_id=meta['id'], obj_subtype=meta['subtype'] %} - {% include 'modals/investigations_register_obj.html' %} - {% endwith %} - +
    + +
    + {% with obj_type=meta['type'], obj_id=meta['id'], obj_subtype=''%} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} +
    + +
    +
    +
    @@ -116,7 +137,11 @@
    - {{ meta['date'] }} +
    +

    + {{ meta['date'] }} +

    +
    From abc10a1203c5957e6a90bacb7de4286812d3da9e Mon Sep 17 00:00:00 2001 From: terrtia Date: Thu, 9 Nov 2023 14:25:29 +0100 Subject: [PATCH 129/238] fix: [doc] simplify ail feeder --- doc/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/README.md b/doc/README.md index aee47955..bd891289 100644 --- a/doc/README.md +++ b/doc/README.md @@ -201,14 +201,14 @@ pyail = PyAIL(URL, API_KEY, ssl=verifycert) #. . . imports #. . . setup code -for content in sys.stdin: - elm = json.loads(content) - tmp = elm['body'] +for elem in sys.stdin: + elm = json.loads(elem) + content = elm['body'] meta = {} meta['jabber:to'] = elm['to'] meta['jabber:from'] = elm['from'] meta['jabber:ts]' = elm['ts'] - pyail.feed_json_item(tmp , meta, feeder_name, feeder_uuid) + pyail.feed_json_item(content , meta, feeder_name, feeder_uuid) ``` # AIL SYNC From 7bf0fe8992a3da7bcf469c98221da73363a0c2a4 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 13 Nov 2023 14:10:24 +0100 Subject: [PATCH 130/238] chg: [chats] add heatmap nb week messages by hour --- bin/lib/chats_viewer.py | 8 + bin/lib/objects/abstract_chat_object.py | 27 ++ bin/packages/Date.py | 14 + var/www/blueprints/chats_explorer.py | 12 + .../templates/chats_explorer/chat_viewer.html | 333 ++++++++++++++++++ var/www/update_thirdparty.sh | 2 +- 6 files changed, 395 insertions(+), 1 deletion(-) diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index a3ad11c6..bae98988 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -313,6 +313,14 @@ def api_get_chat(chat_id, chat_instance_uuid): meta['messages'], meta['tags_messages'] = chat.get_messages() return meta, 200 +def api_get_nb_message_by_week(chat_id, chat_instance_uuid): + chat = Chats.Chat(chat_id, chat_instance_uuid) + if not chat.exists(): + return {"status": "error", "reason": "Unknown chat"}, 404 + week = chat.get_nb_message_this_week() + # week = chat.get_nb_message_by_week('20231109') + return week, 200 + def api_get_subchannel(chat_id, chat_instance_uuid): subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid) if not subchannel.exists(): diff --git a/bin/lib/objects/abstract_chat_object.py b/bin/lib/objects/abstract_chat_object.py index 188f1c5a..d1645913 100755 --- a/bin/lib/objects/abstract_chat_object.py +++ b/bin/lib/objects/abstract_chat_object.py @@ -8,6 +8,7 @@ Base Class for AIL Objects ################################## import os import sys +import time from abc import ABC from datetime import datetime @@ -21,6 +22,7 @@ from lib.objects.abstract_subtype_object import AbstractSubtypeObject from lib.ail_core import get_object_all_subtypes, zscan_iter ################ from lib.ConfigLoader import ConfigLoader from lib.objects import Messages +from packages import Date # from lib.data_retention_engine import update_obj_date @@ -141,6 +143,30 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): def get_last_message(self): return r_object.zrevrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, 0) + def get_nb_message_by_hours(self, date_day, nb_day): + hours = [] + # start=0, end=23 + timestamp = time.mktime(datetime.strptime(date_day, "%Y%m%d").timetuple()) + for i in range(24): + timestamp_end = timestamp + 3600 + nb_messages = r_object.zcount(f'messages:{self.type}:{self.subtype}:{self.id}', timestamp, timestamp_end) + timestamp = timestamp_end + hours.append({'date': f'{date_day[0:4]}-{date_day[4:6]}-{date_day[6:8]}', 'day': nb_day, 'hour': i, 'count': nb_messages}) + return hours + + def get_nb_message_by_week(self, date_day): + date_day = Date.get_date_week_by_date(date_day) + week_messages = [] + i = 0 + for date in Date.daterange_add_days(date_day, 6): + week_messages = week_messages + self.get_nb_message_by_hours(date, i) + i += 1 + return week_messages + + def get_nb_message_this_week(self): + week_date = Date.get_current_week_day() + return self.get_nb_message_by_week(week_date) + def get_message_meta(self, message, timestamp=None): # TODO handle file message message = Messages.Message(message[9:]) meta = message.get_meta(options={'content', 'link', 'parent', 'parent_meta', 'user-account'}, timestamp=timestamp) @@ -205,6 +231,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): self.add_obj_children(obj_global_id, mess_id) + # get_messages_meta ???? # TODO move me to abstract subtype diff --git a/bin/packages/Date.py b/bin/packages/Date.py index 49bf38eb..6e58fb91 100644 --- a/bin/packages/Date.py +++ b/bin/packages/Date.py @@ -85,11 +85,25 @@ def get_today_date_str(separator=False): else: return datetime.date.today().strftime("%Y%m%d") +def get_current_week_day(): + dt = datetime.date.today() + start = dt - datetime.timedelta(days=dt.weekday()) + return start.strftime("%Y%m%d") + +def get_date_week_by_date(date): + dt = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + start = dt - datetime.timedelta(days=dt.weekday()) + return start.strftime("%Y%m%d") + def date_add_day(date, num_day=1): new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day) new_date = str(new_date).replace('-', '') return new_date +def daterange_add_days(date, nb_days): + end_date = date_add_day(date, num_day=nb_days) + return get_daterange(date, end_date) + def date_substract_day(date, num_day=1): new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) - datetime.timedelta(num_day) new_date = str(new_date).replace('-', '') diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index 55cba38c..1a6dfba7 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -87,6 +87,18 @@ def chats_explorer_chat(): chat = chat[0] return render_template('chat_viewer.html', chat=chat, bootstrap_label=bootstrap_label) +@chats_explorer.route("chats/explorer/messages/stats/week", methods=['GET']) +@login_required +@login_read_only +def chats_explorer_messages_stats_week(): + chat_id = request.args.get('id') + instance_uuid = request.args.get('uuid') + week = chats_viewer.api_get_nb_message_by_week(chat_id, instance_uuid) + if week[1] != 200: + return create_json_response(week[0], week[1]) + else: + return jsonify(week[0]) + @chats_explorer.route("/chats/explorer/subchannel", methods=['GET']) @login_required @login_read_only diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html index 8010b508..fed8676b 100644 --- a/var/www/templates/chats_explorer/chat_viewer.html +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -16,6 +16,7 @@ + @@ -147,6 +151,30 @@ {% if chat['messages'] %} + +
    +
    +
    +
    + +
    +
    + +
    +
    + +
    +
    +
    +
    +     +
    @@ -193,6 +221,9 @@ {#
    #}
    {% endif %} + {% if mess['images'] %} + + {% endif %} 
    {{ mess['content'] }}
    {% for tag in mess['tags'] %} {{ tag }} @@ -245,6 +276,17 @@ function toggle_sidebar(){ $('#core_content').addClass('col-lg-10') } } + + +const blur_slider = $('#blur-slider'); +function blur_images(){ + let blurValue = blur_slider.val(); + blurValue = 15 - blurValue; + let images = document.getElementsByClassName('message_image'); + for(i = 0; i < images.length; i++) { + images[i].style.filter = "blur(" + blurValue + "px)"; + } +} @@ -405,9 +447,6 @@ d3.json("{{ url_for('chats_explorer.chats_explorer_messages_stats_week') }}?uuid tooltip.html(d.date + " " + d.hour + "-" + (d.hour + 1) + "h: " + d.count + " messages") } const mouseleave = function(d) { - console.log(d) - console.log(d.hour) - console.log(d.day) tooltip.style("opacity", 0) d3.select(this) .style("stroke", "none") diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 11a85cd7..ebcd84e3 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -684,7 +684,12 @@ if (d.popover) { if (data["img"]) { if (data["tags_safe"]) { - desc = desc + ""; + if (data["type"] === "screenshot") { + desc = desc + ""; } else { desc = desc + ""; } diff --git a/var/www/templates/objects/image/ImageDaterange.html b/var/www/templates/objects/image/ImageDaterange.html new file mode 100644 index 00000000..e88c9b60 --- /dev/null +++ b/var/www/templates/objects/image/ImageDaterange.html @@ -0,0 +1,602 @@ + + + + + Images - AIL + + + + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + +
    +
    +
    + +{# {% include 'image/block_images_search.html' %}#} + +
    + + +
    + +
    +
    +
    Select a date range :
    +
    +
    +
    + +
    +
    +
    + +
    +
    + + +
    + +
    +
    +
    + +
    +
    +
    +
    +
    +
    + + {% if dict_objects %} + {% if date_from|string == date_to|string %} +

    {{ date_from }} Images Name:

    + {% else %} +

    {{ date_from }} to {{ date_to }} Images Name:

    + {% endif %} + + + + + + + + + + + + {% for obj_id in dict_objects %} + + + + + + + + {% endfor %} + +
    First SeenLast SeenTotalLast days
    {{ dict_objects[obj_id]['id'] }}{{ dict_objects[obj_id]['first_seen'] }}{{ dict_objects[obj_id]['last_seen'] }}{{ dict_objects[obj_id]['nb_seen'] }}
    + + + {% else %} + {% if show_objects %} + {% if date_from|string == date_to|string %} +

    {{ date_from }}, No Image

    + {% else %} +

    {{ date_from }} to {{ date_to }}, No Image

    + {% endif %} + {% endif %} + {% endif %} +
    + +
    +
    + + + + + + + + + + + + + + + + + From 36ff2bb2160e7b334091b74fe376db3239f83851 Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 15 Nov 2023 14:18:24 +0100 Subject: [PATCH 132/238] chg: [images] add sidebar shortcut --- var/www/templates/sidebars/sidebar_objects.html | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/var/www/templates/sidebars/sidebar_objects.html b/var/www/templates/sidebars/sidebar_objects.html index 69687677..dba94772 100644 --- a/var/www/templates/sidebars/sidebar_objects.html +++ b/var/www/templates/sidebars/sidebar_objects.html @@ -70,6 +70,12 @@ HHHash +
  • + + + Image + +
  • From 2ea5f82760d67048ba8ddea53349364c4c1416c0 Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 15 Nov 2023 15:45:25 +0100 Subject: [PATCH 133/238] chg: [queues] add new image queue + add exif module --- bin/LAUNCH.sh | 3 + bin/importer/FeederImporter.py | 3 +- bin/importer/feeders/abstract_chats_feeder.py | 14 ++-- bin/lib/objects/Messages.py | 4 +- bin/modules/Exif.py | 60 +++++++++++++++++ bin/modules/Global.py | 4 +- configs/modules.cfg | 65 ++++++++++++------- 7 files changed, 121 insertions(+), 32 deletions(-) create mode 100755 bin/modules/Exif.py diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index c15311e5..1013e546 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -273,6 +273,9 @@ function launching_scripts { screen -S "Script_AIL" -X screen -t "MISP_Thehive_Auto_Push" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./MISP_Thehive_Auto_Push.py; read x" sleep 0.1 + screen -S "Script_AIL" -X screen -t "Exif" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Exif.py; read x" + sleep 0.1 + ################################## # TRACKERS MODULES # ################################## diff --git a/bin/importer/FeederImporter.py b/bin/importer/FeederImporter.py index cc1e3ea8..4d92cfc6 100755 --- a/bin/importer/FeederImporter.py +++ b/bin/importer/FeederImporter.py @@ -98,7 +98,8 @@ class FeederImporter(AbstractImporter): gzip64_content = feeder.get_gzip64_content() return obj, f'{feeder_name} {gzip64_content}' else: # Messages save on DB - return obj, f'{feeder_name}' + if obj.exists(): + return obj, f'{feeder_name}' class FeederModuleImporter(AbstractModule): diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py index 1a8d097d..f62500c7 100755 --- a/bin/importer/feeders/abstract_chats_feeder.py +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -122,7 +122,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # TODO sanitize obj type obj_type = self.get_obj_type() - print(obj_type) if obj_type == 'image': self.obj = Images.Image(self.json_data['data-sha256']) @@ -226,6 +225,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC): """ # meta = self.get_json_meta() + objs = set() + if self.obj: + objs.add(self.obj) + date, timestamp = self.get_message_date_timestamp() # REPLY @@ -245,14 +248,17 @@ class AbstractChatFeeder(DefaultFeeder, ABC): message_id = self.get_message_id() message_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp) message = Messages.Message(message_id) + # create empty message if message don't exists + if not message.exists(): + message.create('') + objs.add(message) + if message.exists(): obj = Images.create(self.get_message_content()) obj.add(date, message) obj.set_parent(obj_global_id=message.get_global_id()) - else: - obj = None - if obj: + for obj in objs: # TODO PERF avoid parsing metas multpile times # CHAT chat = self.process_chat(obj, date, timestamp, reply_id=reply_id) diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index e6808bf4..59d2e125 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -287,7 +287,7 @@ class Message(AbstractObject): # self._set_translation(translated) # return translated - def create(self, content, translation, tags): + def create(self, content, translation=None, tags=[]): self._set_field('content', content) # r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content) if translation: @@ -315,7 +315,7 @@ def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None def create(obj_id, content, translation=None, tags=[]): message = Message(obj_id) if not message.exists(): - message.create(content, translation, tags) + message.create(content, translation=translation, tags=tags) return message diff --git a/bin/modules/Exif.py b/bin/modules/Exif.py new file mode 100755 index 00000000..865cc243 --- /dev/null +++ b/bin/modules/Exif.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +The Exif Module +====================== + +""" + +################################## +# Import External packages +################################## +import os +import sys + +from PIL import Image, ExifTags + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from modules.abstract_module import AbstractModule + + +class Exif(AbstractModule): + """ + CveModule for AIL framework + """ + + def __init__(self): + super(Exif, self).__init__() + + # Waiting time in seconds between to message processed + self.pending_seconds = 1 + + # Send module state to logs + self.logger.info(f'Module {self.module_name} initialized') + + def compute(self, message): + image = self.get_obj() + print(image) + img = Image.open(image.get_filepath()) + img_exif = img.getexif() + print(img_exif) + if img_exif: + for key, val in img_exif.items(): + if key in ExifTags.TAGS: + print(f'{ExifTags.TAGS[key]}:{val}') + else: + print(f'{key}:{val}') + sys.exit(0) + + # tag = 'infoleak:automatic-detection="cve"' + # Send to Tags Queue + # self.add_message_to_queue(message=tag, queue='Tags') + + +if __name__ == '__main__': + + module = Exif() + module.run() diff --git a/bin/modules/Global.py b/bin/modules/Global.py index dd3cd900..d45f4806 100755 --- a/bin/modules/Global.py +++ b/bin/modules/Global.py @@ -124,9 +124,11 @@ class Global(AbstractModule): else: self.logger.info(f"Empty Item: {message} not processed") - elif self.obj: + elif self.obj.type == 'message': # TODO send to specific object queue => image, ... self.add_message_to_queue(obj=self.obj, queue='Item') + elif self.obj.type == 'image': + self.add_message_to_queue(obj=self.obj, queue='Image') else: self.logger.critical(f"Empty obj: {self.obj} {message} not processed") diff --git a/configs/modules.cfg b/configs/modules.cfg index b14e5c8b..41006974 100644 --- a/configs/modules.cfg +++ b/configs/modules.cfg @@ -1,3 +1,8 @@ +######## IMPORTERS ######## + +[Crawler] +publish = Importers,Tags + [ZMQModuleImporter] publish = Importers @@ -13,8 +18,6 @@ publish = Importers [PystemonModuleImporter] publish = Importers -#################################################### - [Mixer] subscribe = Importers publish = SaveObj @@ -22,9 +25,13 @@ publish = SaveObj [Sync_importer] publish = Importers,Tags +######## OBJ SAVER ######## + [Global] subscribe = SaveObj -publish = Item +publish = Item,Image + +######## ITEM + MESSAGE ######## [Duplicates] subscribe = Duplicate @@ -40,25 +47,19 @@ publish = Host subscribe = Host publish = D4_client -[D4Client] -subscribe = D4_client - -[Retro_Hunt_Module] -publish = Tags - -[Tracker_Typo_Squatting] +[Tracker_Typo_Squatting] # TODO MOVE ME subscribe = Host publish = Tags -[Tracker_Term] +[Tracker_Term] # TODO MOVE ME subscribe = Item publish = Tags -[Tracker_Regex] +[Tracker_Regex] # TODO MOVE ME subscribe = Item publish = Tags -[Tracker_Yara] +[Tracker_Yara] # TODO MOVE ME subscribe = Item publish = Tags @@ -70,7 +71,7 @@ publish = Tags subscribe = Item publish = Tags -[Languages] +[Languages] # TODO MOVE ME subscribe = Item [Categ] @@ -106,13 +107,6 @@ publish = Tags subscribe = Url publish = Tags -[Tags] -subscribe = Tags -publish = Tag_feed - -[MISP_Thehive_Auto_Push] -subscribe = Tag_feed - #[SentimentAnalysis] #subscribe = Item @@ -151,9 +145,6 @@ publish = Tags [SubmitPaste] publish = Importers -[Crawler] -publish = Importers,Tags - [IPAddress] subscribe = Item publish = Tags @@ -164,6 +155,32 @@ publish = Tags #[Sync_module] #publish = Sync +######## IMAGE ######## + +[Exif] +subscribe = Image +publish = Tags + + +######## CORE ######## + +[Tags] +subscribe = Tags +publish = Tag_feed + +[Retro_Hunt_Module] +publish = Tags + +######## OTHER ######## + +[D4Client] +subscribe = D4_client + +[MISP_Thehive_Auto_Push] +subscribe = Tag_feed + +######## DOC ######## + # [My_Module_Name] # subscribe = Global # Queue name # publish = Tags # Queue name From 9fbd3f4bb6c87cafaac26cc6962c7734bc998825 Mon Sep 17 00:00:00 2001 From: terrtia Date: Thu, 16 Nov 2023 14:50:03 +0100 Subject: [PATCH 134/238] chg: [chats] add message and blur blocks --- bin/importer/feeders/abstract_chats_feeder.py | 2 +- bin/lib/chats_viewer.py | 2 +- .../templates/chats_explorer/ChatMessage.html | 48 ++-------- .../chats_explorer/SubChannelMessages.html | 50 ++--------- .../chats_explorer/block_message.html | 74 +++++++++++++++ .../chats_explorer/chat_instance.html | 2 +- .../templates/chats_explorer/chat_viewer.html | 89 ++----------------- .../chats_explorer/chats_instance.html | 2 +- .../chats_explorer/chats_protocols.html | 2 +- .../objects/image/block_blur_img_slider.html | 34 +++++++ 10 files changed, 134 insertions(+), 171 deletions(-) create mode 100644 var/www/templates/chats_explorer/block_message.html create mode 100644 var/www/templates/objects/image/block_blur_img_slider.html diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py index f62500c7..c075a312 100755 --- a/bin/importer/feeders/abstract_chats_feeder.py +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -258,7 +258,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC): obj.add(date, message) obj.set_parent(obj_global_id=message.get_global_id()) - for obj in objs: # TODO PERF avoid parsing metas multpile times + for obj in objs: # TODO PERF avoid parsing metas multiple times # CHAT chat = self.process_chat(obj, date, timestamp, reply_id=reply_id) diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index bae98988..8c201c4b 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -337,7 +337,7 @@ def api_get_message(message_id): message = Messages.Message(message_id) if not message.exists(): return {"status": "error", "reason": "Unknown uuid"}, 404 - meta = message.get_meta({'chat', 'content', 'icon', 'link', 'parent', 'parent_meta', 'user-account'}) + meta = message.get_meta({'chat', 'content', 'icon', 'images', 'link', 'parent', 'parent_meta', 'user-account'}) # if meta['chat']: # print(meta['chat']) # # meta['chat'] = diff --git a/var/www/templates/chats_explorer/ChatMessage.html b/var/www/templates/chats_explorer/ChatMessage.html index 0e96015c..5a129e66 100644 --- a/var/www/templates/chats_explorer/ChatMessage.html +++ b/var/www/templates/chats_explorer/ChatMessage.html @@ -134,6 +134,9 @@
    • + {% include 'objects/image/block_blur_img_slider.html' %} + +
    @@ -143,48 +146,9 @@

    -
    -
    - {{ meta['user-account']['id'] }} -
    {{ meta['hour'] }}
    -
    -
    -
    - {% if meta['user-account']['username'] %} - {{ meta['user-account']['username']['id'] }} - {% else %} - {{ meta['user-account']['id'] }} - {% endif %} -
    - {% if meta['reply_to'] %} -
    -
    - {% if meta['reply_to']['user-account']['username'] %} - {{ meta['reply_to']['user-account']['username']['id'] }} - {% else %} - {{ meta['reply_to']['user-account']['id'] }} - {% endif %} -
    - 
    {{ meta['reply_to']['content'] }}
    - {% for tag in meta['reply_to']['tags'] %} - {{ tag }} - {% endfor %} -
    {{ meta['reply_to']['full_date'] }}
    -{#
    #} -{# #} -{# #} -{#
    #} -
    - {% endif %} - 
    {{ meta['content'] }}
    - {% for tag in meta['tags'] %} - {{ tag }} - {% endfor %} -
    - -
    -
    -
    + {% with message=meta %} + {% include 'chats_explorer/block_message.html' %} + {% endwith %}
    diff --git a/var/www/templates/chats_explorer/SubChannelMessages.html b/var/www/templates/chats_explorer/SubChannelMessages.html index 198ad504..922b52da 100644 --- a/var/www/templates/chats_explorer/SubChannelMessages.html +++ b/var/www/templates/chats_explorer/SubChannelMessages.html @@ -144,6 +144,10 @@
    + + {% include 'objects/image/block_blur_img_slider.html' %} + +
    @@ -157,49 +161,9 @@ {% for mess in subchannel['messages'][date] %} -
    -
    - {{ mess['user-account']['id'] }} -
    {{ mess['hour'] }}
    -
    -
    -
    - {% if mess['user-account']['username'] %} - {{ mess['user-account']['username']['id'] }} - {% else %} - {{ mess['user-account']['id'] }} - {% endif %} -
    - {% if mess['reply_to'] %} -
    -
    - {% if mess['reply_to']['user-account']['username'] %} - {{ mess['reply_to']['user-account']['username']['id'] }} - {% else %} - {{ mess['reply_to']['user-account']['id'] }} - {% endif %} -
    - 
    {{ mess['reply_to']['content'] }}
    - {% for tag in mess['reply_to']['tags'] %} - {{ tag }} - {% endfor %} -
    {{ mess['reply_to']['full_date'] }}
    -{#
    #} -{# #} -{# #} -{#
    #} -
    - {% endif %} - 
    {{ mess['content'] }}
    - {% for tag in mess['tags'] %} - {{ tag }} - {% endfor %} -
    - - -
    -
    -
    + {% with message=mess %} + {% include 'chats_explorer/block_message.html' %} + {% endwith %} {% endfor %}
    diff --git a/var/www/templates/chats_explorer/block_message.html b/var/www/templates/chats_explorer/block_message.html new file mode 100644 index 00000000..df2c4d30 --- /dev/null +++ b/var/www/templates/chats_explorer/block_message.html @@ -0,0 +1,74 @@ + + + + + +
    +
    + {{ message['user-account']['id'] }} +
    {{ message['hour'] }}
    +
    +
    +
    + {% if message['user-account']['username'] %} + {{ message['user-account']['username']['id'] }} + {% else %} + {{ message['user-account']['id'] }} + {% endif %} +
    + {% if message['reply_to'] %} +
    +
    + {% if message['reply_to']['user-account']['username'] %} + {{ message['reply_to']['user-account']['username']['id'] }} + {% else %} + {{ message['reply_to']['user-account']['id'] }} + {% endif %} +
    + 
    {{ message['reply_to']['content'] }}
    + {% for tag in message['reply_to']['tags'] %} + {{ tag }} + {% endfor %} +
    {{ message['reply_to']['full_date'] }}
    + {#
    #} + {# #} + {# #} + {#
    #} +
    + {% endif %} + {% if message['images'] %} + + {% endif %} + 
    {{ message['content'] }}
    + {% for tag in message['tags'] %} + {{ tag }} + {% endfor %} +
    + + +
    +
    +
    + + diff --git a/var/www/templates/chats_explorer/chat_instance.html b/var/www/templates/chats_explorer/chat_instance.html index 419b8116..d70df757 100644 --- a/var/www/templates/chats_explorer/chat_instance.html +++ b/var/www/templates/chats_explorer/chat_instance.html @@ -106,7 +106,7 @@ @@ -442,15 +369,15 @@ d3.json("{{ url_for('chats_explorer.chats_explorer_messages_stats_week') }}?uuid tooltip.style("opacity", 1) d3.select(this) .style("stroke", "black") - .style("opacity", 1) + //.style("stroke-opacity", 1) tooltip.html(d.date + " " + d.hour + "-" + (d.hour + 1) + "h: " + d.count + " messages") } const mouseleave = function(d) { tooltip.style("opacity", 0) d3.select(this) - .style("stroke", "none") - .style("opacity", 0.8) + .style("stroke", "white") + //.style("stroke-opacity", 0.8) } /////////////////////////////////////////////////////////////////////////// diff --git a/var/www/templates/chats_explorer/chats_instance.html b/var/www/templates/chats_explorer/chats_instance.html index f7b56e39..a68fab7b 100644 --- a/var/www/templates/chats_explorer/chats_instance.html +++ b/var/www/templates/chats_explorer/chats_instance.html @@ -57,7 +57,7 @@ \ No newline at end of file From 2b8e9b43f3bbd05874cf89e615605e0ae6199b27 Mon Sep 17 00:00:00 2001 From: terrtia Date: Fri, 24 Nov 2023 15:05:19 +0100 Subject: [PATCH 135/238] chg: [chats] factorise heatmap + chat icon --- bin/importer/FeederImporter.py | 25 +- bin/importer/feeders/BgpMonitor.py | 1 + bin/importer/feeders/Default.py | 2 +- bin/importer/feeders/Discord.py | 38 +++ bin/importer/feeders/Jabber.py | 2 +- bin/importer/feeders/Twitter.py | 2 +- bin/importer/feeders/Urlextract.py | 2 + bin/importer/feeders/abstract_chats_feeder.py | 13 +- bin/lib/chats_viewer.py | 6 +- bin/lib/objects/Chats.py | 4 +- bin/lib/objects/abstract_chat_object.py | 12 +- bin/modules/Exif.py | 2 + var/www/blueprints/objects_image.py | 2 +- .../chats_explorer/block_message.html | 4 +- .../chats_explorer/chat_instance.html | 3 +- .../templates/chats_explorer/chat_viewer.html | 231 +----------------- 16 files changed, 103 insertions(+), 246 deletions(-) create mode 100755 bin/importer/feeders/Discord.py diff --git a/bin/importer/FeederImporter.py b/bin/importer/FeederImporter.py index 4d92cfc6..881cb8a9 100755 --- a/bin/importer/FeederImporter.py +++ b/bin/importer/FeederImporter.py @@ -89,17 +89,26 @@ class FeederImporter(AbstractImporter): feeder_name = feeder.get_name() print(f'importing: {feeder_name} feeder') - obj = feeder.get_obj() # TODO replace by a list of objects to import ???? + # Get Data object: + data_obj = feeder.get_obj() + # process meta if feeder.get_json_meta(): - feeder.process_meta() + objs = feeder.process_meta() + if objs is None: + objs = set() + else: + objs = set() - if obj.type == 'item': # object save on disk as file (Items) - gzip64_content = feeder.get_gzip64_content() - return obj, f'{feeder_name} {gzip64_content}' - else: # Messages save on DB - if obj.exists(): - return obj, f'{feeder_name}' + objs.add(data_obj) + + for obj in objs: + if obj.type == 'item': # object save on disk as file (Items) + gzip64_content = feeder.get_gzip64_content() + return obj, f'{feeder_name} {gzip64_content}' + else: # Messages save on DB + if obj.exists(): + return obj, f'{feeder_name}' class FeederModuleImporter(AbstractModule): diff --git a/bin/importer/feeders/BgpMonitor.py b/bin/importer/feeders/BgpMonitor.py index dc926bfd..90c6d7cf 100755 --- a/bin/importer/feeders/BgpMonitor.py +++ b/bin/importer/feeders/BgpMonitor.py @@ -33,3 +33,4 @@ class BgpMonitorFeeder(DefaultFeeder): tag = 'infoleak:automatic-detection=bgp_monitor' item = Item(self.get_item_id()) item.add_tag(tag) + return set() diff --git a/bin/importer/feeders/Default.py b/bin/importer/feeders/Default.py index ced7ac43..25b3d13b 100755 --- a/bin/importer/feeders/Default.py +++ b/bin/importer/feeders/Default.py @@ -84,4 +84,4 @@ class DefaultFeeder: Process JSON meta filed. """ # meta = self.get_json_meta() - pass + return set() diff --git a/bin/importer/feeders/Discord.py b/bin/importer/feeders/Discord.py new file mode 100755 index 00000000..1fa02187 --- /dev/null +++ b/bin/importer/feeders/Discord.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +The Telegram Feeder Importer Module +================ + +Process Telegram JSON + +""" +import os +import sys +import datetime + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from importer.feeders.abstract_chats_feeder import AbstractChatFeeder +from lib.ConfigLoader import ConfigLoader +from lib.objects import ail_objects +from lib.objects.Chats import Chat +from lib.objects import Messages +from lib.objects import UsersAccount +from lib.objects.Usernames import Username + +import base64 + +class DiscordFeeder(AbstractChatFeeder): + + def __init__(self, json_data): + super().__init__('discord', json_data) + + # def get_obj(self):. + # obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) + # obj_id = f'message:telegram:{obj_id}' + # self.obj = ail_objects.get_obj_from_global_id(obj_id) + # return self.obj + diff --git a/bin/importer/feeders/Jabber.py b/bin/importer/feeders/Jabber.py index 8c90adfd..1b22bc79 100755 --- a/bin/importer/feeders/Jabber.py +++ b/bin/importer/feeders/Jabber.py @@ -52,4 +52,4 @@ class JabberFeeder(DefaultFeeder): user_fr = Username(fr, 'jabber') user_to.add(date, item) user_fr.add(date, item) - return None + return set() diff --git a/bin/importer/feeders/Twitter.py b/bin/importer/feeders/Twitter.py index 1c719e73..3f544e67 100755 --- a/bin/importer/feeders/Twitter.py +++ b/bin/importer/feeders/Twitter.py @@ -45,4 +45,4 @@ class TwitterFeeder(DefaultFeeder): user = str(self.json_data['meta']['twitter:id']) username = Username(user, 'twitter') username.add(date, item) - return None + return set() diff --git a/bin/importer/feeders/Urlextract.py b/bin/importer/feeders/Urlextract.py index 1ef19920..00080daf 100755 --- a/bin/importer/feeders/Urlextract.py +++ b/bin/importer/feeders/Urlextract.py @@ -56,3 +56,5 @@ class UrlextractFeeder(DefaultFeeder): item = Item(self.item_id) item.set_parent(parent_id) + return set() + diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py index c075a312..04bdd1e4 100755 --- a/bin/importer/feeders/abstract_chats_feeder.py +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -131,7 +131,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC): self.obj = Messages.Message(obj_id) return self.obj - def process_chat(self, obj, date, timestamp, reply_id=None): # TODO threads + def process_chat(self, new_objs, obj, date, timestamp, reply_id=None): # TODO threads meta = self.json_data['meta']['chat'] # todo replace me by function chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid()) @@ -147,6 +147,12 @@ class AbstractChatFeeder(DefaultFeeder, ABC): if meta.get('date'): # TODO check if already exists chat.set_created_at(int(meta['date']['timestamp'])) + if meta.get('icon'): + img = Images.create(meta['icon'], b64=True) + img.add(date, chat) + chat.set_icon(img.get_global_id()) + new_objs.add(img) + if meta.get('username'): username = Username(meta['username'], self.get_chat_protocol()) chat.update_username_timeline(username.get_global_id(), timestamp) @@ -228,6 +234,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC): objs = set() if self.obj: objs.add(self.obj) + new_objs = set() date, timestamp = self.get_message_date_timestamp() @@ -261,7 +268,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC): for obj in objs: # TODO PERF avoid parsing metas multiple times # CHAT - chat = self.process_chat(obj, date, timestamp, reply_id=reply_id) + chat = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id) # SENDER # TODO HANDLE NULL SENDER user_account = self.process_sender(obj, date, timestamp) @@ -279,6 +286,8 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # -> subchannel ? # -> thread id ? + return new_objs | objs + diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 8c201c4b..76af9f6e 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -175,7 +175,7 @@ class ChatServiceInstance: if 'chats' in options: meta['chats'] = [] for chat_id in self.get_chats(): - meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'created_at', 'nb_subchannels'})) + meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'created_at', 'icon', 'nb_subchannels'})) return meta def get_nb_chats(self): @@ -304,7 +304,7 @@ def api_get_chat(chat_id, chat_instance_uuid): chat = Chats.Chat(chat_id, chat_instance_uuid) if not chat.exists(): return {"status": "error", "reason": "Unknown chat"}, 404 - meta = chat.get_meta({'created_at', 'img', 'info', 'subchannels', 'username'}) + meta = chat.get_meta({'created_at', 'icon', 'info', 'subchannels', 'username'}) if meta['username']: meta['username'] = get_username_meta_from_global_id(meta['username']) if meta['subchannels']: @@ -325,7 +325,7 @@ def api_get_subchannel(chat_id, chat_instance_uuid): subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid) if not subchannel.exists(): return {"status": "error", "reason": "Unknown chat"}, 404 - meta = subchannel.get_meta({'chat', 'created_at', 'img', 'nb_messages'}) + meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages'}) if meta['chat']: meta['chat'] = get_chat_meta_from_global_id(meta['chat']) if meta.get('username'): diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index 8c95488d..22a299c5 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -74,8 +74,8 @@ class Chat(AbstractChatObject): meta = self._get_meta(options=options) meta['name'] = self.get_name() meta['tags'] = self.get_tags(r_list=True) - if 'img': - meta['icon'] = self.get_img() + if 'icon': + meta['icon'] = self.get_icon() if 'info': meta['info'] = self.get_info() if 'username' in options: diff --git a/bin/lib/objects/abstract_chat_object.py b/bin/lib/objects/abstract_chat_object.py index d266eb5c..6094fb60 100755 --- a/bin/lib/objects/abstract_chat_object.py +++ b/bin/lib/objects/abstract_chat_object.py @@ -113,11 +113,13 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): def set_name(self, name): self._set_field('name', name) - def get_img(self): - return self._get_field('img') + def get_icon(self): + icon = self._get_field('icon') + if icon: + return icon.rsplit(':', 1)[1] - def set_img(self, icon): - self._set_field('img', icon) + def set_icon(self, icon): + self._set_field('icon', icon) def get_info(self): return self._get_field('info') @@ -187,7 +189,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): tags = {} messages = {} curr_date = None - for message in self._get_messages(nb=10, page=3): + for message in self._get_messages(nb=30, page=1): timestamp = message[1] date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d') if date_day != curr_date: diff --git a/bin/modules/Exif.py b/bin/modules/Exif.py index 865cc243..190874d9 100755 --- a/bin/modules/Exif.py +++ b/bin/modules/Exif.py @@ -42,6 +42,8 @@ class Exif(AbstractModule): img_exif = img.getexif() print(img_exif) if img_exif: + gps = img_exif.get(34853) + print(gps) for key, val in img_exif.items(): if key in ExifTags.TAGS: print(f'{ExifTags.TAGS[key]}:{val}') diff --git a/var/www/blueprints/objects_image.py b/var/www/blueprints/objects_image.py index 8fd320e8..18d2d715 100644 --- a/var/www/blueprints/objects_image.py +++ b/var/www/blueprints/objects_image.py @@ -40,7 +40,7 @@ def image(filename): abort(404) filename = filename.replace('/', '') image = Images.Image(filename) - return send_from_directory(Images.IMAGE_FOLDER, image.get_rel_path(), as_attachment=True) + return send_from_directory(Images.IMAGE_FOLDER, image.get_rel_path(), as_attachment=False, mimetype='image') @objects_image.route("/objects/images", methods=['GET']) diff --git a/var/www/templates/chats_explorer/block_message.html b/var/www/templates/chats_explorer/block_message.html index df2c4d30..3a9db11e 100644 --- a/var/www/templates/chats_explorer/block_message.html +++ b/var/www/templates/chats_explorer/block_message.html @@ -58,7 +58,9 @@
    {% endif %} {% if message['images'] %} - + {% for message_image in message['images'] %} + + {% endfor %} {% endif %} 
    {{ message['content'] }}
    {% for tag in message['tags'] %} diff --git a/var/www/templates/chats_explorer/chat_instance.html b/var/www/templates/chats_explorer/chat_instance.html index d70df757..557ade0a 100644 --- a/var/www/templates/chats_explorer/chat_instance.html +++ b/var/www/templates/chats_explorer/chat_instance.html @@ -77,7 +77,8 @@ {% for chat in chat_instance["chats"] %} - {{ chat['id'] }} + {{ chat['id'] }} {{ chat['name'] }} {{ chat['id'] }} diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html index bf3df43a..9741fdce 100644 --- a/var/www/templates/chats_explorer/chat_viewer.html +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -17,6 +17,7 @@ + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + +
    +
    + {{ meta["id"] }} +
      +
    • +
      +
      + + + + + + + + + + + + + + + + + + +
      IDParentFirst seenLast seenNb Messages
      + {{ meta['id'] }} + + {% if meta['first_seen'] %} + {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }} + {% endif %} + + {% if meta['last_seen'] %} + {{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }} + {% endif %} + {{ meta['nb_messages'] }}
      +
      +
      +
      • +{#
    • #} +{#
      #} +{#
      #} +{# Tags:#} +{# {% for tag in meta['tags'] %}#} +{# #} +{# {% endfor %}#} +{# #} +{#
      #} +{#
      • #} +
    + +{# {% with obj_type='chat', obj_id=meta['id'], obj_subtype=meta['subtype'] %}#} +{# {% include 'modals/investigations_register_obj.html' %}#} +{# {% endwith %}#} +{# #} + +
    +
    + + {% for tag in meta['tags_messages'] %} + {{ tag }} {{ meta['tags_messages'][tag] }} + {% endfor %} + +
    +
    + {% for date in messages %} + {{ date }} + {% endfor %} +
    +
    + + + {% include 'objects/image/block_blur_img_slider.html' %} + + +
    +
    + + {% for date in meta['messages'] %} + +
    +

    + {{ date }} +

    +
    + + {% for mess in meta['messages'][date] %} + + {% with message=mess %} + {% include 'chats_explorer/block_message.html' %} + {% endwith %} + + {% endfor %} +
    + {% endfor %} + +
    +
    + +
    + +
    +
    + + + + + + diff --git a/var/www/templates/chats_explorer/block_message.html b/var/www/templates/chats_explorer/block_message.html index b06689b9..019de791 100644 --- a/var/www/templates/chats_explorer/block_message.html +++ b/var/www/templates/chats_explorer/block_message.html @@ -74,6 +74,12 @@ {% for reaction in message['reactions'] %} {{ reaction }} {{ message['reactions'][reaction] }} {% endfor %} + {% if message['thread'] %} +
    +
    + {{ message['thread']['nb'] }} Messages +
    + {% endif %} {% for tag in message['tags'] %} {{ tag }} {% endfor %} From 941838ab76a57d3df5a0583658f85c5b432b0463 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 4 Dec 2023 10:26:02 +0100 Subject: [PATCH 140/238] chg: [chats] add discord threads, Forum channel --- bin/importer/feeders/abstract_chats_feeder.py | 11 +++-- bin/lib/chats_viewer.py | 12 ++++- bin/lib/objects/ChatSubChannels.py | 7 ++- bin/lib/objects/ChatThreads.py | 2 + bin/lib/objects/Chats.py | 3 ++ bin/lib/objects/abstract_chat_object.py | 7 +-- .../chats_explorer/SubChannelMessages.html | 45 ++++++++++++++++++- .../chats_explorer/ThreadMessages.html | 4 +- 8 files changed, 78 insertions(+), 13 deletions(-) diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py index c21e0c78..aebf76ef 100755 --- a/bin/importer/feeders/abstract_chats_feeder.py +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -182,8 +182,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC): return chat - # def process_subchannels(self): - # pass def process_subchannel(self, obj, date, timestamp, reply_id=None): # TODO CREATE DATE meta = self.json_data['meta']['chat']['subchannel'] @@ -216,12 +214,17 @@ class AbstractChatFeeder(DefaultFeeder, ABC): p_subchannel_id = meta['parent'].get('subchannel') p_message_id = meta['parent'].get('message') + # print(thread_id, p_chat_id, p_subchannel_id, p_message_id) + if p_chat_id == self.get_chat_id() and p_subchannel_id == self.get_subchannel_id(): thread = ChatThreads.create(thread_id, self.get_chat_instance_uuid(), p_chat_id, p_subchannel_id, p_message_id, obj_chat) thread.add(date, obj) thread.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id) # TODO OTHERS CORRELATIONS TO ADD + if meta.get('name'): + thread.set_name(meta['name']) + return thread # TODO @@ -308,8 +311,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC): else: chat_id = self.get_chat_id() + thread_id = self.get_thread_id() + channel_id = self.get_subchannel_id() message_id = self.get_message_id() - message_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp) + message_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp, channel_id=channel_id, thread_id=thread_id) message = Messages.Message(message_id) # create empty message if message don't exists if not message.exists(): diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 4ddd7065..2740e6ca 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -290,6 +290,12 @@ def get_chat_meta_from_global_id(chat_global_id): chat = Chats.Chat(chat_id, instance_uuid) return chat.get_meta() +def get_threads_metas(threads): + metas = [] + for thread in threads: + metas.append(ChatThreads.ChatThread(thread['id'], thread['subtype']).get_meta(options={'name', 'nb_messages'})) + return metas + def get_username_meta_from_global_id(username_global_id): _, instance_uuid, username_id = username_global_id.split(':', 2) username = Usernames.Username(username_id, instance_uuid) @@ -305,7 +311,7 @@ def api_get_chat(chat_id, chat_instance_uuid): chat = Chats.Chat(chat_id, chat_instance_uuid) if not chat.exists(): return {"status": "error", "reason": "Unknown chat"}, 404 - meta = chat.get_meta({'created_at', 'icon', 'info', 'subchannels', 'username'}) + meta = chat.get_meta({'created_at', 'icon', 'info', 'subchannels', 'threads', 'username'}) if meta['username']: meta['username'] = get_username_meta_from_global_id(meta['username']) if meta['subchannels']: @@ -326,9 +332,11 @@ def api_get_subchannel(chat_id, chat_instance_uuid): subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid) if not subchannel.exists(): return {"status": "error", "reason": "Unknown subchannel"}, 404 - meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages'}) + meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages', 'threads'}) if meta['chat']: meta['chat'] = get_chat_meta_from_global_id(meta['chat']) + if meta.get('threads'): + meta['threads'] = get_threads_metas(meta['threads']) if meta.get('username'): meta['username'] = get_username_meta_from_global_id(meta['username']) meta['messages'], meta['tags_messages'] = subchannel.get_messages() diff --git a/bin/lib/objects/ChatSubChannels.py b/bin/lib/objects/ChatSubChannels.py index f73488a4..38a50a20 100755 --- a/bin/lib/objects/ChatSubChannels.py +++ b/bin/lib/objects/ChatSubChannels.py @@ -84,10 +84,13 @@ class ChatSubChannel(AbstractChatObject): meta['chat'] = self.get_chat() if 'img' in options: meta['img'] = self.get_img() - if 'nb_messages': + if 'nb_messages' in options: meta['nb_messages'] = self.get_nb_messages() - if 'created_at': + if 'created_at' in options: meta['created_at'] = self.get_created_at(date=True) + if 'threads' in options: + meta['threads'] = self.get_threads() + print(meta['threads']) return meta def get_misp_object(self): diff --git a/bin/lib/objects/ChatThreads.py b/bin/lib/objects/ChatThreads.py index 0790d4ae..9514a248 100755 --- a/bin/lib/objects/ChatThreads.py +++ b/bin/lib/objects/ChatThreads.py @@ -73,6 +73,8 @@ class ChatThread(AbstractChatObject): meta['id'] = self.id meta['subtype'] = self.subtype meta['tags'] = self.get_tags(r_list=True) + if 'name': + meta['name'] = self.get_name() if 'nb_messages': meta['nb_messages'] = self.get_nb_messages() # created_at ??? diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index 22a299c5..b631b2d2 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -86,6 +86,9 @@ class Chat(AbstractChatObject): meta['nb_subchannels'] = self.get_nb_subchannels() if 'created_at': meta['created_at'] = self.get_created_at(date=True) + if 'threads' in options: + meta['threads'] = self.get_threads() + print(meta['threads']) return meta def get_misp_object(self): diff --git a/bin/lib/objects/abstract_chat_object.py b/bin/lib/objects/abstract_chat_object.py index ee6387d3..be25eecb 100755 --- a/bin/lib/objects/abstract_chat_object.py +++ b/bin/lib/objects/abstract_chat_object.py @@ -88,10 +88,10 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): def get_threads(self): threads = [] - for obj_global_id in self.get_childrens(): - obj_type, _ = obj_global_id.split(':', 1) + for child in self.get_childrens(): + obj_type, obj_subtype, obj_id = child.split(':', 2) if obj_type == 'chat-thread': - threads.append(obj_global_id) + threads.append({'type': obj_type, 'subtype': obj_subtype, 'id': obj_id}) return threads def get_created_at(self, date=False): @@ -242,6 +242,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): for mess_id in self.get_cached_message_reply(message_id): self.add_obj_children(obj_global_id, mess_id) + # def get_deleted_messages(self, message_id): # get_messages_meta ???? diff --git a/var/www/templates/chats_explorer/SubChannelMessages.html b/var/www/templates/chats_explorer/SubChannelMessages.html index 922b52da..3b807b96 100644 --- a/var/www/templates/chats_explorer/SubChannelMessages.html +++ b/var/www/templates/chats_explorer/SubChannelMessages.html @@ -132,6 +132,43 @@
    + + {% if subchannel['threads'] %} + + + + + + + + + + + + + {% for thread in subchannel['threads'] %} + + + + + + + + {% endfor %} + +
    NameCreated atFirst seenLast seenNb Messages
    + {{ thread['name'] }} {{ thread['nb_messages'] }} Messages + {{ thread["created_at"] }} + {% if thread['first_seen'] %} + {{ thread['first_seen'][0:4] }}-{{ thread['first_seen'][4:6] }}-{{ thread['first_seen'][6:8] }} + {% endif %} + + {% if thread['last_seen'] %} + {{ thread['last_seen'][0:4] }}-{{ thread['last_seen'][4:6] }}-{{ thread['last_seen'][6:8] }} + {% endif %} + {{ thread['nb_messages'] }}
    + {% endif %} + {% for tag in subchannel['tags_messages'] %} {{ tag }} {{ subchannel['tags_messages'][tag] }} {% endfor %} @@ -182,7 +219,13 @@ $(document).ready(function(){ $("#page-Decoded").addClass("active"); $("#nav_chat").addClass("active"); - + {% if subchannel['threads'] %} + $('#tablethreads').DataTable({ + "aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]], + "iDisplayLength": 10, + "order": [[ 3, "desc" ]] + }); + {% endif %} }); function toggle_sidebar(){ diff --git a/var/www/templates/chats_explorer/ThreadMessages.html b/var/www/templates/chats_explorer/ThreadMessages.html index 921861c9..1e8e588e 100644 --- a/var/www/templates/chats_explorer/ThreadMessages.html +++ b/var/www/templates/chats_explorer/ThreadMessages.html @@ -62,7 +62,7 @@ - + @@ -72,7 +72,7 @@ + @@ -94,6 +95,9 @@ {% endif %} +
    IDName Parent First seen Last seen
    - {{ meta['id'] }} + {{ meta['name'] }} {% if meta['first_seen'] %} From bef4e69a681aeaddeb9c77d5e180c86b6a770aca Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 4 Dec 2023 15:47:58 +0100 Subject: [PATCH 141/238] chg: [chats] translate messages on demand --- bin/lib/Language.py | 88 +++++++++++++++++++ bin/lib/chats_viewer.py | 21 +++-- bin/lib/objects/Messages.py | 31 +++++-- bin/lib/objects/abstract_chat_object.py | 8 +- configs/core.cfg.sample | 3 + var/www/blueprints/chats_explorer.py | 20 +++-- .../chats_explorer/SubChannelMessages.html | 3 + .../chats_explorer/ThreadMessages.html | 3 + .../chats_explorer/block_message.html | 8 ++ .../chats_explorer/block_translation.html | 37 ++++++++ .../templates/chats_explorer/chat_viewer.html | 4 + 11 files changed, 205 insertions(+), 21 deletions(-) create mode 100644 var/www/templates/chats_explorer/block_translation.html diff --git a/bin/lib/Language.py b/bin/lib/Language.py index e413c434..dd8df1c2 100755 --- a/bin/lib/Language.py +++ b/bin/lib/Language.py @@ -4,6 +4,20 @@ import os import sys +import cld3 +from libretranslatepy import LibreTranslateAPI + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader + +config_loader = ConfigLoader() +TRANSLATOR_URL = config_loader.get_config_str('Translation', 'libretranslate') +config_loader = None + + dict_iso_languages = { 'af': 'Afrikaans', 'am': 'Amharic', @@ -237,3 +251,77 @@ def get_iso_from_languages(l_languages, sort=False): if sort: l_iso = sorted(l_iso) return l_iso + + +class LanguageDetector: + pass + +def get_translator_instance(): + return TRANSLATOR_URL + +class LanguageTranslator: + + def __init__(self): + self.lt = LibreTranslateAPI(get_translator_instance()) + + def languages(self): + languages = [] + try: + for dict_lang in self.lt.languages(): + languages.append({'iso': dict_lang['code'], 'language': dict_lang['name']}) + except: + pass + return languages + + def detect_cld3(self, content): + for lang in cld3.get_frequent_languages(content, num_langs=1): + return lang.language + + def detect_libretranslate(self, content): + try: + language = self.lt.detect(content) + except: # TODO ERROR MESSAGE + language = None + if language: + return language[0].get('language') + + def detect(self, content): # TODO replace by gcld3 + # cld3 + if len(content) >= 200: + language = self.detect_cld3(content) + # libretranslate + else: + language = self.detect_libretranslate(content) + return language + + def translate(self, content, source=None, target="en"): # TODO source target + translation = None + if content: + if not source: + source = self.detect(content) + # print(source, content) + if source: + if source != target: + try: + # print(content, source, target) + translation = self.lt.translate(content, source, target) + except: + translation = None + # TODO LOG and display error + if translation == content: + print('EQUAL') + translation = None + return translation + + +LIST_LANGUAGES = LanguageTranslator().languages() + +def get_translation_languages(): + return LIST_LANGUAGES + + +if __name__ == '__main__': + t_content = '' + langg = LanguageTranslator() + # lang.translate(t_content, source='ru') + langg.languages() diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 2740e6ca..1d983f0a 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -21,6 +21,7 @@ from lib.objects import Chats from lib.objects import ChatSubChannels from lib.objects import ChatThreads from lib.objects import Messages +from lib.objects import UsersAccount from lib.objects import Usernames config_loader = ConfigLoader() @@ -307,7 +308,7 @@ def api_get_chat_service_instance(chat_instance_uuid): return {"status": "error", "reason": "Unknown uuid"}, 404 return chat_instance.get_meta({'chats'}), 200 -def api_get_chat(chat_id, chat_instance_uuid): +def api_get_chat(chat_id, chat_instance_uuid, translation_target=None): chat = Chats.Chat(chat_id, chat_instance_uuid) if not chat.exists(): return {"status": "error", "reason": "Unknown chat"}, 404 @@ -317,7 +318,7 @@ def api_get_chat(chat_id, chat_instance_uuid): if meta['subchannels']: meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels']) else: - meta['messages'], meta['tags_messages'] = chat.get_messages() + meta['messages'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target) return meta, 200 def api_get_nb_message_by_week(chat_id, chat_instance_uuid): @@ -328,7 +329,7 @@ def api_get_nb_message_by_week(chat_id, chat_instance_uuid): # week = chat.get_nb_message_by_week('20231109') return week, 200 -def api_get_subchannel(chat_id, chat_instance_uuid): +def api_get_subchannel(chat_id, chat_instance_uuid, translation_target=None): subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid) if not subchannel.exists(): return {"status": "error", "reason": "Unknown subchannel"}, 404 @@ -339,17 +340,17 @@ def api_get_subchannel(chat_id, chat_instance_uuid): meta['threads'] = get_threads_metas(meta['threads']) if meta.get('username'): meta['username'] = get_username_meta_from_global_id(meta['username']) - meta['messages'], meta['tags_messages'] = subchannel.get_messages() + meta['messages'], meta['tags_messages'] = subchannel.get_messages(translation_target=translation_target) return meta, 200 -def api_get_thread(thread_id, thread_instance_uuid): +def api_get_thread(thread_id, thread_instance_uuid, translation_target=None): thread = ChatThreads.ChatThread(thread_id, thread_instance_uuid) if not thread.exists(): return {"status": "error", "reason": "Unknown thread"}, 404 meta = thread.get_meta({'chat', 'nb_messages'}) # if meta['chat']: # meta['chat'] = get_chat_meta_from_global_id(meta['chat']) - meta['messages'], meta['tags_messages'] = thread.get_messages() + meta['messages'], meta['tags_messages'] = thread.get_messages(translation_target=translation_target) return meta, 200 def api_get_message(message_id): @@ -362,6 +363,14 @@ def api_get_message(message_id): # # meta['chat'] = return meta, 200 +def api_get_user_account(user_id, instance_uuid): + user_account = UsersAccount.UserAccount(user_id, instance_uuid) + if not user_account.exists(): + return {"status": "error", "reason": "Unknown user-account"}, 404 + meta = user_account.get_meta({'icon', 'username'}) + print(meta) + return meta, 200 + # # # # # # # # # # LATER # # # ChatCategory # diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 6fb1bb35..e8d422fd 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -18,6 +18,7 @@ sys.path.append(os.environ['AIL_BIN']) from lib.ail_core import get_ail_uuid from lib.objects.abstract_object import AbstractObject from lib.ConfigLoader import ConfigLoader +from lib import Language from lib.objects import UsersAccount from lib.data_retention_engine import update_obj_date, get_obj_date_first # TODO Set all messages ??? @@ -76,7 +77,13 @@ class Message(AbstractObject): """ Returns content """ - content = self._get_field('content') + global_id = self.get_global_id() + content = r_cache.get(f'content:{global_id}') + if not content: + content = self._get_field('content') + if content: + r_cache.set(f'content:{global_id}', content) + r_cache.expire(f'content:{global_id}', 300) if r_type == 'str': return content elif r_type == 'bytes': @@ -153,11 +160,23 @@ class Message(AbstractObject): # message from channel ??? # message media - def get_translation(self): # TODO support multiple translated languages ????? + def get_translation(self, content=None, source=None, target='fr'): """ Returns translated content """ - return self._get_field('translated') # TODO multiples translation ... -> use set + # return self._get_field('translated') + global_id = self.get_global_id() + translation = r_cache.get(f'translation:{target}:{global_id}') + r_cache.expire(f'translation:{target}:{global_id}', 0) + if translation: + return translation + if not content: + content = self.get_content() + translation = Language.LanguageTranslator().translate(content, source=source, target=target) + if translation: + r_cache.set(f'translation:{target}:{global_id}', translation) + r_cache.expire(f'translation:{target}:{global_id}', 300) + return translation def _set_translation(self, translation): """ @@ -209,7 +228,7 @@ class Message(AbstractObject): # return r_object.hget(f'meta:item::{self.id}', 'url') # options: set of optional meta fields - def get_meta(self, options=None, timestamp=None): + def get_meta(self, options=None, timestamp=None, translation_target='en'): """ :type options: set :type timestamp: float @@ -239,7 +258,7 @@ class Message(AbstractObject): parent_type, _, parent_id = meta['parent'].split(':', 3) if parent_type == 'message': message = Message(parent_id) - meta['reply_to'] = message.get_meta(options=options) + meta['reply_to'] = message.get_meta(options=options, translation_target=translation_target) if 'investigations' in options: meta['investigations'] = self.get_investigations() if 'link' in options: @@ -262,6 +281,8 @@ class Message(AbstractObject): meta['files-names'] = self.get_files_names() if 'reactions' in options: meta['reactions'] = self.get_reactions() + if 'translation' in options and translation_target: + meta['translation'] = self.get_translation(content=meta.get('content'), target=translation_target) # meta['encoding'] = None return meta diff --git a/bin/lib/objects/abstract_chat_object.py b/bin/lib/objects/abstract_chat_object.py index be25eecb..2ae5ab56 100755 --- a/bin/lib/objects/abstract_chat_object.py +++ b/bin/lib/objects/abstract_chat_object.py @@ -179,12 +179,12 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): week_date = Date.get_current_week_day() return self.get_nb_message_by_week(week_date) - def get_message_meta(self, message, timestamp=None): # TODO handle file message + def get_message_meta(self, message, timestamp=None, translation_target='en'): # TODO handle file message message = Messages.Message(message[9:]) - meta = message.get_meta(options={'content', 'files-names', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'user-account'}, timestamp=timestamp) + meta = message.get_meta(options={'content', 'files-names', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'}, timestamp=timestamp, translation_target=translation_target) return meta - def get_messages(self, start=0, page=1, nb=500, unread=False): # threads ???? # TODO ADD last/first message timestamp + return page + def get_messages(self, start=0, page=1, nb=500, unread=False, translation_target='en'): # threads ???? # TODO ADD last/first message timestamp + return page # TODO return message meta tags = {} messages = {} @@ -195,7 +195,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): if date_day != curr_date: messages[date_day] = [] curr_date = date_day - mess_dict = self.get_message_meta(message[0], timestamp=timestamp) + mess_dict = self.get_message_meta(message[0], timestamp=timestamp, translation_target=translation_target) messages[date_day].append(mess_dict) if mess_dict.get('tags'): diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index 105fdc74..d152578b 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -262,6 +262,9 @@ default_har = True default_screenshot = True onion_proxy = onion.foundation +[Translation] +libretranslate = + [IP] # list of comma-separated CIDR that you wish to be alerted for. e.g: #networks = 192.168.34.0/24,10.0.0.0/8,192.168.33.0/24 diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index 925efacd..9e488273 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -21,6 +21,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from lib import ail_core from lib import chats_viewer +from lib import Language from lib import Tag # ============ BLUEPRINT ============ @@ -80,12 +81,14 @@ def chats_explorer_instance(): def chats_explorer_chat(): chat_id = request.args.get('id') instance_uuid = request.args.get('uuid') - chat = chats_viewer.api_get_chat(chat_id, instance_uuid) + target = request.args.get('target') + chat = chats_viewer.api_get_chat(chat_id, instance_uuid, translation_target=target) if chat[1] != 200: return create_json_response(chat[0], chat[1]) else: chat = chat[0] - return render_template('chat_viewer.html', chat=chat, bootstrap_label=bootstrap_label) + languages = Language.get_translation_languages() + return render_template('chat_viewer.html', chat=chat, bootstrap_label=bootstrap_label, translation_languages=languages, translation_target=target) @chats_explorer.route("chats/explorer/messages/stats/week", methods=['GET']) @login_required @@ -105,12 +108,14 @@ def chats_explorer_messages_stats_week(): def objects_subchannel_messages(): subchannel_id = request.args.get('id') instance_uuid = request.args.get('uuid') - subchannel = chats_viewer.api_get_subchannel(subchannel_id, instance_uuid) + target = request.args.get('target') + subchannel = chats_viewer.api_get_subchannel(subchannel_id, instance_uuid, translation_target=target) if subchannel[1] != 200: return create_json_response(subchannel[0], subchannel[1]) else: subchannel = subchannel[0] - return render_template('SubChannelMessages.html', subchannel=subchannel, bootstrap_label=bootstrap_label) + languages = Language.get_translation_languages() + return render_template('SubChannelMessages.html', subchannel=subchannel, bootstrap_label=bootstrap_label, translation_languages=languages, translation_target=target) @chats_explorer.route("/chats/explorer/thread", methods=['GET']) @login_required @@ -118,12 +123,14 @@ def objects_subchannel_messages(): def objects_thread_messages(): thread_id = request.args.get('id') instance_uuid = request.args.get('uuid') - thread = chats_viewer.api_get_thread(thread_id, instance_uuid) + target = request.args.get('target') + thread = chats_viewer.api_get_thread(thread_id, instance_uuid, translation_target=target) if thread[1] != 200: return create_json_response(thread[0], thread[1]) else: meta = thread[0] - return render_template('ThreadMessages.html', meta=meta, bootstrap_label=bootstrap_label) + languages = Language.get_translation_languages() + return render_template('ThreadMessages.html', meta=meta, bootstrap_label=bootstrap_label, translation_languages=languages, translation_target=target) @chats_explorer.route("/objects/message", methods=['GET']) @login_required @@ -135,5 +142,6 @@ def objects_message(): return create_json_response(message[0], message[1]) else: message = message[0] + languages = Language.get_translation_languages() return render_template('ChatMessage.html', meta=message, bootstrap_label=bootstrap_label, modal_add_tags=Tag.get_modal_add_tags(message['id'], object_type='message')) diff --git a/var/www/templates/chats_explorer/SubChannelMessages.html b/var/www/templates/chats_explorer/SubChannelMessages.html index 3b807b96..f9d42e5b 100644 --- a/var/www/templates/chats_explorer/SubChannelMessages.html +++ b/var/www/templates/chats_explorer/SubChannelMessages.html @@ -184,6 +184,9 @@ {% include 'objects/image/block_blur_img_slider.html' %} + {% with translate_url=url_for('chats_explorer.objects_subchannel_messages', uuid=subchannel['subtype']), obj_id=subchannel['id'] %} + {% include 'chats_explorer/block_translation.html' %} + {% endwith %}
    diff --git a/var/www/templates/chats_explorer/ThreadMessages.html b/var/www/templates/chats_explorer/ThreadMessages.html index 1e8e588e..be4f854b 100644 --- a/var/www/templates/chats_explorer/ThreadMessages.html +++ b/var/www/templates/chats_explorer/ThreadMessages.html @@ -133,6 +133,9 @@ {% include 'objects/image/block_blur_img_slider.html' %} + {% with translate_url=url_for('chats_explorer.objects_thread_messages', uuid=meta['subtype']), obj_id=meta['id'] %} + {% include 'chats_explorer/block_translation.html' %} + {% endwith %}
    diff --git a/var/www/templates/chats_explorer/block_message.html b/var/www/templates/chats_explorer/block_message.html index 019de791..1a60814c 100644 --- a/var/www/templates/chats_explorer/block_message.html +++ b/var/www/templates/chats_explorer/block_message.html @@ -48,6 +48,10 @@ {% endif %} 
    {{ message['reply_to']['content'] }}
    + {% if message['reply_to']['translation'] %} +
    +
    {{ message['reply_to']['translation'] }}
    + {% endif %} {% for tag in message['reply_to']['tags'] %} {{ tag }} {% endfor %} @@ -71,6 +75,10 @@ {% endfor %} {% endif %} 
    {{ message['content'] }}
    + {% if message['translation'] %} +
    + 
    {{ message['translation'] }}
    + {% endif %} {% for reaction in message['reactions'] %} {{ reaction }} {{ message['reactions'][reaction] }} {% endfor %} diff --git a/var/www/templates/chats_explorer/block_translation.html b/var/www/templates/chats_explorer/block_translation.html new file mode 100644 index 00000000..84749798 --- /dev/null +++ b/var/www/templates/chats_explorer/block_translation.html @@ -0,0 +1,37 @@ +
    +
    +
    +
    + Translation +
    +
    + +
    +
    + +
    +
    +
    +
    + + + + \ No newline at end of file diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html index 9741fdce..51023081 100644 --- a/var/www/templates/chats_explorer/chat_viewer.html +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -159,6 +159,10 @@ {% include 'objects/image/block_blur_img_slider.html' %} + {% with translate_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), obj_id=chat['id'] %} + {% include 'chats_explorer/block_translation.html' %} + {% endwith %} +
    From 38ce17bc8a86032ebb96570e0effdf4aa124fe0d Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 6 Dec 2023 16:26:26 +0100 Subject: [PATCH 142/238] chg: [chats] add chats participants + user-accounts basic template --- bin/importer/feeders/abstract_chats_feeder.py | 67 +++---- bin/lib/ail_core.py | 24 ++- bin/lib/chats_viewer.py | 47 ++++- bin/lib/objects/ChatSubChannels.py | 7 +- bin/lib/objects/ChatThreads.py | 4 + bin/lib/objects/Chats.py | 8 +- bin/lib/objects/UsersAccount.py | 47 +++-- bin/lib/objects/abstract_chat_object.py | 7 +- var/www/blueprints/chats_explorer.py | 27 +++ var/www/blueprints/objects_subtypes.py | 12 +- .../chats_explorer/SubChannelMessages.html | 4 + .../chats_explorer/ThreadMessages.html | 5 + .../chats_explorer/block_message.html | 6 +- .../chats_explorer/chat_participants.html | 174 +++++++++++++++++ .../templates/chats_explorer/chat_viewer.html | 6 +- .../chats_explorer/user_account.html | 184 ++++++++++++++++++ .../templates/sidebars/sidebar_objects.html | 6 + 17 files changed, 558 insertions(+), 77 deletions(-) create mode 100644 var/www/templates/chats_explorer/chat_participants.html create mode 100644 var/www/templates/chats_explorer/user_account.html diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py index aebf76ef..a966e951 100755 --- a/bin/importer/feeders/abstract_chats_feeder.py +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -11,7 +11,7 @@ import datetime import os import sys -from abc import abstractmethod, ABC +from abc import ABC sys.path.append(os.environ['AIL_BIN']) ################################## @@ -144,6 +144,8 @@ class AbstractChatFeeder(DefaultFeeder, ABC): def process_chat(self, new_objs, obj, date, timestamp, reply_id=None): meta = self.json_data['meta']['chat'] # todo replace me by function chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid()) + subchannel = None + thread = None # date stat + correlation chat.add(date, obj) @@ -168,24 +170,26 @@ class AbstractChatFeeder(DefaultFeeder, ABC): chat.update_username_timeline(username.get_global_id(), timestamp) if meta.get('subchannel'): - subchannel = self.process_subchannel(obj, date, timestamp, reply_id=reply_id) + subchannel, thread = self.process_subchannel(obj, date, timestamp, reply_id=reply_id) chat.add_children(obj_global_id=subchannel.get_global_id()) else: if obj.type == 'message': if self.get_thread_id(): - self.process_thread(obj, chat, date, timestamp, reply_id=reply_id) + thread = self.process_thread(obj, chat, date, timestamp, reply_id=reply_id) else: chat.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id) - - # if meta.get('subchannels'): # TODO Update icon + names - - return chat - + chats_obj = [chat] + if subchannel: + chats_obj.append(subchannel) + if thread: + chats_obj.append(thread) + return chats_obj def process_subchannel(self, obj, date, timestamp, reply_id=None): # TODO CREATE DATE meta = self.json_data['meta']['chat']['subchannel'] subchannel = ChatSubChannels.ChatSubChannel(f'{self.get_chat_id()}/{meta["id"]}', self.get_chat_instance_uuid()) + thread = None # TODO correlation with obj = message/image subchannel.add(date) @@ -202,10 +206,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC): if obj.type == 'message': if self.get_thread_id(): - self.process_thread(obj, subchannel, date, timestamp, reply_id=reply_id) + thread = self.process_thread(obj, subchannel, date, timestamp, reply_id=reply_id) else: subchannel.add_message(obj.get_global_id(), self.get_message_id(), timestamp, reply_id=reply_id) - return subchannel + return subchannel, thread def process_thread(self, obj, obj_chat, date, timestamp, reply_id=None): meta = self.json_data['meta']['thread'] @@ -231,7 +235,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # else: # # ADD NEW MESSAGE REF (used by discord) - def process_sender(self, new_objs, obj, date, timestamp): meta = self.json_data['meta']['sender'] user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid()) @@ -267,12 +270,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC): return user_account - # Create abstract class: -> new API endpoint ??? => force field, check if already imported ? - # 1) Create/Get MessageInstance - # TODO uuidv5 + text like discord and telegram for default - # 2) Create/Get CHAT ID - Done - # 3) Create/Get Channel IF is in channel - # 4) Create/Get Thread IF is in thread - # 5) Create/Update Username and User-account - Done def process_meta(self): # TODO CHECK MANDATORY FIELDS """ Process JSON meta filed. @@ -316,7 +313,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC): message_id = self.get_message_id() message_id = Messages.create_obj_id(self.get_chat_instance_uuid(), chat_id, message_id, timestamp, channel_id=channel_id, thread_id=thread_id) message = Messages.Message(message_id) - # create empty message if message don't exists + # create empty message if message don't exist if not message.exists(): message.create('') objs.add(message) @@ -336,46 +333,26 @@ class AbstractChatFeeder(DefaultFeeder, ABC): for obj in objs: # TODO PERF avoid parsing metas multiple times + # TODO get created subchannel + thread + # => create correlation user-account with object + # CHAT - chat = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id) + chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id) # SENDER # TODO HANDLE NULL SENDER user_account = self.process_sender(new_objs, obj, date, timestamp) - # UserAccount---Chat - user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id) + # UserAccount---ChatObjects + for obj_chat in chat_objs: + user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id) # if chat: # TODO Chat---Username correlation ??? # # Chat---Username => need to handle members and participants # chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id) - # TODO Sender image -> correlation # image # -> subchannel ? # -> thread id ? return new_objs | objs - - - - - - - - - - - - - - - - - - - - - - - diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 3e34e4b8..00cefd4d 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -40,7 +40,7 @@ def get_all_objects(): return AIL_OBJECTS def get_objects_with_subtypes(): - return ['chat', 'cryptocurrency', 'pgp', 'username'] + return ['chat', 'cryptocurrency', 'pgp', 'username', 'user-account'] def get_object_all_subtypes(obj_type): # TODO Dynamic subtype if obj_type == 'chat': @@ -53,6 +53,8 @@ def get_object_all_subtypes(obj_type): # TODO Dynamic subtype return ['key', 'mail', 'name'] if obj_type == 'username': return ['telegram', 'twitter', 'jabber'] + if obj_type == 'user-account': + return r_object.smembers(f'all_chat:subtypes') return [] def get_objects_tracked(): @@ -75,10 +77,28 @@ def get_all_objects_with_subtypes_tuple(): def unpack_obj_global_id(global_id, r_type='tuple'): if r_type == 'dict': obj = global_id.split(':', 2) - return {'type': obj[0], 'subtype': obj[1], 'id': obj['2']} + return {'type': obj[0], 'subtype': obj[1], 'id': obj[2]} else: # tuple(type, subtype, id) return global_id.split(':', 2) +def unpack_objs_global_id(objs_global_id, r_type='tuple'): + objs = [] + for global_id in objs_global_id: + objs.append(unpack_obj_global_id(global_id, r_type=r_type)) + return objs + +def unpack_correl_obj__id(obj_type, global_id, r_type='tuple'): + obj = global_id.split(':', 1) + if r_type == 'dict': + return {'type': obj_type, 'subtype': obj[0], 'id': obj[1]} + else: # tuple(type, subtype, id) + return obj_type, obj[0], obj[1] + +def unpack_correl_objs_id(obj_type, correl_objs_id, r_type='tuple'): + objs = [] + for correl_obj_id in correl_objs_id: + objs.append(unpack_correl_obj__id(obj_type, correl_obj_id, r_type=r_type)) + return objs ##-- AIL OBJECTS --## diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 1d983f0a..b083537a 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -278,6 +278,27 @@ def create_chat_service_instance(protocol, network=None, address=None): ####################################################################################### +def get_obj_chat(chat_type, chat_subtype, chat_id): + print(chat_type, chat_subtype, chat_id) + if chat_type == 'chat': + return Chats.Chat(chat_id, chat_subtype) + elif chat_type == 'chat-subchannel': + return ChatSubChannels.ChatSubChannel(chat_id, chat_subtype) + elif chat_type == 'chat-thread': + return ChatThreads.ChatThread(chat_id, chat_subtype) + +def get_obj_chat_meta(obj_chat, new_options=set()): + options = {} + if obj_chat.type == 'chat': + options = {'created_at', 'icon', 'info', 'subchannels', 'threads', 'username'} + elif obj_chat.type == 'chat-subchannel': + options = {'chat', 'created_at', 'icon', 'nb_messages', 'threads'} + elif obj_chat.type == 'chat-thread': + options = {'chat', 'nb_messages'} + for option in new_options: + options.add(option) + return obj_chat.get_meta(options=options) + def get_subchannels_meta_from_global_id(subchannels): meta = [] for sub in subchannels: @@ -302,6 +323,8 @@ def get_username_meta_from_global_id(username_global_id): username = Usernames.Username(username_id, instance_uuid) return username.get_meta() +#### API #### + def api_get_chat_service_instance(chat_instance_uuid): chat_instance = ChatServiceInstance(chat_instance_uuid) if not chat_instance.exists(): @@ -312,7 +335,7 @@ def api_get_chat(chat_id, chat_instance_uuid, translation_target=None): chat = Chats.Chat(chat_id, chat_instance_uuid) if not chat.exists(): return {"status": "error", "reason": "Unknown chat"}, 404 - meta = chat.get_meta({'created_at', 'icon', 'info', 'subchannels', 'threads', 'username'}) + meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'username'}) if meta['username']: meta['username'] = get_username_meta_from_global_id(meta['username']) if meta['subchannels']: @@ -329,11 +352,26 @@ def api_get_nb_message_by_week(chat_id, chat_instance_uuid): # week = chat.get_nb_message_by_week('20231109') return week, 200 +def api_get_chat_participants(chat_type, chat_subtype, chat_id): + if chat_type not in ['chat', 'chat-subchannel', 'chat-thread']: + return {"status": "error", "reason": "Unknown chat type"}, 400 + chat_obj = get_obj_chat(chat_type, chat_subtype, chat_id) + if not chat_obj.exists(): + return {"status": "error", "reason": "Unknown chat"}, 404 + else: + meta = get_obj_chat_meta(chat_obj, new_options={'participants'}) + chat_participants = [] + for participant in meta['participants']: + user_account = UsersAccount.UserAccount(participant['id'], participant['subtype']) + chat_participants.append(user_account.get_meta({'icon', 'info', 'username'})) + meta['participants'] = chat_participants + return meta, 200 + def api_get_subchannel(chat_id, chat_instance_uuid, translation_target=None): subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid) if not subchannel.exists(): return {"status": "error", "reason": "Unknown subchannel"}, 404 - meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages', 'threads'}) + meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages', 'nb_participants', 'threads'}) if meta['chat']: meta['chat'] = get_chat_meta_from_global_id(meta['chat']) if meta.get('threads'): @@ -347,7 +385,7 @@ def api_get_thread(thread_id, thread_instance_uuid, translation_target=None): thread = ChatThreads.ChatThread(thread_id, thread_instance_uuid) if not thread.exists(): return {"status": "error", "reason": "Unknown thread"}, 404 - meta = thread.get_meta({'chat', 'nb_messages'}) + meta = thread.get_meta({'chat', 'nb_messages', 'nb_participants'}) # if meta['chat']: # meta['chat'] = get_chat_meta_from_global_id(meta['chat']) meta['messages'], meta['tags_messages'] = thread.get_messages(translation_target=translation_target) @@ -367,8 +405,7 @@ def api_get_user_account(user_id, instance_uuid): user_account = UsersAccount.UserAccount(user_id, instance_uuid) if not user_account.exists(): return {"status": "error", "reason": "Unknown user-account"}, 404 - meta = user_account.get_meta({'icon', 'username'}) - print(meta) + meta = user_account.get_meta({'chats', 'icon', 'info', 'subchannels', 'threads', 'username', 'username_meta'}) return meta, 200 # # # # # # # # # # LATER diff --git a/bin/lib/objects/ChatSubChannels.py b/bin/lib/objects/ChatSubChannels.py index 38a50a20..7a799240 100755 --- a/bin/lib/objects/ChatSubChannels.py +++ b/bin/lib/objects/ChatSubChannels.py @@ -70,7 +70,7 @@ class ChatSubChannel(AbstractChatObject): # else: # style = 'fas' # icon = '\uf007' - style = 'fas' + style = 'far' icon = '\uf086' return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5} @@ -90,7 +90,10 @@ class ChatSubChannel(AbstractChatObject): meta['created_at'] = self.get_created_at(date=True) if 'threads' in options: meta['threads'] = self.get_threads() - print(meta['threads']) + if 'participants' in options: + meta['participants'] = self.get_participants() + if 'nb_participants' in options: + meta['nb_participants'] = self.get_nb_participants() return meta def get_misp_object(self): diff --git a/bin/lib/objects/ChatThreads.py b/bin/lib/objects/ChatThreads.py index 9514a248..0609faab 100755 --- a/bin/lib/objects/ChatThreads.py +++ b/bin/lib/objects/ChatThreads.py @@ -77,6 +77,10 @@ class ChatThread(AbstractChatObject): meta['name'] = self.get_name() if 'nb_messages': meta['nb_messages'] = self.get_nb_messages() + if 'participants': + meta['participants'] = self.get_participants() + if 'nb_participants': + meta['nb_participants'] = self.get_nb_participants() # created_at ??? return meta diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index b631b2d2..040c3ea5 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -74,10 +74,14 @@ class Chat(AbstractChatObject): meta = self._get_meta(options=options) meta['name'] = self.get_name() meta['tags'] = self.get_tags(r_list=True) - if 'icon': + if 'icon' in options: meta['icon'] = self.get_icon() - if 'info': + if 'info' in options: meta['info'] = self.get_info() + if 'participants' in options: + meta['participants'] = self.get_participants() + if 'nb_participants' in options: + meta['nb_participants'] = self.get_nb_participants() if 'username' in options: meta['username'] = self.get_username() if 'subchannels' in options: diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py index ced2d682..92076f24 100755 --- a/bin/lib/objects/UsersAccount.py +++ b/bin/lib/objects/UsersAccount.py @@ -54,16 +54,7 @@ class UserAccount(AbstractSubtypeObject): return url def get_svg_icon(self): # TODO change icon/color - if self.subtype == 'telegram': - style = 'fab' - icon = '\uf2c6' - elif self.subtype == 'twitter': - style = 'fab' - icon = '\uf099' - else: - style = 'fas' - icon = '\uf007' - return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5} + return {'style': 'fas', 'icon': '\uf2bd', 'color': '#4dffff', 'radius': 5} def get_first_name(self): return self._get_field('firstname') @@ -97,6 +88,25 @@ class UserAccount(AbstractSubtypeObject): def set_info(self, info): return self._set_field('info', info) + # TODO MESSAGES: + # 1) ALL MESSAGES + NB + # 2) ALL MESSAGES TIMESTAMP + # 3) ALL MESSAGES TIMESTAMP By: - chats + # - subchannel + # - thread + + def get_chats(self): + chats = self.get_correlation('chat')['chat'] + return chats + + def get_chat_subchannels(self): + chats = self.get_correlation('chat-subchannel')['chat-subchannel'] + return chats + + def get_chat_threads(self): + chats = self.get_correlation('chat-thread')['chat-thread'] + return chats + def _get_timeline_username(self): return Timeline(self.get_global_id(), 'username') @@ -109,20 +119,31 @@ class UserAccount(AbstractSubtypeObject): def update_username_timeline(self, username_global_id, timestamp): self._get_timeline_username().add_timestamp(timestamp, username_global_id) - def get_meta(self, options=set()): + def get_meta(self, options=set()): # TODO Username timeline meta = self._get_meta(options=options) meta['id'] = self.id meta['subtype'] = self.subtype meta['tags'] = self.get_tags(r_list=True) # TODO add in options ???? if 'username' in options: meta['username'] = self.get_username() - if meta['username'] and 'username_meta' in options: + if meta['username']: _, username_account_subtype, username_account_id = meta['username'].split(':', 3) - meta['username'] = Usernames.Username(username_account_id, username_account_subtype).get_meta() + if 'username_meta' in options: + meta['username'] = Usernames.Username(username_account_id, username_account_subtype).get_meta() + else: + meta['username'] = {'type': 'username', 'subtype': username_account_subtype, 'id': username_account_id} if 'usernames' in options: meta['usernames'] = self.get_usernames() if 'icon' in options: meta['icon'] = self.get_icon() + if 'info' in options: + meta['info'] = self.get_info() + if 'chats' in options: + meta['chats'] = self.get_chats() + if 'subchannels' in options: + meta['subchannels'] = self.get_chat_subchannels() + if 'threads' in options: + meta['threads'] = self.get_chat_threads() return meta def get_misp_object(self): diff --git a/bin/lib/objects/abstract_chat_object.py b/bin/lib/objects/abstract_chat_object.py index 2ae5ab56..5311cb85 100755 --- a/bin/lib/objects/abstract_chat_object.py +++ b/bin/lib/objects/abstract_chat_object.py @@ -19,7 +19,7 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from lib.objects.abstract_subtype_object import AbstractSubtypeObject -from lib.ail_core import get_object_all_subtypes, zscan_iter ################ +from lib.ail_core import unpack_correl_objs_id, zscan_iter ################ from lib.ConfigLoader import ConfigLoader from lib.objects import Messages from packages import Date @@ -244,8 +244,11 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): # def get_deleted_messages(self, message_id): + def get_participants(self): + return unpack_correl_objs_id('user-account', self.get_correlation('user-account')['user-account'], r_type='dict') - # get_messages_meta ???? + def get_nb_participants(self): + return self.get_nb_correlation('user-account') # TODO move me to abstract subtype class AbstractChatObjects(ABC): diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index 9e488273..54ae3d25 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -132,6 +132,20 @@ def objects_thread_messages(): languages = Language.get_translation_languages() return render_template('ThreadMessages.html', meta=meta, bootstrap_label=bootstrap_label, translation_languages=languages, translation_target=target) +@chats_explorer.route("/chats/explorer/participants", methods=['GET']) +@login_required +@login_read_only +def chats_explorer_chat_participants(): + chat_type = request.args.get('type') + chat_id = request.args.get('id') + chat_subtype = request.args.get('subtype') + meta = chats_viewer.api_get_chat_participants(chat_type, chat_subtype,chat_id) + if meta[1] != 200: + return create_json_response(meta[0], meta[1]) + else: + meta = meta[0] + return render_template('chat_participants.html', meta=meta, bootstrap_label=bootstrap_label) + @chats_explorer.route("/objects/message", methods=['GET']) @login_required @login_read_only @@ -145,3 +159,16 @@ def objects_message(): languages = Language.get_translation_languages() return render_template('ChatMessage.html', meta=message, bootstrap_label=bootstrap_label, modal_add_tags=Tag.get_modal_add_tags(message['id'], object_type='message')) + +@chats_explorer.route("/objects/user-account", methods=['GET']) +@login_required +@login_read_only +def objects_user_account(): + instance_uuid = request.args.get('subtype') + user_id = request.args.get('id') + user_account = chats_viewer.api_get_user_account(user_id, instance_uuid) + if user_account[1] != 200: + return create_json_response(user_account[0], user_account[1]) + else: + user_account = user_account[0] + return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label) \ No newline at end of file diff --git a/var/www/blueprints/objects_subtypes.py b/var/www/blueprints/objects_subtypes.py index a41066a4..97229ff1 100644 --- a/var/www/blueprints/objects_subtypes.py +++ b/var/www/blueprints/objects_subtypes.py @@ -43,7 +43,8 @@ def subtypes_objects_dashboard(obj_type, f_request): date_to = f_request.form.get('to') subtype = f_request.form.get('subtype') show_objects = bool(f_request.form.get('show_objects')) - endpoint_dashboard = url_for(f'objects_subtypes.objects_dashboard_{obj_type}') + t_obj_type = obj_type.replace('-', '_') + endpoint_dashboard = url_for(f'objects_subtypes.objects_dashboard_{t_obj_type}') endpoint_dashboard = f'{endpoint_dashboard}?from={date_from}&to={date_to}' if subtype: if subtype == 'All types': @@ -81,7 +82,8 @@ def subtypes_objects_dashboard(obj_type, f_request): for obj_t, obj_subtype, obj_id in subtypes_objs: objs.append(ail_objects.get_object_meta(obj_t, obj_subtype, obj_id, options={'sparkline'}, flask_context=True)) - endpoint_dashboard = f'objects_subtypes.objects_dashboard_{obj_type}' + t_obj_type = obj_type.replace('-', '_') + endpoint_dashboard = f'objects_subtypes.objects_dashboard_{t_obj_type}' return render_template('subtypes_objs_dashboard.html', date_from=date_from, date_to=date_to, daily_type_chart = daily_type_chart, show_objects=show_objects, obj_type=obj_type, subtype=subtype, objs=objs, @@ -115,6 +117,12 @@ def objects_dashboard_pgp(): def objects_dashboard_username(): return subtypes_objects_dashboard('username', request) +@objects_subtypes.route("/objects/user-accounts", methods=['GET']) +@login_required +@login_read_only +def objects_dashboard_user_account(): + return subtypes_objects_dashboard('user-account', request) + # TODO REDIRECT @objects_subtypes.route("/objects/subtypes/post", methods=['POST']) @login_required diff --git a/var/www/templates/chats_explorer/SubChannelMessages.html b/var/www/templates/chats_explorer/SubChannelMessages.html index f9d42e5b..7ca7da41 100644 --- a/var/www/templates/chats_explorer/SubChannelMessages.html +++ b/var/www/templates/chats_explorer/SubChannelMessages.html @@ -68,6 +68,7 @@
    		Last seen Username Nb MessagesParticipants
    {{ subchannel['nb_messages'] }} + {{ subchannel['nb_participants']}} +
    diff --git a/var/www/templates/chats_explorer/ThreadMessages.html b/var/www/templates/chats_explorer/ThreadMessages.html index be4f854b..49d26ce9 100644 --- a/var/www/templates/chats_explorer/ThreadMessages.html +++ b/var/www/templates/chats_explorer/ThreadMessages.html @@ -67,6 +67,7 @@ First seen Last seen Nb Messages + Participants @@ -74,6 +75,7 @@ {{ meta['name'] }} + {% if meta['first_seen'] %} {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }} @@ -85,6 +87,9 @@ {% endif %} {{ meta['nb_messages'] }} + + {{ meta['nb_participants']}} + diff --git a/var/www/templates/chats_explorer/block_message.html b/var/www/templates/chats_explorer/block_message.html index 1a60814c..cf407128 100644 --- a/var/www/templates/chats_explorer/block_message.html +++ b/var/www/templates/chats_explorer/block_message.html @@ -26,8 +26,10 @@
    - {{ message['user-account']['id'] }} + + {{ message['user-account']['id'] }} +
    {{ message['hour'] }}
    diff --git a/var/www/templates/chats_explorer/chat_participants.html b/var/www/templates/chats_explorer/chat_participants.html new file mode 100644 index 00000000..455a39b2 --- /dev/null +++ b/var/www/templates/chats_explorer/chat_participants.html @@ -0,0 +1,174 @@ + + + + + Chats Protocols - AIL + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + +{#
    #} TODO CHAT abstract metadata +{##} +{#
    #} +{#

    {% if chat['username'] %}{{ chat["username"]["id"] }} {% else %} {{ chat['name'] }}{% endif %} :

    #} +{# {% if chat['icon'] %}#} +{#
    {{ chat['id'] }}
    #} +{# {% endif %}#} +{#
      #} +{#
    • #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{#
      NameIDCreated atFirst SeenLast SeenNB Sub-ChannelsParticipants
      {{ chat['name'] }}{{ chat['id'] }}{{ chat['created_at'] }}#} +{# {% if chat['first_seen'] %}#} +{# {{ chat['first_seen'][0:4] }}-{{ chat['first_seen'][4:6] }}-{{ chat['first_seen'][6:8] }}#} +{# {% endif %}#} +{# #} +{# {% if chat['last_seen'] %}#} +{# {{ chat['last_seen'][0:4] }}-{{ chat['last_seen'][4:6] }}-{{ chat['last_seen'][6:8] }}#} +{# {% endif %}#} +{# {{ chat['nb_subchannels'] }}#} +{# {{ chat['participants'] | length}}#} +{##} +{#
      #} +{# {% if chat['info'] %}#} +{#
    • #} +{# 
      {{ chat['info'] }}
      #} +{#
      • #} +{# {% endif %}#} +{# #} +{#
    #} +{##} +{#
    #} +{#
    #} + + +

    Participants:

    + + + + + + + + + + + + + + {% for user_meta in meta["participants"] %} + + + + + + + + + + {% endfor %} + +
    IconUsernameIDinfoFirst SeenLast SeenNB Messages
    + + {{ user_meta['id'] }} + + + {% if user_meta['username'] %} + {{ user_meta['username']['id'] }} + {% endif %} + {{ user_meta['id'] }} + {% if user_meta['info'] %} + {{ user_meta['info'] }} + {% endif %} + + {% if user_meta['first_seen'] %} + {{ user_meta['first_seen'][0:4] }}-{{ user_meta['first_seen'][4:6] }}-{{ user_meta['first_seen'][6:8] }} + {% endif %} + + {% if user_meta['last_seen'] %} + {{ user_meta['last_seen'][0:4] }}-{{ user_meta['last_seen'][4:6] }}-{{ user_meta['last_seen'][6:8] }} + {% endif %} + {{ user_meta['nb_messages'] }}
    + +
    + +
    +
    + + + + + + diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html index 51023081..81f7e407 100644 --- a/var/www/templates/chats_explorer/chat_viewer.html +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -66,18 +66,17 @@ - + - @@ -92,6 +91,9 @@ {% endif %} +
    Icon Name ID Created at First Seen Last Seen NB Sub-ChannelsParticipants
    {{ chat['name'] }} {{ chat['id'] }} {{ chat['created_at'] }} {{ chat['nb_subchannels'] }} + {{ chat['nb_participants']}} +
    diff --git a/var/www/templates/chats_explorer/user_account.html b/var/www/templates/chats_explorer/user_account.html new file mode 100644 index 00000000..ee8d4e88 --- /dev/null +++ b/var/www/templates/chats_explorer/user_account.html @@ -0,0 +1,184 @@ + + + + + User Account - AIL + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + +
    + +
    +

    {% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['id'] }}{% endif %}

    + {% if meta['icon'] %} +
    {{ meta['id'] }}
    + {% endif %} +
      +
    • + + + + + + + + + + + + + + + + + + + + + +
      usernameIDCreated atFirst SeenLast SeenNB Chats
      {{ meta['username']['id'] }}{{ meta['id'] }}{{ meta['created_at'] }} + {% if meta['first_seen'] %} + {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }} + {% endif %} + + {% if meta['last_seen'] %} + {{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }} + {% endif %} + {{ meta['chats'] | length }}
      + {% if meta['info'] %} +
    • + 
      {{ meta['info'] }}
      +
      • + {% endif %} + +
    + +
    + +{#
    #} +{# {% with obj_type=meta['type'], obj_id=meta['id'], obj_subtype=''%}#} +{# {% include 'modals/investigations_register_obj.html' %}#} +{# {% endwith %}#} +{#
    #} +{# #} +{#
    #} +{#
    #} +
    + +
    +
    + + +{# {% if meta['subchannels'] %}#} +{#

    Sub-Channels:

    #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# {% for meta in meta["subchannels"] %}#} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# #} +{# {% endfor %}#} +{# #} +{#
    #} +{# {{ meta['id'] }}#} +{# {{ meta['name'] }}{{ meta['id'] }}{{ meta['created_at'] }}#} +{# {% if meta['first_seen'] %}#} +{# {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }}#} +{# {% endif %}#} +{# #} +{# {% if meta['last_seen'] %}#} +{# {{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }}#} +{# {% endif %}#} +{# {{ meta['nb_messages'] }}
    #} +{##} +{# {% endif %}#} + + +
    + +
    +
    + + + + + + + + + + diff --git a/var/www/templates/sidebars/sidebar_objects.html b/var/www/templates/sidebars/sidebar_objects.html index dba94772..d0a64df6 100644 --- a/var/www/templates/sidebars/sidebar_objects.html +++ b/var/www/templates/sidebars/sidebar_objects.html @@ -106,6 +106,12 @@ Username +
  • + + + User-Acoount + +
    • From a382b572c6c614a53928953b5dcc03e161442cf9 Mon Sep 17 00:00:00 2001 From: terrtia Date: Thu, 7 Dec 2023 11:28:35 +0100 Subject: [PATCH 143/238] chg: [crawler] push onion discovery capture_uuid to another AIL --- bin/crawlers/Crawler.py | 18 ++++++ bin/lib/crawlers.py | 80 +++++++++++++++++++----- configs/core.cfg.sample | 2 + var/www/modules/restApi/Flask_restApi.py | 14 +++++ 4 files changed, 97 insertions(+), 17 deletions(-) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index eb492207..06ebe982 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -6,6 +6,7 @@ import logging.config import sys import time +from pyail import PyAIL from requests.exceptions import ConnectionError sys.path.append(os.environ['AIL_BIN']) @@ -44,6 +45,15 @@ class Crawler(AbstractModule): self.default_screenshot = config_loader.get_config_boolean('Crawler', 'default_screenshot') self.default_depth_limit = config_loader.get_config_int('Crawler', 'default_depth_limit') + ail_url_to_push_discovery = config_loader.get_config_str('Crawler', 'ail_url_to_push_onion_discovery') + ail_key_to_push_discovery = config_loader.get_config_str('Crawler', 'ail_key_to_push_onion_discovery') + if ail_url_to_push_discovery and ail_key_to_push_discovery: + ail = PyAIL(ail_url_to_push_discovery, ail_key_to_push_discovery, ssl=False) + if ail.ping_ail(): + self.ail_to_push_discovery = ail + else: + self.ail_to_push_discovery = None + # TODO: LIMIT MAX NUMBERS OF CRAWLED PAGES # update hardcoded blacklist @@ -183,6 +193,14 @@ class Crawler(AbstractModule): crawlers.create_capture(capture_uuid, task_uuid) print(task.uuid, capture_uuid, 'launched') + + if self.ail_to_push_discovery: + if task.get_depth() == 1 and priority < 10 and task.get_domain().endswith('.onion'): + har = task.get_har() + screenshot = task.get_screenshot() + self.ail_to_push_discovery.add_crawler_capture(task_uuid, capture_uuid, url, har=har, + screenshot=screenshot, depth_limit=1, proxy='force_tor') + print(task.uuid, capture_uuid, 'Added to ail_to_push_discovery') return capture_uuid # CRAWL DOMAIN diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 3484afa0..d256daee 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -309,6 +309,16 @@ def get_all_har_ids(): har_ids.append(har_id) return har_ids +def get_month_har_ids(year, month): + har_ids = [] + month_path = os.path.join(HAR_DIR, year, month) + for root, dirs, files in os.walk(month_path): + for file in files: + har_id = os.path.relpath(os.path.join(root, file), HAR_DIR) + har_ids.append(har_id) + return har_ids + + def get_har_content(har_id): har_path = os.path.join(HAR_DIR, har_id) try: @@ -1519,7 +1529,7 @@ class CrawlerTask: # TODO SANITIZE PRIORITY # PRIORITY: discovery = 0/10, feeder = 10, manual = 50, auto = 40, test = 100 def create(self, url, depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None, - user_agent=None, tags=[], parent='manual', priority=0): + user_agent=None, tags=[], parent='manual', priority=0, external=False): if self.exists(): raise Exception('Error: Task already exists') @@ -1576,8 +1586,8 @@ class CrawlerTask: r_crawler.hset('crawler:queue:hash', hash_query, self.uuid) self._set_field('hash', hash_query) - r_crawler.zadd('crawler:queue', {self.uuid: priority}) - self.add_to_db_crawler_queue(priority) + if not external: + self.add_to_db_crawler_queue(priority) # UI domain_type = dom.get_domain_type() r_crawler.sadd(f'crawler:queue:type:{domain_type}', self.uuid) @@ -1637,7 +1647,7 @@ def add_task_to_lacus_queue(): # PRIORITY: discovery = 0/10, feeder = 10, manual = 50, auto = 40, test = 100 def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None, - user_agent=None, tags=[], parent='manual', priority=0, task_uuid=None): + user_agent=None, tags=[], parent='manual', priority=0, task_uuid=None, external=False): if task_uuid: if CrawlerTask(task_uuid).exists(): task_uuid = gen_uuid() @@ -1645,7 +1655,8 @@ def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar= task_uuid = gen_uuid() task = CrawlerTask(task_uuid) task_uuid = task.create(url, depth=depth, har=har, screenshot=screenshot, header=header, cookiejar=cookiejar, - proxy=proxy, user_agent=user_agent, tags=tags, parent=parent, priority=priority) + proxy=proxy, user_agent=user_agent, tags=tags, parent=parent, priority=priority, + external=external) return task_uuid @@ -1655,7 +1666,8 @@ def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar= # # TODO: ADD user agent # # TODO: sanitize URL -def api_add_crawler_task(data, user_id=None): + +def api_parse_task_dict_basic(data, user_id): url = data.get('url', None) if not url or url == '\n': return {'status': 'error', 'reason': 'No url supplied'}, 400 @@ -1681,6 +1693,31 @@ def api_add_crawler_task(data, user_id=None): else: depth_limit = 0 + # PROXY + proxy = data.get('proxy', None) + if proxy == 'onion' or proxy == 'tor' or proxy == 'force_tor': + proxy = 'force_tor' + elif proxy: + verify = api_verify_proxy(proxy) + if verify[1] != 200: + return verify + + tags = data.get('tags', []) + + return {'url': url, 'depth_limit': depth_limit, 'har': har, 'screenshot': screenshot, 'proxy': proxy, 'tags': tags}, 200 + +def api_add_crawler_task(data, user_id=None): + task, resp = api_parse_task_dict_basic(data, user_id) + if resp != 200: + return task, resp + + url = task['url'] + screenshot = task['screenshot'] + har = task['har'] + depth_limit = task['depth_limit'] + proxy = task['proxy'] + tags = task['tags'] + cookiejar_uuid = data.get('cookiejar', None) if cookiejar_uuid: cookiejar = Cookiejar(cookiejar_uuid) @@ -1725,17 +1762,6 @@ def api_add_crawler_task(data, user_id=None): return {'error': 'Invalid frequency'}, 400 frequency = f'{months}:{weeks}:{days}:{hours}:{minutes}' - # PROXY - proxy = data.get('proxy', None) - if proxy == 'onion' or proxy == 'tor' or proxy == 'force_tor': - proxy = 'force_tor' - elif proxy: - verify = api_verify_proxy(proxy) - if verify[1] != 200: - return verify - - tags = data.get('tags', []) - if frequency: # TODO verify user task_uuid = create_schedule(frequency, user_id, url, depth=depth_limit, har=har, screenshot=screenshot, header=None, @@ -1752,6 +1778,26 @@ def api_add_crawler_task(data, user_id=None): #### #### +# TODO cookiejar - cookies - frequency +def api_add_crawler_capture(data, user_id): + task, resp = api_parse_task_dict_basic(data, user_id) + if resp != 200: + return task, resp + + task_uuid = data.get('task_uuid') + if not task_uuid: + return {'error': 'Invalid task_uuid', 'task_uuid': task_uuid}, 400 + capture_uuid = data.get('capture_uuid') + if not capture_uuid: + return {'error': 'Invalid capture_uuid', 'task_uuid': capture_uuid}, 400 + + # TODO parent + create_task(task['url'], depth=task['depth_limit'], har=task['har'], screenshot=task['screenshot'], + proxy=task['proxy'], tags=task['tags'], + parent='AIL_capture', task_uuid=task_uuid, external=True) + + create_capture(capture_uuid, task_uuid) + return capture_uuid, 200 ################################################################################### ################################################################################### diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index d152578b..852590c5 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -261,6 +261,8 @@ default_depth_limit = 1 default_har = True default_screenshot = True onion_proxy = onion.foundation +ail_url_to_push_onion_discovery = +ail_key_to_push_onion_discovery = [Translation] libretranslate = diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index 0c95d0a0..90dba6d6 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -523,6 +523,20 @@ def add_crawler_task(): return create_json_response(dict_res, 200) +@restApi.route("api/v1/add/crawler/capture", methods=['POST']) +@token_required('analyst') +def add_crawler_task(): + data = request.get_json() + user_token = get_auth_from_header() + user_id = Users.get_token_user(user_token) + res = crawlers.api_add_crawler_capture(data, user_id) + if res: + return create_json_response(res[0], res[1]) + + dict_res = {'url': data['url']} + return create_json_response(dict_res, 200) + + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # DOMAIN # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # From 3e591d95bce870daae7105c2d904594df3fa696b Mon Sep 17 00:00:00 2001 From: terrtia Date: Thu, 7 Dec 2023 14:40:51 +0100 Subject: [PATCH 144/238] fix: [retro_hunt] fix daterange --- bin/lib/objects/Items.py | 5 ++++- bin/trackers/Retro_Hunt.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/bin/lib/objects/Items.py b/bin/lib/objects/Items.py index 8aaab0b2..d8888fa0 100755 --- a/bin/lib/objects/Items.py +++ b/bin/lib/objects/Items.py @@ -498,7 +498,10 @@ def get_all_items_objects(filters={}): daterange = Date.get_daterange(date_from, date_to) else: date_from = get_obj_date_first('item') - daterange = Date.get_daterange(date_from, Date.get_today_date_str()) + if date_from: + daterange = Date.get_daterange(date_from, Date.get_today_date_str()) + else: + daterange = [] if start_date: if int(start_date) > int(date_from): i = 0 diff --git a/bin/trackers/Retro_Hunt.py b/bin/trackers/Retro_Hunt.py index 2c63abd0..7a96ce97 100755 --- a/bin/trackers/Retro_Hunt.py +++ b/bin/trackers/Retro_Hunt.py @@ -111,7 +111,10 @@ class Retro_Hunt_Module(AbstractModule): self.redis_logger.warning(f'{self.module_name}, Retro Hunt {task_uuid} completed') def update_progress(self): - new_progress = self.nb_done * 100 / self.nb_objs + if self.nb_objs == 0: + new_progress = 100 + else: + new_progress = self.nb_done * 100 / self.nb_objs if int(self.progress) != int(new_progress): print(new_progress) self.retro_hunt.set_progress(new_progress) From 25c467f11a6b8591cbbde981198cddb0e57420b1 Mon Sep 17 00:00:00 2001 From: terrtia Date: Thu, 7 Dec 2023 14:46:00 +0100 Subject: [PATCH 145/238] fix: [requirement] fix libretranslate, fix #189 + fix #190 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 0412c6bd..6c813802 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,6 +41,7 @@ scrapy-splash>=0.7.2 # Languages pycld3>0.20 +libretranslatepy #Graph numpy>1.18.1 From cea96863ba59977024e8e9b9e1e2befa5529cd9a Mon Sep 17 00:00:00 2001 From: terrtia Date: Thu, 7 Dec 2023 14:59:40 +0100 Subject: [PATCH 146/238] fix: [language] libretranslate unreachable --- bin/lib/Language.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/lib/Language.py b/bin/lib/Language.py index dd8df1c2..7b688e2b 100755 --- a/bin/lib/Language.py +++ b/bin/lib/Language.py @@ -313,8 +313,10 @@ class LanguageTranslator: translation = None return translation - -LIST_LANGUAGES = LanguageTranslator().languages() +try: + LIST_LANGUAGES = LanguageTranslator().languages() +except Exception: + LIST_LANGUAGES = [] def get_translation_languages(): return LIST_LANGUAGES From 3642eb5429008f60777d221e673ced601225675b Mon Sep 17 00:00:00 2001 From: terrtia Date: Thu, 7 Dec 2023 15:01:47 +0100 Subject: [PATCH 147/238] fix [api] fix add_crawler_capture endpoint --- var/www/modules/restApi/Flask_restApi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index 90dba6d6..1b41bd4e 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -525,7 +525,7 @@ def add_crawler_task(): @restApi.route("api/v1/add/crawler/capture", methods=['POST']) @token_required('analyst') -def add_crawler_task(): +def add_crawler_capture(): data = request.get_json() user_token = get_auth_from_header() user_id = Users.get_token_user(user_token) From 1c52c187adcc56cebfd9877b56099bd518eac6a5 Mon Sep 17 00:00:00 2001 From: terrtia Date: Fri, 8 Dec 2023 10:37:58 +0100 Subject: [PATCH 148/238] fix: [api] fix add crawler capture return --- bin/lib/crawlers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index d256daee..101c8a33 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -1791,13 +1791,15 @@ def api_add_crawler_capture(data, user_id): if not capture_uuid: return {'error': 'Invalid capture_uuid', 'task_uuid': capture_uuid}, 400 + # parent = data.get('parent') + # TODO parent create_task(task['url'], depth=task['depth_limit'], har=task['har'], screenshot=task['screenshot'], proxy=task['proxy'], tags=task['tags'], parent='AIL_capture', task_uuid=task_uuid, external=True) create_capture(capture_uuid, task_uuid) - return capture_uuid, 200 + return {'uuid': capture_uuid}, 200 ################################################################################### ################################################################################### From 5b808ed416ce388dfa7dd5f5f10da57cf2e51286 Mon Sep 17 00:00:00 2001 From: terrtia Date: Fri, 8 Dec 2023 14:38:55 +0100 Subject: [PATCH 149/238] fix: [translate] fix exception --- bin/lib/Language.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bin/lib/Language.py b/bin/lib/Language.py index 7b688e2b..12ad8843 100755 --- a/bin/lib/Language.py +++ b/bin/lib/Language.py @@ -269,8 +269,8 @@ class LanguageTranslator: try: for dict_lang in self.lt.languages(): languages.append({'iso': dict_lang['code'], 'language': dict_lang['name']}) - except: - pass + except Exception as e: + print(e) return languages def detect_cld3(self, content): @@ -315,7 +315,8 @@ class LanguageTranslator: try: LIST_LANGUAGES = LanguageTranslator().languages() -except Exception: +except Exception as e: + print(e) LIST_LANGUAGES = [] def get_translation_languages(): From 73185f19fd8ac50e63f0aa3a1c63b0328d29a682 Mon Sep 17 00:00:00 2001 From: terrtia Date: Fri, 8 Dec 2023 15:40:05 +0100 Subject: [PATCH 150/238] chg: [categ] messages, bypass categ module + fix correlation --- bin/lib/correlations_engine.py | 2 +- bin/modules/Categ.py | 44 ++++++++++++++++++---------------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index 6a52caed..60479b1f 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -59,7 +59,7 @@ CORRELATION_TYPES_BY_OBJ = { "pgp": ["domain", "item", "message"], "screenshot": ["domain", "item"], "title": ["domain", "item"], - "user-account": ["chat", "chat-subchannel", "chat-thread", "message"], + "user-account": ["chat", "chat-subchannel", "chat-thread", "image", "message"], "username": ["domain", "item", "message"], # TODO chat-user/account } diff --git a/bin/modules/Categ.py b/bin/modules/Categ.py index 124f92bc..d5d2de82 100755 --- a/bin/modules/Categ.py +++ b/bin/modules/Categ.py @@ -6,14 +6,14 @@ The ZMQ_PubSub_Categ Module Each words files created under /files/ are representing categories. This modules take these files and compare them to -the content of an item. +the content of an obj. -When a word from a item match one or more of these words file, the filename of -the item / zhe item id is published/forwarded to the next modules. +When a word from a obj match one or more of these words file, the filename of +the obj / the obj id is published/forwarded to the next modules. Each category (each files) are representing a dynamic channel. This mean that if you create 1000 files under /files/ you'll have 1000 channels -where every time there is a matching word to a category, the item containing +where every time there is a matching word to a category, the obj containing this word will be pushed to this specific channel. ..note:: The channel will have the name of the file created. @@ -44,7 +44,6 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from modules.abstract_module import AbstractModule from lib.ConfigLoader import ConfigLoader -from lib.objects.Items import Item class Categ(AbstractModule): @@ -81,27 +80,32 @@ class Categ(AbstractModule): self.categ_words = tmp_dict.items() def compute(self, message, r_result=False): - # Create Item Object - item = self.get_obj() - # Get item content - content = item.get_content() + # Get obj Object + obj = self.get_obj() + # Get obj content + content = obj.get_content() categ_found = [] - # Search for pattern categories in item content + # Search for pattern categories in obj content for categ, pattern in self.categ_words: - found = set(re.findall(pattern, content)) - lenfound = len(found) - if lenfound >= self.matchingThreshold: - categ_found.append(categ) - msg = str(lenfound) + if obj.type == 'message': + self.add_message_to_queue(message='0', queue=categ) + else: - # Export message to categ queue - print(msg, categ) - self.add_message_to_queue(message=msg, queue=categ) + found = set(re.findall(pattern, content)) + lenfound = len(found) + if lenfound >= self.matchingThreshold: + categ_found.append(categ) + msg = str(lenfound) + + # Export message to categ queue + print(msg, categ) + self.add_message_to_queue(message=msg, queue=categ) + + self.redis_logger.debug( + f'Categ;{obj.get_source()};{obj.get_date()};{obj.get_basename()};Detected {lenfound} as {categ};{obj.get_id()}') - self.redis_logger.debug( - f'Categ;{item.get_source()};{item.get_date()};{item.get_basename()};Detected {lenfound} as {categ};{item.get_id()}') if r_result: return categ_found From 3add9b0a3377590bcc2cd29487ba90de83eeb3a6 Mon Sep 17 00:00:00 2001 From: terrtia Date: Fri, 8 Dec 2023 16:05:10 +0100 Subject: [PATCH 151/238] chg: [tags] search messages by tags --- var/www/blueprints/tags_ui.py | 11 +++++++++- var/www/templates/tags/menu_sidebar.html | 6 ++++++ .../templates/tags/search_obj_by_tags.html | 21 +++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/var/www/blueprints/tags_ui.py b/var/www/blueprints/tags_ui.py index 7199b98c..f833ab16 100644 --- a/var/www/blueprints/tags_ui.py +++ b/var/www/blueprints/tags_ui.py @@ -275,6 +275,15 @@ def tags_search_items(): dict_tagged['date'] = Date.sanitise_date_range('', '', separator='-') return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged) +@tags_ui.route('/tag/search/message') +@login_required +@login_read_only +def tags_search_messages(): + object_type = 'message' + dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s"} + dict_tagged['date'] = Date.sanitise_date_range('', '', separator='-') + return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged) + @tags_ui.route('/tag/search/domain') @login_required @login_read_only @@ -337,7 +346,7 @@ def get_obj_by_tags(): # TODO REPLACE ME dict_obj = Tag.get_obj_by_tags(object_type, list_tag, date_from=date_from, date_to=date_to, page=page) - print(dict_obj) + # print(dict_obj) if dict_obj['tagged_obj']: dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s", diff --git a/var/www/templates/tags/menu_sidebar.html b/var/www/templates/tags/menu_sidebar.html index f9da0d41..b4da0792 100644 --- a/var/www/templates/tags/menu_sidebar.html +++ b/var/www/templates/tags/menu_sidebar.html @@ -16,6 +16,12 @@ Search Items by Tags +
  • + + + Search Messages by Tags + +
  • diff --git a/var/www/templates/tags/search_obj_by_tags.html b/var/www/templates/tags/search_obj_by_tags.html index fd6e8e84..d60471bd 100644 --- a/var/www/templates/tags/search_obj_by_tags.html +++ b/var/www/templates/tags/search_obj_by_tags.html @@ -67,6 +67,9 @@ Date Item Action + {%elif dict_tagged["object_type"]=="message"%} + Date + Message {%endif%} @@ -123,6 +126,24 @@ {% endfor %} + {%elif dict_tagged["object_type"]=="message"%} + {% for dict_obj in dict_tagged["tagged_obj"] %} + + {{ dict_obj['full_date'] }} + + +
    {{ dict_obj['id'] }}
    +     +
    + {% for tag in dict_obj['tags'] %} + + {{ tag }} + + {% endfor %} +
    + + + {% endfor %} {%elif dict_tagged["object_type"]=="item"%} {% for dict_obj in dict_tagged["tagged_obj"] %} From f6db9fe888d3779e6cb2271ccb18aee4867b1a87 Mon Sep 17 00:00:00 2001 From: terrtia Date: Fri, 8 Dec 2023 16:31:56 +0100 Subject: [PATCH 152/238] fix: [sidebar] typo --- var/www/templates/sidebars/sidebar_objects.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/templates/sidebars/sidebar_objects.html b/var/www/templates/sidebars/sidebar_objects.html index d0a64df6..b6c2effb 100644 --- a/var/www/templates/sidebars/sidebar_objects.html +++ b/var/www/templates/sidebars/sidebar_objects.html @@ -109,7 +109,7 @@
  • - User-Acoount + User-Account
    • From 943a8731240343f3af8b1de8336e3b8ee05e6ad2 Mon Sep 17 00:00:00 2001 From: terrtia Date: Sat, 9 Dec 2023 16:50:43 +0100 Subject: [PATCH 153/238] chg: [tags] searech messages tags by daterange --- bin/lib/Tag.py | 13 +++++++++++-- var/www/templates/tags/block_obj_tags_search.html | 6 +++--- var/www/templates/tags/search_obj_by_tags.html | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index b9c86503..af193890 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -64,7 +64,7 @@ unsafe_tags = build_unsafe_tags() # get set_keys: intersection def get_obj_keys_by_tags(tags, obj_type, subtype='', date=None): l_set_keys = [] - if obj_type == 'item': + if obj_type == 'item' or obj_type == 'message': for tag in tags: l_set_keys.append(f'{obj_type}:{subtype}:{tag}:{date}') else: @@ -657,6 +657,15 @@ def add_object_tag(tag, obj_type, obj_id, subtype=''): domain = item_basic.get_item_domain(obj_id) add_object_tag(tag, "domain", domain) + update_tag_metadata(tag, date) + # MESSAGE + elif obj_type == 'message': + timestamp = obj_id.split('/')[1] + date = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y%m%d') + r_tags.sadd(f'{obj_type}:{subtype}:{tag}:{date}', obj_id) + + # TODO ADD CHAT TAGS ???? + update_tag_metadata(tag, date) else: r_tags.sadd(f'{obj_type}:{subtype}:{tag}', obj_id) @@ -729,7 +738,7 @@ def delete_object_tags(obj_type, subtype, obj_id): def get_obj_by_tags(obj_type, l_tags, date_from=None, date_to=None, nb_obj=50, page=1): # with daterange l_tagged_obj = [] - if obj_type=='item': + if obj_type=='item' or obj_type=='message': #sanityze date date_range = sanitise_tags_date_range(l_tags, date_from=date_from, date_to=date_to) l_dates = Date.substract_date(date_range['date_from'], date_range['date_to']) diff --git a/var/www/templates/tags/block_obj_tags_search.html b/var/www/templates/tags/block_obj_tags_search.html index c44cef33..f62246d8 100644 --- a/var/www/templates/tags/block_obj_tags_search.html +++ b/var/www/templates/tags/block_obj_tags_search.html @@ -4,7 +4,7 @@
    - {%if object_type=='item'%} + {%if object_type=='item' or object_type=='message'%}
    @@ -56,7 +56,7 @@ function searchTags() { var data = ltags.getValue(); var parameter = "?ltags=" + data + "&object_type={{ object_type }}{%if page%}&page={{ page }}{%endif%}"; - {%if object_type=='item'%} + {%if object_type=='item' or object_type=='message'%} var date_from = $('#date-range-from-input').val(); var date_to =$('#date-range-to-input').val(); parameter = parameter + "&date_from=" + date_from + "&date_to=" + date_to; @@ -68,7 +68,7 @@ } -{%if object_type=='item'%} +{%if object_type=='item' or object_type=='message'%} $('#date-range-from').dateRangePicker({ separator : ' to ', getValue: function(){ diff --git a/var/www/templates/tags/search_obj_by_tags.html b/var/www/templates/tags/search_obj_by_tags.html index d60471bd..a814dc06 100644 --- a/var/www/templates/tags/search_obj_by_tags.html +++ b/var/www/templates/tags/search_obj_by_tags.html @@ -201,7 +201,7 @@ {% with page=dict_tagged['page'], nb_page_max=dict_tagged['nb_pages'], nb_first_elem=dict_tagged['nb_first_elem'], nb_last_elem=dict_tagged['nb_last_elem'], nb_all_elem=dict_tagged['nb_all_elem'] %} {% set object_name= dict_tagged['object_name'] %} {% set target_url=url_for('tags_ui.get_obj_by_tags') + "?object_type=" + dict_tagged['object_type'] + "<ags=" + dict_tagged['current_tags_str'] %} - {%if dict_tagged["object_type"]=="item"%} + {%if dict_tagged["object_type"]=="item" or dict_tagged["object_type"]=="message"%} {% set target_url= target_url + "&date_from=" + dict_tagged['date']['date_from'] + "&date_to=" + dict_tagged['date']['date_to'] %} {%endif%} {% include 'pagination.html' %} From 759bb9a2f0c3d221f4fe4ba4720af578061cb259 Mon Sep 17 00:00:00 2001 From: terrtia Date: Sat, 9 Dec 2023 16:59:00 +0100 Subject: [PATCH 154/238] chg: [chats] add participants corrlation shortcut --- .../chats_explorer/chat_participants.html | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/var/www/templates/chats_explorer/chat_participants.html b/var/www/templates/chats_explorer/chat_participants.html index 455a39b2..3b44f291 100644 --- a/var/www/templates/chats_explorer/chat_participants.html +++ b/var/www/templates/chats_explorer/chat_participants.html @@ -98,6 +98,7 @@ First Seen Last Seen NB Messages + @@ -131,6 +132,20 @@ {% endif %} {{ user_meta['nb_messages'] }} + +
    +{#
    #} +{# #} +{# #} +{# #} +{#
    #} +
    + + + +
    +
    + {% endfor %} From 5fc9b1403fabbafc4cbc1659f12051d0aafb2e66 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 11 Dec 2023 00:46:15 +0100 Subject: [PATCH 155/238] chg: [chats] add pagination --- bin/lib/chats_viewer.py | 12 +-- bin/lib/objects/abstract_chat_object.py | 49 ++++++++--- var/www/blueprints/chats_explorer.py | 12 ++- .../chats_explorer/SubChannelMessages.html | 32 +++++-- .../chats_explorer/ThreadMessages.html | 20 +++++ .../chats_explorer/block_obj_time_search.html | 85 +++++++++++++++++++ .../chats_explorer/chat_participants.html | 2 +- .../templates/chats_explorer/chat_viewer.html | 20 +++++ .../templates/chats_explorer/pagination.html | 50 +++++++++++ 9 files changed, 256 insertions(+), 26 deletions(-) create mode 100644 var/www/templates/chats_explorer/block_obj_time_search.html create mode 100644 var/www/templates/chats_explorer/pagination.html diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index b083537a..e1809f47 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -331,7 +331,7 @@ def api_get_chat_service_instance(chat_instance_uuid): return {"status": "error", "reason": "Unknown uuid"}, 404 return chat_instance.get_meta({'chats'}), 200 -def api_get_chat(chat_id, chat_instance_uuid, translation_target=None): +def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, page=-1): chat = Chats.Chat(chat_id, chat_instance_uuid) if not chat.exists(): return {"status": "error", "reason": "Unknown chat"}, 404 @@ -341,7 +341,7 @@ def api_get_chat(chat_id, chat_instance_uuid, translation_target=None): if meta['subchannels']: meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels']) else: - meta['messages'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target) + meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page) return meta, 200 def api_get_nb_message_by_week(chat_id, chat_instance_uuid): @@ -367,7 +367,7 @@ def api_get_chat_participants(chat_type, chat_subtype, chat_id): meta['participants'] = chat_participants return meta, 200 -def api_get_subchannel(chat_id, chat_instance_uuid, translation_target=None): +def api_get_subchannel(chat_id, chat_instance_uuid, translation_target=None, nb=-1, page=-1): subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid) if not subchannel.exists(): return {"status": "error", "reason": "Unknown subchannel"}, 404 @@ -378,17 +378,17 @@ def api_get_subchannel(chat_id, chat_instance_uuid, translation_target=None): meta['threads'] = get_threads_metas(meta['threads']) if meta.get('username'): meta['username'] = get_username_meta_from_global_id(meta['username']) - meta['messages'], meta['tags_messages'] = subchannel.get_messages(translation_target=translation_target) + meta['messages'], meta['pagination'], meta['tags_messages'] = subchannel.get_messages(translation_target=translation_target, nb=nb, page=page) return meta, 200 -def api_get_thread(thread_id, thread_instance_uuid, translation_target=None): +def api_get_thread(thread_id, thread_instance_uuid, translation_target=None, nb=-1, page=-1): thread = ChatThreads.ChatThread(thread_id, thread_instance_uuid) if not thread.exists(): return {"status": "error", "reason": "Unknown thread"}, 404 meta = thread.get_meta({'chat', 'nb_messages', 'nb_participants'}) # if meta['chat']: # meta['chat'] = get_chat_meta_from_global_id(meta['chat']) - meta['messages'], meta['tags_messages'] = thread.get_messages(translation_target=translation_target) + meta['messages'], meta['pagination'], meta['tags_messages'] = thread.get_messages(translation_target=translation_target, nb=nb, page=page) return meta, 200 def api_get_message(message_id): diff --git a/bin/lib/objects/abstract_chat_object.py b/bin/lib/objects/abstract_chat_object.py index 5311cb85..1073008f 100755 --- a/bin/lib/objects/abstract_chat_object.py +++ b/bin/lib/objects/abstract_chat_object.py @@ -130,18 +130,36 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): def get_nb_messages(self): return r_object.zcard(f'messages:{self.type}:{self.subtype}:{self.id}') - def _get_messages(self, nb=-1, page=1): + def _get_messages(self, nb=-1, page=-1): if nb < 1: - return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True) + messages = r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True) + nb_pages = 0 + page = 1 + total = len(messages) + nb_first = 1 + nb_last = total else: + total = r_object.zcard(f'messages:{self.type}:{self.subtype}:{self.id}') + nb_pages = total / nb + if not nb_pages.is_integer(): + nb_pages = int(nb_pages) + 1 + else: + nb_pages = int(nb_pages) + if page > nb_pages or page < 1: + page = nb_pages + if page > 1: - start = page - 1 + nb + start = (page - 1) * nb else: start = 0 - messages = r_object.zrevrange(f'messages:{self.type}:{self.subtype}:{self.id}', start, start+nb-1, withscores=True) - if messages: - messages = reversed(messages) - return messages + messages = r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', start, start+nb-1, withscores=True) + # if messages: + # messages = reversed(messages) + nb_first = start+1 + nb_last = start+nb + if nb_last > total: + nb_last = total + return messages, {'nb': nb, 'page': page, 'nb_pages': nb_pages, 'total': total, 'nb_first': nb_first, 'nb_last': nb_last} def get_timestamp_first_message(self): return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, 0, withscores=True) @@ -184,12 +202,23 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): meta = message.get_meta(options={'content', 'files-names', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'}, timestamp=timestamp, translation_target=translation_target) return meta - def get_messages(self, start=0, page=1, nb=500, unread=False, translation_target='en'): # threads ???? # TODO ADD last/first message timestamp + return page + def get_messages(self, start=0, page=-1, nb=500, unread=False, translation_target='en'): # threads ???? # TODO ADD last/first message timestamp + return page # TODO return message meta tags = {} messages = {} curr_date = None - for message in self._get_messages(nb=2000, page=1): + try: + nb = int(nb) + except TypeError: + nb = 500 + if not page: + page = -1 + try: + page = int(page) + except TypeError: + page = 1 + mess, pagination = self._get_messages(nb=nb, page=page) + for message in mess: timestamp = message[1] date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d') if date_day != curr_date: @@ -203,7 +232,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC): if tag not in tags: tags[tag] = 0 tags[tag] += 1 - return messages, tags + return messages, pagination, tags # TODO REWRITE ADD OR ADD MESSAGE ???? # add diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index 54ae3d25..081df950 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -82,7 +82,9 @@ def chats_explorer_chat(): chat_id = request.args.get('id') instance_uuid = request.args.get('uuid') target = request.args.get('target') - chat = chats_viewer.api_get_chat(chat_id, instance_uuid, translation_target=target) + nb_messages = request.args.get('nb') + page = request.args.get('page') + chat = chats_viewer.api_get_chat(chat_id, instance_uuid, translation_target=target, nb=nb_messages, page=page) if chat[1] != 200: return create_json_response(chat[0], chat[1]) else: @@ -109,7 +111,9 @@ def objects_subchannel_messages(): subchannel_id = request.args.get('id') instance_uuid = request.args.get('uuid') target = request.args.get('target') - subchannel = chats_viewer.api_get_subchannel(subchannel_id, instance_uuid, translation_target=target) + nb_messages = request.args.get('nb') + page = request.args.get('page') + subchannel = chats_viewer.api_get_subchannel(subchannel_id, instance_uuid, translation_target=target, nb=nb_messages, page=page) if subchannel[1] != 200: return create_json_response(subchannel[0], subchannel[1]) else: @@ -124,7 +128,9 @@ def objects_thread_messages(): thread_id = request.args.get('id') instance_uuid = request.args.get('uuid') target = request.args.get('target') - thread = chats_viewer.api_get_thread(thread_id, instance_uuid, translation_target=target) + nb_messages = request.args.get('nb') + page = request.args.get('page') + thread = chats_viewer.api_get_thread(thread_id, instance_uuid, translation_target=target, nb=nb_messages, page=page) if thread[1] != 200: return create_json_response(thread[0], thread[1]) else: diff --git a/var/www/templates/chats_explorer/SubChannelMessages.html b/var/www/templates/chats_explorer/SubChannelMessages.html index 7ca7da41..5720acf1 100644 --- a/var/www/templates/chats_explorer/SubChannelMessages.html +++ b/var/www/templates/chats_explorer/SubChannelMessages.html @@ -126,12 +126,12 @@ - {% with obj_type='chat', obj_id=subchannel['id'], obj_subtype=subchannel['subtype'] %} - {% include 'modals/investigations_register_obj.html' %} - {% endwith %} - +{# {% with obj_type='chat', obj_id=subchannel['id'], obj_subtype=subchannel['subtype'] %}#} +{# {% include 'modals/investigations_register_obj.html' %}#} +{# {% endwith %}#} +{# #}
    @@ -191,6 +191,18 @@ {% with translate_url=url_for('chats_explorer.objects_subchannel_messages', uuid=subchannel['subtype']), obj_id=subchannel['id'] %} {% include 'chats_explorer/block_translation.html' %} {% endwith %} + {% with obj_subtype=subchannel['subtype'], obj_id=subchannel['id'], url_endpoint=url_for("chats_explorer.objects_subchannel_messages"), nb=subchannel['pagination']['nb'] %} + {% set date_from=subchannel['first_seen'] %} + {% set date_to=subchannel['last_seen'] %} + {% include 'block_obj_time_search.html' %} + {% endwith %} + {% with endpoint_url=url_for('chats_explorer.objects_subchannel_messages', uuid=subchannel['subtype']), pagination=subchannel['pagination'] %} + {% set endpoint_url = endpoint_url + "&id=" + subchannel['id'] + "&nb=" + subchannel['pagination']['nb'] | string %} + {% if translation_target %} + {% set endpoint_url = endpoint_url + "&target=" + translation_target %} + {% endif %} + {% include 'chats_explorer/pagination.html' %} + {% endwith %}
    @@ -216,6 +228,14 @@
    + {% with endpoint_url=url_for('chats_explorer.objects_subchannel_messages', uuid=subchannel['subtype']), pagination=subchannel['pagination'] %} + {% set endpoint_url = endpoint_url + "&id=" + subchannel['id'] + "&nb=" + subchannel['pagination']['nb'] | string %} + {% if translation_target %} + {% set endpoint_url = endpoint_url + "&target=" + translation_target %} + {% endif %} + {% include 'chats_explorer/pagination.html' %} + {% endwith %} +
    diff --git a/var/www/templates/chats_explorer/ThreadMessages.html b/var/www/templates/chats_explorer/ThreadMessages.html index 49d26ce9..ba5a02a5 100644 --- a/var/www/templates/chats_explorer/ThreadMessages.html +++ b/var/www/templates/chats_explorer/ThreadMessages.html @@ -141,6 +141,18 @@ {% with translate_url=url_for('chats_explorer.objects_thread_messages', uuid=meta['subtype']), obj_id=meta['id'] %} {% include 'chats_explorer/block_translation.html' %} {% endwith %} + {% with obj_subtype=meta['subtype'], obj_id=meta['id'], url_endpoint=url_for("chats_explorer.objects_thread_messages"), nb=meta['pagination']['nb'] %} + {% set date_from=meta['first_seen'] %} + {% set date_to=meta['last_seen'] %} + {% include 'block_obj_time_search.html' %} + {% endwith %} + {% with endpoint_url=url_for('chats_explorer.objects_thread_messages', uuid=meta['subtype']), pagination=meta['pagination'] %} + {% set endpoint_url = endpoint_url + "&id=" + meta['id'] + "&nb=" + meta['pagination']['nb'] | string %} + {% if translation_target %} + {% set endpoint_url = endpoint_url + "&target=" + translation_target %} + {% endif %} + {% include 'chats_explorer/pagination.html' %} + {% endwith %}
    @@ -166,6 +178,14 @@
    + {% with endpoint_url=url_for('chats_explorer.objects_thread_messages', uuid=meta['subtype']), pagination=meta['pagination'] %} + {% set endpoint_url = endpoint_url + "&id=" + meta['id'] + "&nb=" + meta['pagination']['nb'] | string %} + {% if translation_target %} + {% set endpoint_url = endpoint_url + "&target=" + translation_target %} + {% endif %} + {% include 'chats_explorer/pagination.html' %} + {% endwith %} +
    diff --git a/var/www/templates/chats_explorer/block_obj_time_search.html b/var/www/templates/chats_explorer/block_obj_time_search.html new file mode 100644 index 00000000..515a8ea7 --- /dev/null +++ b/var/www/templates/chats_explorer/block_obj_time_search.html @@ -0,0 +1,85 @@ +
    +
    +
    Filter by Time :
    + +{#
    #} +{#
    #} +{#
    #} +{#
    #} +{# #} +{#
    #} +{#
    #} +{#
    #} +{# #} +{#
    #} +{#
    #} +{#
    #} +{#
    #} +{#
    #} +{# #} +{#
    #} +{#
    #} +{#
    #} +{# #} +{#
    #} +{#
    #} +{#
    #} + +
    +
    +
    Numbers by page
    + +
    +
    + + + +
    +
    + + + + + diff --git a/var/www/templates/chats_explorer/chat_participants.html b/var/www/templates/chats_explorer/chat_participants.html index 3b44f291..b8e367b0 100644 --- a/var/www/templates/chats_explorer/chat_participants.html +++ b/var/www/templates/chats_explorer/chat_participants.html @@ -97,7 +97,7 @@ info First Seen Last Seen - NB Messages + diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html index 81f7e407..41cdd3cd 100644 --- a/var/www/templates/chats_explorer/chat_viewer.html +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -164,6 +164,18 @@ {% with translate_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), obj_id=chat['id'] %} {% include 'chats_explorer/block_translation.html' %} {% endwith %} + {% with obj_subtype=chat['subtype'], obj_id=chat['id'], url_endpoint=url_for("chats_explorer.chats_explorer_chat"), nb=chat['pagination']['nb'] %} + {% set date_from=chat['first_seen'] %} + {% set date_to=chat['last_seen'] %} + {% include 'block_obj_time_search.html' %} + {% endwith %} + {% with endpoint_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), pagination=chat['pagination'] %} + {% set endpoint_url = endpoint_url + "&id=" + chat['id'] + "&nb=" + chat['pagination']['nb'] | string %} + {% if translation_target %} + {% set endpoint_url = endpoint_url + "&target=" + translation_target %} + {% endif %} + {% include 'chats_explorer/pagination.html' %} + {% endwith %}
    @@ -189,6 +201,14 @@
    + {% with endpoint_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), pagination=chat['pagination'] %} + {% set endpoint_url = endpoint_url + "&id=" + chat['id'] + "&nb=" + chat['pagination']['nb'] | string %} + {% if translation_target %} + {% set endpoint_url = endpoint_url + "&target=" + translation_target %} + {% endif %} + {% include 'chats_explorer/pagination.html' %} + {% endwith %} + {% endif %}
    diff --git a/var/www/templates/chats_explorer/pagination.html b/var/www/templates/chats_explorer/pagination.html new file mode 100644 index 00000000..0bde498d --- /dev/null +++ b/var/www/templates/chats_explorer/pagination.html @@ -0,0 +1,50 @@ +
    +
    + +
    + + {%if pagination['total'] %} +
    + + results:  + {{ pagination['nb_first'] }}-{{ pagination['nb_last'] }} + / + {{ pagination['total'] }} + +
    +
    +
    +
    + {%endif%} +
    From 235539ea421f7acac78df9f732eb96903e9d9ab7 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 11 Dec 2023 09:30:09 +0100 Subject: [PATCH 156/238] fix: [crawler] fix capture start time --- bin/crawlers/Crawler.py | 13 +++++++++++-- bin/lib/crawlers.py | 3 +-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index 06ebe982..fd86da8a 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -121,7 +121,9 @@ class Crawler(AbstractModule): if crawlers.get_nb_crawler_captures() < crawlers.get_crawler_max_captures(): task_row = crawlers.add_task_to_lacus_queue() if task_row: - task_uuid, priority = task_row + task, priority = task_row + task.start() + task_uuid = task.uuid try: self.enqueue_capture(task_uuid, priority) except ConnectionError: @@ -195,10 +197,17 @@ class Crawler(AbstractModule): print(task.uuid, capture_uuid, 'launched') if self.ail_to_push_discovery: + if task.get_depth() == 1 and priority < 10 and task.get_domain().endswith('.onion'): har = task.get_har() screenshot = task.get_screenshot() - self.ail_to_push_discovery.add_crawler_capture(task_uuid, capture_uuid, url, har=har, + # parent_id = task.get_parent() + # if parent_id != 'manual' and parent_id != 'auto': + # parent = parent_id[19:-36] + # else: + # parent = 'AIL_capture' + + self.ail_to_push_discovery.add_crawler_capture(task_uuid, capture_uuid, url, har=har, # parent=parent, screenshot=screenshot, depth_limit=1, proxy='force_tor') print(task.uuid, capture_uuid, 'Added to ail_to_push_discovery') return capture_uuid diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 101c8a33..d6ec4f1e 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -1642,8 +1642,7 @@ def add_task_to_lacus_queue(): return None task_uuid, priority = task_uuid[0] task = CrawlerTask(task_uuid) - task.start() - return task.uuid, priority + return task, priority # PRIORITY: discovery = 0/10, feeder = 10, manual = 50, auto = 40, test = 100 def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None, From 2e38a966865e61099024b711b1d1111b88d2883e Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 11 Dec 2023 12:01:34 +0100 Subject: [PATCH 157/238] fix: [chat] fix translation pagination --- var/www/templates/chats_explorer/SubChannelMessages.html | 2 +- var/www/templates/chats_explorer/ThreadMessages.html | 2 +- var/www/templates/chats_explorer/block_translation.html | 2 +- var/www/templates/chats_explorer/chat_viewer.html | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/var/www/templates/chats_explorer/SubChannelMessages.html b/var/www/templates/chats_explorer/SubChannelMessages.html index 5720acf1..858a1159 100644 --- a/var/www/templates/chats_explorer/SubChannelMessages.html +++ b/var/www/templates/chats_explorer/SubChannelMessages.html @@ -188,7 +188,7 @@ {% include 'objects/image/block_blur_img_slider.html' %} - {% with translate_url=url_for('chats_explorer.objects_subchannel_messages', uuid=subchannel['subtype']), obj_id=subchannel['id'] %} + {% with translate_url=url_for('chats_explorer.objects_subchannel_messages', uuid=subchannel['subtype']), obj_id=subchannel['id'], pagination=subchannel['pagination'] %} {% include 'chats_explorer/block_translation.html' %} {% endwith %} {% with obj_subtype=subchannel['subtype'], obj_id=subchannel['id'], url_endpoint=url_for("chats_explorer.objects_subchannel_messages"), nb=subchannel['pagination']['nb'] %} diff --git a/var/www/templates/chats_explorer/ThreadMessages.html b/var/www/templates/chats_explorer/ThreadMessages.html index ba5a02a5..8fdd27f6 100644 --- a/var/www/templates/chats_explorer/ThreadMessages.html +++ b/var/www/templates/chats_explorer/ThreadMessages.html @@ -138,7 +138,7 @@ {% include 'objects/image/block_blur_img_slider.html' %} - {% with translate_url=url_for('chats_explorer.objects_thread_messages', uuid=meta['subtype']), obj_id=meta['id'] %} + {% with translate_url=url_for('chats_explorer.objects_thread_messages', uuid=meta['subtype']), obj_id=meta['id'], pagination=meta['pagination'] %} {% include 'chats_explorer/block_translation.html' %} {% endwith %} {% with obj_subtype=meta['subtype'], obj_id=meta['id'], url_endpoint=url_for("chats_explorer.objects_thread_messages"), nb=meta['pagination']['nb'] %} diff --git a/var/www/templates/chats_explorer/block_translation.html b/var/www/templates/chats_explorer/block_translation.html index 84749798..63d47b7c 100644 --- a/var/www/templates/chats_explorer/block_translation.html +++ b/var/www/templates/chats_explorer/block_translation.html @@ -32,6 +32,6 @@ function translate_selector(){ var t = document.getElementById("translation_selector_target"); var target = t.value - window.location.replace("{{ translate_url }}&id={{ obj_id }}&target=" + target); + window.location.replace("{{ translate_url }}&id={{ obj_id }}{% if pagination %}&nb={{ pagination['nb'] }}&page={{ pagination['page'] }}{% endif %}&target=" + target); } \ No newline at end of file diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html index 41cdd3cd..64f14295 100644 --- a/var/www/templates/chats_explorer/chat_viewer.html +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -161,7 +161,7 @@ {% include 'objects/image/block_blur_img_slider.html' %} - {% with translate_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), obj_id=chat['id'] %} + {% with translate_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), obj_id=chat['id'], pagination=chat['pagination'] %} {% include 'chats_explorer/block_translation.html' %} {% endwith %} {% with obj_subtype=chat['subtype'], obj_id=chat['id'], url_endpoint=url_for("chats_explorer.chats_explorer_chat"), nb=chat['pagination']['nb'] %} From 4529a76d13efcfae087dd0a8b9d9d7eec69a8918 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 12 Dec 2023 10:14:59 +0100 Subject: [PATCH 158/238] fix: [zmq importer] fix object source name --- bin/importer/ZMQImporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/importer/ZMQImporter.py b/bin/importer/ZMQImporter.py index 509b136e..18b57fa3 100755 --- a/bin/importer/ZMQImporter.py +++ b/bin/importer/ZMQImporter.py @@ -76,7 +76,7 @@ class ZMQModuleImporter(AbstractModule): obj_id, gzip64encoded = message.split(' ', 1) # TODO ADD LOGS splitted = obj_id.split('>>', 1) - if splitted == 2: + if len(splitted) == 2: feeder_name, obj_id = splitted else: feeder_name = self.default_feeder_name From c20c41c50ff992ed05bb2388c3372468861f7565 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 12 Dec 2023 10:30:40 +0100 Subject: [PATCH 159/238] fix: [libinjection] memory leak, disable module --- bin/LAUNCH.sh | 4 ++-- bin/modules/LibInjection.py | 3 --- configs/modules.cfg | 7 ++++--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 1013e546..68c20ac7 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -265,8 +265,8 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./SQLInjectionDetection.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "LibInjection" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./LibInjection.py; read x" - sleep 0.1 +# screen -S "Script_AIL" -X screen -t "LibInjection" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./LibInjection.py; read x" +# sleep 0.1 # screen -S "Script_AIL" -X screen -t "Pasties" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Pasties.py; read x" # sleep 0.1 diff --git a/bin/modules/LibInjection.py b/bin/modules/LibInjection.py index eb1174f6..de4d1287 100755 --- a/bin/modules/LibInjection.py +++ b/bin/modules/LibInjection.py @@ -25,9 +25,6 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from modules.abstract_module import AbstractModule -from lib.ConfigLoader import ConfigLoader -from lib.objects.Items import Item -# from lib import Statistics class LibInjection(AbstractModule): """docstring for LibInjection module.""" diff --git a/configs/modules.cfg b/configs/modules.cfg index 41006974..848fe0b0 100644 --- a/configs/modules.cfg +++ b/configs/modules.cfg @@ -99,9 +99,10 @@ publish = Tags subscribe = Urls publish = Url -[LibInjection] -subscribe = Url -publish = Tags +# disabled +#[LibInjection] +#subscribe = Url +#publish = Tags [SQLInjectionDetection] subscribe = Url From e51ee7ab5578f37a16c286f4c9b33d5a58397517 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 12 Dec 2023 10:35:33 +0100 Subject: [PATCH 160/238] chg: [crawlers] add endpoints to reset captures --- var/www/blueprints/crawler_splash.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index 1b7f4454..6fa53577 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -83,6 +83,13 @@ def crawler_dashboard_json(): return jsonify({'crawlers_status': crawlers_status, 'stats': crawlers_latest_stats}) +@crawler_splash.route("/crawlers/dashboard/captures/delete", methods=['GET']) +@login_required +@login_admin +def crawlers_dashboard(): + crawlers.delete_captures() + return redirect(url_for('crawler_splash.crawlers_dashboard')) + @crawler_splash.route("/crawlers/manual", methods=['GET']) @login_required From d376612d511e0d942ebc37fddcc83cf2a3bb8c52 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 12 Dec 2023 10:36:42 +0100 Subject: [PATCH 161/238] chg: [crawlers] add endpoints to reset captures --- var/www/blueprints/crawler_splash.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index 6fa53577..c8983ace 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -86,7 +86,7 @@ def crawler_dashboard_json(): @crawler_splash.route("/crawlers/dashboard/captures/delete", methods=['GET']) @login_required @login_admin -def crawlers_dashboard(): +def crawlers_dashboard_captures_delete(): crawlers.delete_captures() return redirect(url_for('crawler_splash.crawlers_dashboard')) From 847d004c1379066f3713fe048a1437dd9463990f Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 12 Dec 2023 11:05:23 +0100 Subject: [PATCH 162/238] fix: [crawler] debug --- bin/crawlers/Crawler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index fd86da8a..417dab4f 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -219,6 +219,8 @@ class Crawler(AbstractModule): task = capture.get_task() domain = task.get_domain() print(domain) + if not domain: + raise Exception(f'Error: domain {domain}') self.domain = Domain(domain) self.original_domain = Domain(domain) From cdfc9f64e530860afd393339230472d5fbd3ad78 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 12 Dec 2023 11:13:58 +0100 Subject: [PATCH 163/238] fix: [crawler] debug --- bin/crawlers/Crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index 417dab4f..0fe9de2c 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -209,7 +209,7 @@ class Crawler(AbstractModule): self.ail_to_push_discovery.add_crawler_capture(task_uuid, capture_uuid, url, har=har, # parent=parent, screenshot=screenshot, depth_limit=1, proxy='force_tor') - print(task.uuid, capture_uuid, 'Added to ail_to_push_discovery') + print(task.uuid, capture_uuid, url, 'Added to ail_to_push_discovery') return capture_uuid # CRAWL DOMAIN From 7e9ea48c8183d33949784dcc6afdf794016c1600 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 12 Dec 2023 11:20:21 +0100 Subject: [PATCH 164/238] fix: [crawler] debug --- bin/crawlers/Crawler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index 0fe9de2c..0d6cb67c 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -207,6 +207,9 @@ class Crawler(AbstractModule): # else: # parent = 'AIL_capture' + if not url: + raise Exception(f'Error: url is None, {task.uuid}, {capture_uuid}, {url}') + self.ail_to_push_discovery.add_crawler_capture(task_uuid, capture_uuid, url, har=har, # parent=parent, screenshot=screenshot, depth_limit=1, proxy='force_tor') print(task.uuid, capture_uuid, url, 'Added to ail_to_push_discovery') From 9221e532c4b5b595b8733366b1dae93157aa7f6f Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 12 Dec 2023 11:32:33 +0100 Subject: [PATCH 165/238] fix: [crawlers] fix task start --- bin/lib/crawlers.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index d6ec4f1e..13c8f75f 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -1793,11 +1793,12 @@ def api_add_crawler_capture(data, user_id): # parent = data.get('parent') # TODO parent - create_task(task['url'], depth=task['depth_limit'], har=task['har'], screenshot=task['screenshot'], - proxy=task['proxy'], tags=task['tags'], - parent='AIL_capture', task_uuid=task_uuid, external=True) - + task_uuid = create_task(task['url'], depth=task['depth_limit'], har=task['har'], screenshot=task['screenshot'], + proxy=task['proxy'], tags=task['tags'], + parent='manual', task_uuid=task_uuid, external=True) + task = CrawlerTask(task_uuid) create_capture(capture_uuid, task_uuid) + task.start() return {'uuid': capture_uuid}, 200 ################################################################################### From 70bb6757f8c6e587752983572b2bb5602e65a328 Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 13 Dec 2023 11:51:53 +0100 Subject: [PATCH 166/238] chg: [correlation] UI chats filters + correation user-account/username --- bin/lib/correlations_engine.py | 4 +-- var/www/blueprints/correlation.py | 19 ++++++++-- .../correlation/show_correlation.html | 36 ++++++++++++++++--- 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index 60479b1f..fe37e7fe 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -59,8 +59,8 @@ CORRELATION_TYPES_BY_OBJ = { "pgp": ["domain", "item", "message"], "screenshot": ["domain", "item"], "title": ["domain", "item"], - "user-account": ["chat", "chat-subchannel", "chat-thread", "image", "message"], - "username": ["domain", "item", "message"], # TODO chat-user/account + "user-account": ["chat", "chat-subchannel", "chat-thread", "image", "message", "username"], + "username": ["domain", "item", "message", "user-account"], } def get_obj_correl_types(obj_type): diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index d5d672b1..4cf9ce53 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -121,9 +121,24 @@ def show_correlation(): correl_option = request.form.get('ItemCheck') if correl_option: filter_types.append('item') - correl_option = request.form.get('TitleCheck') + correl_option = request.form.get('chatCheck') if correl_option: - filter_types.append('title') + filter_types.append('chat') + correl_option = request.form.get('subchannelCheck') + if correl_option: + filter_types.append('chat-subchannel') + correl_option = request.form.get('threadCheck') + if correl_option: + filter_types.append('chat-thread') + correl_option = request.form.get('messageCheck') + if correl_option: + filter_types.append('message') + correl_option = request.form.get('imageCheck') + if correl_option: + filter_types.append('image') + correl_option = request.form.get('user_accountCheck') + if correl_option: + filter_types.append('user-account') # list as params filter_types = ",".join(filter_types) diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index ebcd84e3..cda58f1c 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -248,10 +248,6 @@
    -
    - - -
    @@ -261,6 +257,38 @@
    +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    + + +
    + +
    +
    + + +
    +
    + + +
    + {#
  • #} {#
    #} From 38157f8a017f57cff2d493bc6f498e5589b2822b Mon Sep 17 00:00:00 2001 From: Markus Lassfolk Date: Mon, 1 Jan 2024 14:10:42 +0100 Subject: [PATCH 167/238] Typo in CRAWLED_SCREENSHOT There was a typo in the CRAWLED_SCREENSHOT --- reset_AIL.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reset_AIL.sh b/reset_AIL.sh index 6fae95d3..29e5ba13 100755 --- a/reset_AIL.sh +++ b/reset_AIL.sh @@ -67,8 +67,8 @@ function reset_dir { popd fi - if [ -d CRAWLED_SCREESHOT/ ]; then - pushd CRAWLED_SCREESHOT/ + if [ -d CRAWLED_SCREENSHOT/ ]; then + pushd CRAWLED_SCREENSHOT/ rm -r * echo 'cleaned CRAWLED_SCREENSHOT' popd From 81f9a860d4cbc3027b63d7921e4c142b3a88218d Mon Sep 17 00:00:00 2001 From: Markus Lassfolk Date: Mon, 1 Jan 2024 21:54:29 +0100 Subject: [PATCH 168/238] Fix IndexError in get_last_tag_from_remote function This commit adds a check to ensure that the output from the subprocess command in the get_last_tag_from_remote function has a sufficient number of lines before attempting to access specific indices. This change prevents the IndexError that occurred when the git command's output was shorter than expected. --- bin/packages/git_status.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/bin/packages/git_status.py b/bin/packages/git_status.py index 7b9c467c..19bf7769 100755 --- a/bin/packages/git_status.py +++ b/bin/packages/git_status.py @@ -129,17 +129,25 @@ def get_last_tag_from_local(verbose=False): print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) return '' -# Get last local tag +# Get last remote tag def get_last_tag_from_remote(verbose=False): if verbose: print('retrieving last remote tag ...') #print('git ls-remote --tags') + process = subprocess.run(['git', 'ls-remote', '--tags'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if process.returncode == 0: - res = process.stdout.split(b'\n')[-2].split(b'/')[-1].replace(b'^{}', b'').decode() - if verbose: - print(res) - return res + output_lines = process.stdout.split(b'\n') + if len(output_lines) > 1: + # Assuming we want the second-to-last line as before + res = output_lines[-2].split(b'/')[-1].replace(b'^{}', b'').decode() + if verbose: + print(res) + return res + else: + if verbose: + print("No tags found or insufficient output from git command.") + return '' else: if verbose: From c05f4d7833912eee005e616bb39d961b6d017e3c Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 2 Jan 2024 17:15:45 +0100 Subject: [PATCH 169/238] chg: [chats] get user message ids by chat --- bin/importer/feeders/abstract_chats_feeder.py | 4 +++- bin/lib/correlations_engine.py | 3 +++ bin/lib/objects/UsersAccount.py | 12 ++++++++---- bin/lib/objects/abstract_object.py | 8 +++++++- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py index a966e951..1dadb970 100755 --- a/bin/importer/feeders/abstract_chats_feeder.py +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -318,7 +318,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC): message.create('') objs.add(message) - if message.exists(): + if message.exists(): # TODO Correlation user-account image/filename ???? obj = Images.create(self.get_message_content()) obj.add(date, message) obj.set_parent(obj_global_id=message.get_global_id()) @@ -336,6 +336,8 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # TODO get created subchannel + thread # => create correlation user-account with object + print(obj.id) + # CHAT chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id) diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index fe37e7fe..378687bf 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -123,6 +123,9 @@ def is_obj_correlated(obj_type, subtype, obj_id, obj2_type, subtype2, obj2_id): except: return False +def get_obj_inter_correlation(obj_type1, subtype1, obj_id1, obj_type2, subtype2, obj_id2, correl_type): + return r_metadata.sinter(f'correlation:obj:{obj_type1}:{subtype1}:{correl_type}:{obj_id1}', f'correlation:obj:{obj_type2}:{subtype2}:{correl_type}:{obj_id2}') + def add_obj_correlation(obj1_type, subtype1, obj1_id, obj2_type, subtype2, obj2_id): if subtype1 is None: subtype1 = '' diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py index 92076f24..2148697a 100755 --- a/bin/lib/objects/UsersAccount.py +++ b/bin/lib/objects/UsersAccount.py @@ -119,6 +119,9 @@ class UserAccount(AbstractSubtypeObject): def update_username_timeline(self, username_global_id, timestamp): self._get_timeline_username().add_timestamp(timestamp, username_global_id) + def get_messages_by_chat_obj(self, chat_obj): + return self.get_correlation_iter_obj(chat_obj, 'message') + def get_meta(self, options=set()): # TODO Username timeline meta = self._get_meta(options=options) meta['id'] = self.id @@ -191,7 +194,8 @@ def get_all_by_subtype(subtype): return get_all_id('user-account', subtype) -# if __name__ == '__main__': -# name_to_search = 'co' -# subtype = 'telegram' -# print(search_usernames_by_name(name_to_search, subtype)) +if __name__ == '__main__': + from lib.objects import Chats + chat = Chats.Chat('', '00098785-7e70-5d12-a120-c5cdc1252b2b') + account = UserAccount('', '00098785-7e70-5d12-a120-c5cdc1252b2b') + print(account.get_messages_by_chat_obj(chat)) diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index cac9d58c..d651761f 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -22,7 +22,7 @@ from lib import ail_logger from lib import Tag from lib.ConfigLoader import ConfigLoader from lib import Duplicate -from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type +from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers @@ -270,6 +270,12 @@ class AbstractObject(ABC): return is_obj_correlated(self.type, self.subtype, self.id, object2.get_type(), object2.get_subtype(r_str=True), object2.get_id()) + def get_correlation_iter(self, obj_type2, subtype2, obj_id2, correl_type): + return get_obj_inter_correlation(self.type, self.get_subtype(r_str=True), self.id, obj_type2, subtype2, obj_id2, correl_type) + + def get_correlation_iter_obj(self, object2, correl_type): + return self.get_correlation_iter(object2.get_type(), object2.get_subtype(r_str=True), object2.get_id(), correl_type) + def delete_correlation(self, type2, subtype2, id2): """ Get object correlations From b23683d98ba581b9ee35042790aee96e861ccfd9 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 2 Jan 2024 18:14:27 +0100 Subject: [PATCH 170/238] chg: [thirdparties] remove sb-admin + debug #194 --- install_virtualenv.sh | 2 +- var/www/modules/search/templates/search.html | 3 +- .../sentiment_analysis_plot_tool.html | 3 +- .../sentiment_analysis_trending.html | 3 +- .../trendingcharts/templates/Trending.html | 3 +- .../templates/Moduletrending.html | 3 +- var/www/templates/base_template.html | 10 +-- .../templates/objects/item/show_item_min.html | 2 +- var/www/update_thirdparty.sh | 62 ++++++++----------- 9 files changed, 35 insertions(+), 56 deletions(-) diff --git a/install_virtualenv.sh b/install_virtualenv.sh index 214b613c..5909ed75 100755 --- a/install_virtualenv.sh +++ b/install_virtualenv.sh @@ -3,7 +3,7 @@ # halt on errors set -e -## bash debug mode togle below +## bash debug mode toggle below #set -x if [ -z "$VIRTUAL_ENV" ]; then diff --git a/var/www/modules/search/templates/search.html b/var/www/modules/search/templates/search.html index 7d599692..493601cc 100644 --- a/var/www/modules/search/templates/search.html +++ b/var/www/modules/search/templates/search.html @@ -11,7 +11,6 @@ - @@ -126,7 +125,7 @@
    • - +
    diff --git a/var/www/modules/sentiment/templates/sentiment_analysis_plot_tool.html b/var/www/modules/sentiment/templates/sentiment_analysis_plot_tool.html index 4b3eb4fd..78f32976 100644 --- a/var/www/modules/sentiment/templates/sentiment_analysis_plot_tool.html +++ b/var/www/modules/sentiment/templates/sentiment_analysis_plot_tool.html @@ -11,11 +11,10 @@ - - + diff --git a/var/www/modules/sentiment/templates/sentiment_analysis_trending.html b/var/www/modules/sentiment/templates/sentiment_analysis_trending.html index 4c1bcb8f..13a6c14b 100644 --- a/var/www/modules/sentiment/templates/sentiment_analysis_trending.html +++ b/var/www/modules/sentiment/templates/sentiment_analysis_trending.html @@ -11,9 +11,8 @@ - - + diff --git a/var/www/modules/trendingcharts/templates/Trending.html b/var/www/modules/trendingcharts/templates/Trending.html index f66b50c0..9e3728ca 100644 --- a/var/www/modules/trendingcharts/templates/Trending.html +++ b/var/www/modules/trendingcharts/templates/Trending.html @@ -15,7 +15,6 @@ - @@ -149,7 +148,7 @@ var chart_2_num_day = 15;
    - + diff --git a/var/www/modules/trendingmodules/templates/Moduletrending.html b/var/www/modules/trendingmodules/templates/Moduletrending.html index 5c24110a..7e4a3085 100644 --- a/var/www/modules/trendingmodules/templates/Moduletrending.html +++ b/var/www/modules/trendingmodules/templates/Moduletrending.html @@ -11,7 +11,6 @@ - @@ -68,7 +67,7 @@
    - + diff --git a/var/www/templates/base_template.html b/var/www/templates/base_template.html index 7043cf59..46d6fb1e 100644 --- a/var/www/templates/base_template.html +++ b/var/www/templates/base_template.html @@ -8,16 +8,11 @@ Analysis Information Leak framework Dashboard - + - - - + - - - @@ -42,7 +37,6 @@ $("#"+activePage).addClass("active"); }); - diff --git a/var/www/templates/objects/item/show_item_min.html b/var/www/templates/objects/item/show_item_min.html index 359627df..50c2e619 100644 --- a/var/www/templates/objects/item/show_item_min.html +++ b/var/www/templates/objects/item/show_item_min.html @@ -11,7 +11,7 @@ - + diff --git a/var/www/update_thirdparty.sh b/var/www/update_thirdparty.sh index 4033bc79..9b5c95e0 100755 --- a/var/www/update_thirdparty.sh +++ b/var/www/update_thirdparty.sh @@ -1,13 +1,12 @@ #!/bin/bash -set -e +# set -e # submodules git submodule update wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js -SBADMIN_VERSION='3.3.7' BOOTSTRAP_VERSION='4.2.1' FONT_AWESOME_VERSION='5.7.1' D3_JS_VERSION='5.5.0' @@ -15,20 +14,16 @@ D3_JS_VERSION='5.5.0' rm -rf temp mkdir temp -wget -q https://github.com/twbs/bootstrap/releases/download/v${BOOTSTRAP_VERSION}/bootstrap-${BOOTSTRAP_VERSION}-dist.zip -O temp/bootstrap${BOOTSTRAP_VERSION}.zip -wget -q https://github.com/BlackrockDigital/startbootstrap-sb-admin/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}.zip -wget -q https://github.com/BlackrockDigital/startbootstrap-sb-admin-2/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}-2.zip -wget -q https://github.com/FortAwesome/Font-Awesome/archive/v4.7.0.zip -O temp/FONT_AWESOME_4.7.0.zip -wget -q https://github.com/FortAwesome/Font-Awesome/archive/5.7.1.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -wget -q https://github.com/d3/d3/releases/download/v${D3_JS_VERSION}/d3.zip -O temp/d3_${D3_JS_VERSION}.zip +wget https://github.com/twbs/bootstrap/releases/download/v${BOOTSTRAP_VERSION}/bootstrap-${BOOTSTRAP_VERSION}-dist.zip -O temp/bootstrap${BOOTSTRAP_VERSION}.zip +wget https://github.com/FortAwesome/Font-Awesome/archive/v4.7.0.zip -O temp/FONT_AWESOME_4.7.0.zip +wget https://github.com/FortAwesome/Font-Awesome/archive/5.7.1.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip +wget https://github.com/d3/d3/releases/download/v${D3_JS_VERSION}/d3.zip -O temp/d3_${D3_JS_VERSION}.zip # dateRangePicker -wget -q https://github.com/moment/moment/archive/2.24.0.zip -O temp/moment.zip -wget -q https://github.com/longbill/jquery-date-range-picker/archive/v0.20.0.zip -O temp/daterangepicker.zip +wget https://github.com/moment/moment/archive/2.24.0.zip -O temp/moment.zip +wget https://github.com/longbill/jquery-date-range-picker/archive/v0.20.0.zip -O temp/daterangepicker.zip unzip -qq temp/bootstrap${BOOTSTRAP_VERSION}.zip -d temp/ -unzip -qq temp/${SBADMIN_VERSION}.zip -d temp/ -unzip -qq temp/${SBADMIN_VERSION}-2.zip -d temp/ unzip -qq temp/FONT_AWESOME_4.7.0.zip -d temp/ unzip -qq temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/ unzip -qq temp/d3_${D3_JS_VERSION}.zip -d temp/ @@ -41,8 +36,6 @@ mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/js/bootstrap.min.js.map ./static/js/ mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css ./static/css/bootstrap4.min.css mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css.map ./static/css/bootstrap4.min.css.map -mv temp/startbootstrap-sb-admin-${SBADMIN_VERSION} temp/sb-admin -mv temp/startbootstrap-sb-admin-2-${SBADMIN_VERSION} temp/sb-admin-2 mv temp/Font-Awesome-4.7.0 temp/font-awesome rm -rf ./static/webfonts/ @@ -50,15 +43,11 @@ mv temp/Font-Awesome-${FONT_AWESOME_VERSION}/css/all.min.css ./static/css/font-a mv temp/Font-Awesome-${FONT_AWESOME_VERSION}/webfonts ./static/webfonts rm -rf ./static/js/plugins -mv temp/sb-admin/js/* ./static/js/ rm -rf ./static/fonts/ ./static/font-awesome/ -mv temp/sb-admin/fonts/ ./static/ mv temp/font-awesome/ ./static/ rm -rf ./static/css/plugins/ -mv temp/sb-admin/css/* ./static/css/ -mv temp/sb-admin-2/dist/css/* ./static/css/ mv temp/jquery-date-range-picker-0.20.0/dist/daterangepicker.min.css ./static/css/ mv temp/d3.min.js ./static/js/ @@ -66,34 +55,35 @@ mv temp/moment-2.24.0/min/moment.min.js ./static/js/ mv temp/jquery-date-range-picker-0.20.0/dist/jquery.daterangepicker.min.js ./static/js/ JQVERSION="3.4.1" -wget -q http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js +wget http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js #Ressources for dataTable -wget -q https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js -O ./static/js/jquery.dataTables.min.js -wget -q https://cdn.datatables.net/plug-ins/1.10.20/integration/bootstrap/3/dataTables.bootstrap.css -O ./static/css/dataTables.bootstrap.css -wget -q https://cdn.datatables.net/plug-ins/1.10.20/integration/bootstrap/3/dataTables.bootstrap.js -O ./static/js/dataTables.bootstrap.js +wget https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js -O ./static/js/jquery.dataTables.min.js +wget https://cdn.datatables.net/plug-ins/1.10.20/integration/bootstrap/3/dataTables.bootstrap.css -O ./static/css/dataTables.bootstrap.css +wget https://cdn.datatables.net/plug-ins/1.10.20/integration/bootstrap/3/dataTables.bootstrap.js -O ./static/js/dataTables.bootstrap.js -wget -q https://cdn.datatables.net/1.10.20/css/dataTables.bootstrap4.min.css -O ./static/css/dataTables.bootstrap.min.css -wget -q https://cdn.datatables.net/1.10.20/js/dataTables.bootstrap4.min.js -O ./static/js/dataTables.bootstrap.min.js +wget https://cdn.datatables.net/1.10.20/css/dataTables.bootstrap4.min.css -O ./static/css/dataTables.bootstrap.min.css +wget https://cdn.datatables.net/1.10.20/js/dataTables.bootstrap4.min.js -O ./static/js/dataTables.bootstrap.min.js #Ressources for bootstrap popover POPPER_VERSION="1.16.1" -wget -q https://github.com/FezVrasta/popper.js/archive/v${POPPER_VERSION}.zip -O temp/popper.zip +wget https://github.com/FezVrasta/popper.js/archive/v${POPPER_VERSION}.zip -O temp/popper.zip unzip -qq temp/popper.zip -d temp/ mv temp/floating-ui-${POPPER_VERSION}/dist/umd/popper.min.js ./static/js/ mv temp/floating-ui-${POPPER_VERSION}/dist/umd/popper.min.js.map ./static/js/ #Ressource for graph -wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.js -O ./static/js/jquery.flot.js -wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.pie.js -O ./static/js/jquery.flot.pie.js -wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.time.js -O ./static/js/jquery.flot.time.js -wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.stack.js -O ./static/js/jquery.flot.stack.js +# DASHBOARD # TODO REFACTOR DASHBOARD GRAPHS +wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.js -O ./static/js/jquery.flot.js +wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.pie.js -O ./static/js/jquery.flot.pie.js +wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.time.js -O ./static/js/jquery.flot.time.js +wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.stack.js -O ./static/js/jquery.flot.stack.js #Ressources for sparkline and canvasJS and slider -wget -q http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js -wget -q https://canvasjs.com/assets/script/canvasjs.min.js -O ./static/js/jquery.canvasjs.min.js +#wget http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js +#wget https://canvasjs.com/assets/script/canvasjs.min.js -O ./static/js/jquery.canvasjs.min.js -wget -q https://jqueryui.com/resources/download/jquery-ui-1.12.1.zip -O temp/jquery-ui.zip +wget https://jqueryui.com/resources/download/jquery-ui-1.12.1.zip -O temp/jquery-ui.zip unzip -qq temp/jquery-ui.zip -d temp/ mv temp/jquery-ui-1.12.1/jquery-ui.min.js ./static/js/jquery-ui.min.js mv temp/jquery-ui-1.12.1/jquery-ui.min.css ./static/css/jquery-ui.min.css @@ -126,11 +116,11 @@ then fi #Update MISP Taxonomies and Galaxies -pip3 install git+https://github.com/MISP/PyTaxonomies --upgrade -pip3 install git+https://github.com/MISP/PyMISPGalaxies --upgrade +pip3 install git+https://github.com/MISP/PyTaxonomies --upgrade # TODO move to requirement +pip3 install git+https://github.com/MISP/PyMISPGalaxies --upgrade # TODO move to requirement #Update PyMISP -pip3 install git+https://github.com/MISP/PyMISP --upgrade +pip3 install pymisp --upgrade #Update the Hive -pip3 install thehive4py --upgrade +pip3 install thehive4py --upgrade # TODO move to requirement From 6c77f4219cc607b0759429a723e1162da9d6c838 Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Sat, 6 Jan 2024 23:38:35 +0900 Subject: [PATCH 171/238] fix: [README] api.md link typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b5818fc..617e0d48 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ CIRCL organises training on how to use or extend the AIL framework. AIL training ## API -The API documentation is available in [doc/README.md](doc/README.md) +The API documentation is available in [doc/api.md](doc/api.md) ## HOWTO From 0af5ea9d488687c1fc70ee5ddd06a4c7cb54839b Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 8 Jan 2024 13:50:56 +0100 Subject: [PATCH 172/238] chg: [queues] timeout obj after 2 days --- bin/core/Sync_module.py | 10 +++++++--- bin/lib/ail_core.py | 3 +++ bin/lib/ail_queues.py | 25 +++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/bin/core/Sync_module.py b/bin/core/Sync_module.py index c6bdfeeb..065be8f3 100755 --- a/bin/core/Sync_module.py +++ b/bin/core/Sync_module.py @@ -22,18 +22,18 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from core import ail_2_ail -from lib.ail_queues import get_processed_end_obj +from lib.ail_queues import get_processed_end_obj, timeout_processed_objs from lib.exceptions import ModuleQueueError from lib.objects import ail_objects from modules.abstract_module import AbstractModule -class Sync_module(AbstractModule): # TODO KEEP A QUEUE ??????????????????????????????????????????????? +class Sync_module(AbstractModule): """ Sync_module module for AIL framework """ - def __init__(self, queue=False): # FIXME MODIFY/ADD QUEUE + def __init__(self, queue=False): # FIXME MODIFY/ADD QUEUE super(Sync_module, self).__init__(queue=queue) # Waiting time in seconds between to message processed @@ -81,6 +81,10 @@ class Sync_module(AbstractModule): # TODO KEEP A QUEUE ????????????????????????? # Endless loop processing messages from the input queue while self.proceed: + + # Timeout queues + timeout_processed_objs() + # Get one message (paste) from the QueueIn (copy of Redis_Global publish) global_id = get_processed_end_obj() if global_id: diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 00cefd4d..1006be73 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -57,6 +57,9 @@ def get_object_all_subtypes(obj_type): # TODO Dynamic subtype return r_object.smembers(f'all_chat:subtypes') return [] +def get_obj_queued(): + return ['item', 'image'] + def get_objects_tracked(): return ['decoded', 'item', 'pgp', 'title'] diff --git a/bin/lib/ail_queues.py b/bin/lib/ail_queues.py index 9ce647e0..b4218a07 100755 --- a/bin/lib/ail_queues.py +++ b/bin/lib/ail_queues.py @@ -14,10 +14,12 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from lib.exceptions import ModuleQueueError from lib.ConfigLoader import ConfigLoader +from lib import ail_core config_loader = ConfigLoader() r_queues = config_loader.get_redis_conn("Redis_Queues") r_obj_process = config_loader.get_redis_conn("Redis_Process") +timeout_queue_obj = 172800 config_loader = None MODULES_FILE = os.path.join(os.environ['AIL_HOME'], 'configs', 'modules.cfg') @@ -248,6 +250,29 @@ def rename_processed_obj(new_id, old_id): r_obj_process.srem(f'objs:process', old_id) add_processed_obj(new_id, x_hash, module=module) +def timeout_process_obj(obj_global_id): + for q in get_processed_obj_queues(obj_global_id): + queue, x_hash = q.split(':', 1) + r_obj_process.zrem(f'obj:queues:{obj_global_id}', f'{queue}:{x_hash}') + for m in get_processed_obj_modules(obj_global_id): + module, x_hash = m.split(':', 1) + r_obj_process.zrem(f'obj:modules:{obj_global_id}', f'{module}:{x_hash}') + + obj_type = obj_global_id.split(':', 1)[0] + r_obj_process.zrem(f'objs:process:{obj_type}', obj_global_id) + r_obj_process.srem(f'objs:process', obj_global_id) + + r_obj_process.sadd(f'objs:processed', obj_global_id) + print(f'timeout: {obj_global_id}') + + +def timeout_processed_objs(): + curr_time = int(time.time()) + time_limit = curr_time - timeout_queue_obj + for obj_type in ail_core.get_obj_queued(): + for obj_global_id in r_obj_process.zrangebyscore(f'objs:process:{obj_type}', 0, time_limit): + timeout_process_obj(obj_global_id) + def delete_processed_obj(obj_global_id): for q in get_processed_obj_queues(obj_global_id): queue, x_hash = q.split(':', 1) From ba6f45dd4ff9a28fde86e3aae79c1ad6a77de92a Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 8 Jan 2024 14:15:03 +0100 Subject: [PATCH 173/238] chg: [install] remove python package from thirdparty --- requirements.txt | 2 ++ var/www/update_thirdparty.sh | 6 ------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6c813802..7e550eee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,8 @@ pymisp>=2.4.144 d4-pyclient>=0.1.6 thehive4py +git+https://github.com/MISP/PyTaxonomies +git+https://github.com/MISP/PyMISPGalaxies # Core redis>4.4.4 diff --git a/var/www/update_thirdparty.sh b/var/www/update_thirdparty.sh index 863fa7c2..44252d62 100755 --- a/var/www/update_thirdparty.sh +++ b/var/www/update_thirdparty.sh @@ -115,12 +115,6 @@ then source ./../../AILENV/bin/activate fi -#Update MISP Taxonomies and Galaxies -pip3 install git+https://github.com/MISP/PyTaxonomies --upgrade # TODO move to requirement -pip3 install git+https://github.com/MISP/PyMISPGalaxies --upgrade # TODO move to requirement - #Update PyMISP pip3 install pymisp --upgrade -#Update the Hive -pip3 install thehive4py --upgrade # TODO move to requirement From d7c826265362f52875be794aa351ecf3e23f53d5 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 8 Jan 2024 14:24:51 +0100 Subject: [PATCH 174/238] fix: [keys module] fix tags --- bin/modules/Keys.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/bin/modules/Keys.py b/bin/modules/Keys.py index e14523cf..a2e7288d 100755 --- a/bin/modules/Keys.py +++ b/bin/modules/Keys.py @@ -71,26 +71,26 @@ class Keys(AbstractModule): # find = True if KeyEnum.PGP_PUBLIC_KEY_BLOCK.value in content: - tag = f'infoleak:automatic-detection="pgp-public-key-block";{item.get_id()}' + tag = 'infoleak:automatic-detection="pgp-public-key-block"' self.add_message_to_queue(message=tag, queue='Tags') get_pgp_content = True if KeyEnum.PGP_SIGNATURE.value in content: - tag = f'infoleak:automatic-detection="pgp-signature";{item.get_id()}' + tag = 'infoleak:automatic-detection="pgp-signature"' self.add_message_to_queue(message=tag, queue='Tags') get_pgp_content = True if KeyEnum.PGP_PRIVATE_KEY_BLOCK.value in content: self.redis_logger.warning(f'{item.get_basename()} has a pgp private key block message') - tag = f'infoleak:automatic-detection="pgp-private-key";{item.get_id()}' + tag = 'infoleak:automatic-detection="pgp-private-key"' self.add_message_to_queue(message=tag, queue='Tags') get_pgp_content = True if KeyEnum.CERTIFICATE.value in content: self.redis_logger.warning(f'{item.get_basename()} has a certificate message') - tag = f'infoleak:automatic-detection="certificate";{item.get_id()}' + tag = 'infoleak:automatic-detection="certificate"' self.add_message_to_queue(message=tag, queue='Tags') # find = True @@ -98,7 +98,7 @@ class Keys(AbstractModule): self.redis_logger.warning(f'{item.get_basename()} has a RSA private key message') print('rsa private key message found') - tag = f'infoleak:automatic-detection="rsa-private-key";{item.get_id()}' + tag = 'infoleak:automatic-detection="rsa-private-key"' self.add_message_to_queue(message=tag, queue='Tags') # find = True @@ -106,7 +106,7 @@ class Keys(AbstractModule): self.redis_logger.warning(f'{item.get_basename()} has a private key message') print('private key message found') - tag = f'infoleak:automatic-detection="private-key";{item.get_id()}' + tag = 'infoleak:automatic-detection="private-key"' self.add_message_to_queue(message=tag, queue='Tags') # find = True @@ -114,7 +114,7 @@ class Keys(AbstractModule): self.redis_logger.warning(f'{item.get_basename()} has an encrypted private key message') print('encrypted private key message found') - tag = f'infoleak:automatic-detection="encrypted-private-key";{item.get_id()}' + tag = 'infoleak:automatic-detection="encrypted-private-key"' self.add_message_to_queue(message=tag, queue='Tags') # find = True @@ -122,7 +122,7 @@ class Keys(AbstractModule): self.redis_logger.warning(f'{item.get_basename()} has an openssh private key message') print('openssh private key message found') - tag = f'infoleak:automatic-detection="private-ssh-key";{item.get_id()}' + tag = 'infoleak:automatic-detection="private-ssh-key"' self.add_message_to_queue(message=tag, queue='Tags') # find = True @@ -130,7 +130,7 @@ class Keys(AbstractModule): self.redis_logger.warning(f'{item.get_basename()} has an ssh2 private key message') print('SSH2 private key message found') - tag = f'infoleak:automatic-detection="private-ssh-key";{item.get_id()}' + tag = 'infoleak:automatic-detection="private-ssh-key"' self.add_message_to_queue(message=tag, queue='Tags') # find = True @@ -138,28 +138,28 @@ class Keys(AbstractModule): self.redis_logger.warning(f'{item.get_basename()} has an openssh private key message') print('OpenVPN Static key message found') - tag = f'infoleak:automatic-detection="vpn-static-key";{item.get_id()}' + tag = 'infoleak:automatic-detection="vpn-static-key"' self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.DSA_PRIVATE_KEY.value in content: self.redis_logger.warning(f'{item.get_basename()} has a dsa private key message') - tag = f'infoleak:automatic-detection="dsa-private-key";{item.get_id()}' + tag = 'infoleak:automatic-detection="dsa-private-key"' self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.EC_PRIVATE_KEY.value in content: self.redis_logger.warning(f'{item.get_basename()} has an ec private key message') - tag = f'infoleak:automatic-detection="ec-private-key";{item.get_id()}' + tag = 'infoleak:automatic-detection="ec-private-key"' self.add_message_to_queue(message=tag, queue='Tags') # find = True if KeyEnum.PUBLIC_KEY.value in content: self.redis_logger.warning(f'{item.get_basename()} has a public key message') - tag = f'infoleak:automatic-detection="public-key";{item.get_id()}' + tag = 'infoleak:automatic-detection="public-key"' self.add_message_to_queue(message=tag, queue='Tags') # find = True From 8bf67cf3b635d53577fa05ef667c6c23a992bb8c Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 8 Jan 2024 14:51:40 +0100 Subject: [PATCH 175/238] fix: [tags] remove invalid tags --- bin/lib/Tag.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index af193890..ade0f026 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -635,7 +635,7 @@ def update_tag_metadata(tag, date, delete=False): # # TODO: delete Tags # r_tags.smembers(f'{tag}:{date}') # r_tags.smembers(f'{obj_type}:{tag}') def get_tag_objects(tag, obj_type, subtype='', date=''): - if obj_type == 'item': + if obj_type == 'item' or obj_type == 'message': return r_tags.smembers(f'{obj_type}:{subtype}:{tag}:{date}') else: return r_tags.smembers(f'{obj_type}:{subtype}:{tag}') @@ -716,6 +716,12 @@ def delete_object_tag(tag, obj_type, id, subtype=''): date = item_basic.get_item_date(id) r_tags.srem(f'{obj_type}:{subtype}:{tag}:{date}', id) + update_tag_metadata(tag, date, delete=True) + elif obj_type == 'message': + timestamp = id.split('/')[1] + date = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y%m%d') + r_tags.srem(f'{obj_type}:{subtype}:{tag}:{date}', id) + update_tag_metadata(tag, date, delete=True) else: r_tags.srem(f'{obj_type}:{subtype}:{tag}', id) @@ -1458,6 +1464,18 @@ def get_list_of_solo_tags_to_export_by_type(export_type): # by type return None #r_serv_db.smembers('whitelist_hive') +def _fix_tag_obj_id(): + for obj_type in ail_core.get_all_objects(): + for tag in get_all_obj_tags(obj_type): + if ';' in tag: + new_tag = tag.split(';')[0] + for raw in get_obj_by_tags(obj_type, [new_tag], nb_obj=500000): + for global_id in raw.get('tagged_obj', []): + obj_type, subtype, obj_id = global_id.split(':', 2) + delete_object_tag(tag, obj_type, obj_id, subtype=subtype) + add_object_tag(new_tag, obj_type, obj_id, subtype=subtype) + + # if __name__ == '__main__': # taxo = 'accessnow' # # taxo = TAXONOMIES.get(taxo) From 0abc3fee0e9ed057ff288b725e0f2dd3a496fc07 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 8 Jan 2024 14:56:20 +0100 Subject: [PATCH 176/238] fix: [tags] debug --- bin/lib/Tag.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index ade0f026..8fbd00b9 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -1468,10 +1468,13 @@ def _fix_tag_obj_id(): for obj_type in ail_core.get_all_objects(): for tag in get_all_obj_tags(obj_type): if ';' in tag: + print(tag) new_tag = tag.split(';')[0] + print(new_tag) for raw in get_obj_by_tags(obj_type, [new_tag], nb_obj=500000): for global_id in raw.get('tagged_obj', []): obj_type, subtype, obj_id = global_id.split(':', 2) + print(global_id) delete_object_tag(tag, obj_type, obj_id, subtype=subtype) add_object_tag(new_tag, obj_type, obj_id, subtype=subtype) From efb8b2d0d3fafb9049adb26e479cdeaa006cd1f0 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 8 Jan 2024 14:58:18 +0100 Subject: [PATCH 177/238] fix: [tags] debug --- bin/lib/Tag.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index 8fbd00b9..291586b4 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -1471,12 +1471,12 @@ def _fix_tag_obj_id(): print(tag) new_tag = tag.split(';')[0] print(new_tag) - for raw in get_obj_by_tags(obj_type, [new_tag], nb_obj=500000): - for global_id in raw.get('tagged_obj', []): - obj_type, subtype, obj_id = global_id.split(':', 2) - print(global_id) - delete_object_tag(tag, obj_type, obj_id, subtype=subtype) - add_object_tag(new_tag, obj_type, obj_id, subtype=subtype) + raw = get_obj_by_tags(obj_type, [new_tag], nb_obj=500000) + for global_id in raw.get('tagged_obj', []): + obj_type, subtype, obj_id = global_id.split(':', 2) + print(global_id) + delete_object_tag(tag, obj_type, obj_id, subtype=subtype) + add_object_tag(new_tag, obj_type, obj_id, subtype=subtype) # if __name__ == '__main__': From 07c51e111f5a06080cf4cb045f6fff86cdbba987 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 8 Jan 2024 14:59:58 +0100 Subject: [PATCH 178/238] fix: [tags] debug --- bin/lib/Tag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index 291586b4..9bf8f87c 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -1473,8 +1473,8 @@ def _fix_tag_obj_id(): print(new_tag) raw = get_obj_by_tags(obj_type, [new_tag], nb_obj=500000) for global_id in raw.get('tagged_obj', []): - obj_type, subtype, obj_id = global_id.split(':', 2) print(global_id) + obj_type, subtype, obj_id = global_id.split(':', 2) delete_object_tag(tag, obj_type, obj_id, subtype=subtype) add_object_tag(new_tag, obj_type, obj_id, subtype=subtype) From 4b21cc2939b32664f21cde24ce072169dcfb13f8 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 8 Jan 2024 15:01:40 +0100 Subject: [PATCH 179/238] fix: [tags] debug --- bin/lib/Tag.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index 9bf8f87c..92da8a47 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -1472,11 +1472,10 @@ def _fix_tag_obj_id(): new_tag = tag.split(';')[0] print(new_tag) raw = get_obj_by_tags(obj_type, [new_tag], nb_obj=500000) - for global_id in raw.get('tagged_obj', []): - print(global_id) - obj_type, subtype, obj_id = global_id.split(':', 2) - delete_object_tag(tag, obj_type, obj_id, subtype=subtype) - add_object_tag(new_tag, obj_type, obj_id, subtype=subtype) + for obj_id in raw.get('tagged_obj', []): + print(obj_id) + delete_object_tag(tag, obj_type, obj_id) + add_object_tag(new_tag, obj_type, obj_id) # if __name__ == '__main__': From f95d32d6dc16193f25c667ac68c0a4ce94326f22 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 8 Jan 2024 15:02:39 +0100 Subject: [PATCH 180/238] fix: [tags] debug --- bin/lib/Tag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index 92da8a47..98de66e9 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -1473,7 +1473,7 @@ def _fix_tag_obj_id(): print(new_tag) raw = get_obj_by_tags(obj_type, [new_tag], nb_obj=500000) for obj_id in raw.get('tagged_obj', []): - print(obj_id) + # print(obj_id) delete_object_tag(tag, obj_type, obj_id) add_object_tag(new_tag, obj_type, obj_id) From bfc018f9292bd491c79c3f870f4dc74693387bb6 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 8 Jan 2024 15:21:37 +0100 Subject: [PATCH 181/238] fix: [tags] debug --- bin/lib/Tag.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index 98de66e9..3d9ba488 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -684,7 +684,7 @@ def confirm_tag(tag, obj): # TODO REVIEW ME def update_tag_global_by_obj_type(tag, obj_type, subtype=''): tag_deleted = False - if obj_type == 'item': + if obj_type == 'item' or obj_type == 'message': if not r_tags.exists(f'tag_metadata:{tag}'): tag_deleted = True else: @@ -1466,6 +1466,7 @@ def get_list_of_solo_tags_to_export_by_type(export_type): # by type def _fix_tag_obj_id(): for obj_type in ail_core.get_all_objects(): + print(obj_type) for tag in get_all_obj_tags(obj_type): if ';' in tag: print(tag) From 259f29c10cb3cf9ca695ad4cbecb3a95b1aa428c Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 8 Jan 2024 15:27:08 +0100 Subject: [PATCH 182/238] fix: [tags] debug --- bin/lib/Tag.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index 3d9ba488..d9f7896a 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -1464,7 +1464,8 @@ def get_list_of_solo_tags_to_export_by_type(export_type): # by type return None #r_serv_db.smembers('whitelist_hive') -def _fix_tag_obj_id(): +def _fix_tag_obj_id(date_from): + date_to = datetime.date.today().strftime("%Y%m%d") for obj_type in ail_core.get_all_objects(): print(obj_type) for tag in get_all_obj_tags(obj_type): @@ -1472,7 +1473,7 @@ def _fix_tag_obj_id(): print(tag) new_tag = tag.split(';')[0] print(new_tag) - raw = get_obj_by_tags(obj_type, [new_tag], nb_obj=500000) + raw = get_obj_by_tags(obj_type, [new_tag], nb_obj=500000, date_from=date_from, date_to=date_to) for obj_id in raw.get('tagged_obj', []): # print(obj_id) delete_object_tag(tag, obj_type, obj_id) From be4feb7799ea9d6552adc6f1abff420e4a0e4d08 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 8 Jan 2024 15:28:31 +0100 Subject: [PATCH 183/238] fix: [tags] debug --- bin/lib/Tag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index d9f7896a..d16bcbb2 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -1473,7 +1473,7 @@ def _fix_tag_obj_id(date_from): print(tag) new_tag = tag.split(';')[0] print(new_tag) - raw = get_obj_by_tags(obj_type, [new_tag], nb_obj=500000, date_from=date_from, date_to=date_to) + raw = get_obj_by_tags(obj_type, [tag], nb_obj=500000, date_from=date_from, date_to=date_to) for obj_id in raw.get('tagged_obj', []): # print(obj_id) delete_object_tag(tag, obj_type, obj_id) From a14c0484af41f6acc3abadffec8f4a7d29d159b4 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 8 Jan 2024 15:34:32 +0100 Subject: [PATCH 184/238] fix: [tags] debug --- bin/lib/Tag.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index d16bcbb2..6d1f91a9 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -1474,11 +1474,13 @@ def _fix_tag_obj_id(date_from): new_tag = tag.split(';')[0] print(new_tag) raw = get_obj_by_tags(obj_type, [tag], nb_obj=500000, date_from=date_from, date_to=date_to) - for obj_id in raw.get('tagged_obj', []): - # print(obj_id) - delete_object_tag(tag, obj_type, obj_id) - add_object_tag(new_tag, obj_type, obj_id) - + if raw.get('tagged_obj', []): + for obj_id in raw['tagged_obj']: + # print(obj_id) + delete_object_tag(tag, obj_type, obj_id) + add_object_tag(new_tag, obj_type, obj_id) + else: + update_tag_global_by_obj_type(tag, obj_type) # if __name__ == '__main__': # taxo = 'accessnow' From bd2ca4b31933a16fbb815200b9ed9fef7fe34643 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 9 Jan 2024 09:47:49 +0100 Subject: [PATCH 185/238] fix: [crawler] fix api create_task --- bin/lib/crawlers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 13c8f75f..f98af175 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -1788,7 +1788,7 @@ def api_add_crawler_capture(data, user_id): return {'error': 'Invalid task_uuid', 'task_uuid': task_uuid}, 400 capture_uuid = data.get('capture_uuid') if not capture_uuid: - return {'error': 'Invalid capture_uuid', 'task_uuid': capture_uuid}, 400 + return {'error': 'Invalid capture_uuid', 'capture_uuid': capture_uuid}, 400 # parent = data.get('parent') @@ -1796,6 +1796,8 @@ def api_add_crawler_capture(data, user_id): task_uuid = create_task(task['url'], depth=task['depth_limit'], har=task['har'], screenshot=task['screenshot'], proxy=task['proxy'], tags=task['tags'], parent='manual', task_uuid=task_uuid, external=True) + if not task_uuid: + return {'error': 'Aborted by Crawler', 'task_uuid': task_uuid, 'capture_uuid': capture_uuid}, 400 task = CrawlerTask(task_uuid) create_capture(capture_uuid, task_uuid) task.start() From f851cc9f4205931275e0f181e693492959165484 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 9 Jan 2024 11:19:01 +0100 Subject: [PATCH 186/238] fix: [queue] save last timout in cache --- bin/core/Sync_module.py | 10 ++++++++-- bin/lib/ail_queues.py | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/bin/core/Sync_module.py b/bin/core/Sync_module.py index 065be8f3..6dc89672 100755 --- a/bin/core/Sync_module.py +++ b/bin/core/Sync_module.py @@ -22,7 +22,7 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from core import ail_2_ail -from lib.ail_queues import get_processed_end_obj, timeout_processed_objs +from lib.ail_queues import get_processed_end_obj, timeout_processed_objs, get_last_queue_timeout from lib.exceptions import ModuleQueueError from lib.objects import ail_objects from modules.abstract_module import AbstractModule @@ -41,6 +41,7 @@ class Sync_module(AbstractModule): self.dict_sync_queues = ail_2_ail.get_all_sync_queue_dict() self.last_refresh = time.time() + self.last_refresh_queues = time.time() print(self.dict_sync_queues) @@ -83,7 +84,12 @@ class Sync_module(AbstractModule): while self.proceed: # Timeout queues - timeout_processed_objs() + # timeout_processed_objs() + if self.last_refresh_queues < time.time(): + timeout_processed_objs() + self.last_refresh_queues = time.time() + 120 + self.redis_logger.debug('Timeout queues') + # print('Timeout queues') # Get one message (paste) from the QueueIn (copy of Redis_Global publish) global_id = get_processed_end_obj() diff --git a/bin/lib/ail_queues.py b/bin/lib/ail_queues.py index b4218a07..451fcc47 100755 --- a/bin/lib/ail_queues.py +++ b/bin/lib/ail_queues.py @@ -250,6 +250,12 @@ def rename_processed_obj(new_id, old_id): r_obj_process.srem(f'objs:process', old_id) add_processed_obj(new_id, x_hash, module=module) +def get_last_queue_timeout(): + epoch_update = r_obj_process.get('queue:obj:timeout:last') + if not epoch_update: + epoch_update = 0 + return float(epoch_update) + def timeout_process_obj(obj_global_id): for q in get_processed_obj_queues(obj_global_id): queue, x_hash = q.split(':', 1) @@ -272,6 +278,7 @@ def timeout_processed_objs(): for obj_type in ail_core.get_obj_queued(): for obj_global_id in r_obj_process.zrangebyscore(f'objs:process:{obj_type}', 0, time_limit): timeout_process_obj(obj_global_id) + r_obj_process.set('queue:obj:timeout:last', time.time()) def delete_processed_obj(obj_global_id): for q in get_processed_obj_queues(obj_global_id): From 5c25ec0feaab2002cfb0ba711ed57de2023b3ecf Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 9 Jan 2024 11:24:54 +0100 Subject: [PATCH 187/238] fix: [DomClassifier] improve perf --- bin/modules/DomClassifier.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/modules/DomClassifier.py b/bin/modules/DomClassifier.py index e7b77d26..063354a2 100755 --- a/bin/modules/DomClassifier.py +++ b/bin/modules/DomClassifier.py @@ -61,6 +61,8 @@ class DomClassifier(AbstractModule): self.c.text(rawtext=host) print(self.c.domain) + if not self.c.domain: + return self.c.validdomain(passive_dns=True, extended=False) # self.logger.debug(self.c.vdomain) From 7263a9777c1dae78fc5658705f137ebacd0ff61d Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 9 Jan 2024 11:26:05 +0100 Subject: [PATCH 188/238] fix: [DomClassifier] improve perf --- bin/modules/DomClassifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/modules/DomClassifier.py b/bin/modules/DomClassifier.py index 063354a2..ba555a76 100755 --- a/bin/modules/DomClassifier.py +++ b/bin/modules/DomClassifier.py @@ -60,9 +60,9 @@ class DomClassifier(AbstractModule): try: self.c.text(rawtext=host) - print(self.c.domain) if not self.c.domain: return + print(self.c.domain) self.c.validdomain(passive_dns=True, extended=False) # self.logger.debug(self.c.vdomain) From 5094b2dcbbbad646a62367fa3dc58424c51d60d1 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 9 Jan 2024 11:38:54 +0100 Subject: [PATCH 189/238] fix: [DomClassifier] improve perf --- bin/modules/DomClassifier.py | 18 ++++++++++-------- configs/core.cfg.sample | 7 +++++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/bin/modules/DomClassifier.py b/bin/modules/DomClassifier.py index ba555a76..505ef53c 100755 --- a/bin/modules/DomClassifier.py +++ b/bin/modules/DomClassifier.py @@ -73,15 +73,17 @@ class DomClassifier(AbstractModule): for dns_record in self.c.vdomain: self.add_message_to_queue(obj=None, message=dns_record) - localizeddomains = self.c.include(expression=self.cc_tld) - if localizeddomains: - print(localizeddomains) - self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{item.get_id()}") + if self.cc_tld: + localizeddomains = self.c.include(expression=self.cc_tld) + if localizeddomains: + print(localizeddomains) + self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{item.get_id()}") - localizeddomains = self.c.localizedomain(cc=self.cc) - if localizeddomains: - print(localizeddomains) - self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{item.get_id()}") + if self.cc: + localizeddomains = self.c.localizedomain(cc=self.cc) + if localizeddomains: + print(localizeddomains) + self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{item.get_id()}") if r_result: return self.c.vdomain diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index 852590c5..0e0b900f 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -222,10 +222,13 @@ password = ail_trackers cc_critical = DE [DomClassifier] -cc = DE -cc_tld = r'\.de$' +#cc = DE +#cc_tld = r'\.de$' +cc = +cc_tld = dns = 8.8.8.8 + [Mail] dns = 8.8.8.8 From bdaa4c51c980d8b6081ba7ba9899449b2e918c46 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 9 Jan 2024 12:15:40 +0100 Subject: [PATCH 190/238] fix: [hosts] fix number of hosts extracted --- bin/modules/DomClassifier.py | 1 - bin/modules/Hosts.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/modules/DomClassifier.py b/bin/modules/DomClassifier.py index 505ef53c..94cf53db 100755 --- a/bin/modules/DomClassifier.py +++ b/bin/modules/DomClassifier.py @@ -22,7 +22,6 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from modules.abstract_module import AbstractModule -from lib.objects.Items import Item from lib.ConfigLoader import ConfigLoader from lib import d4 diff --git a/bin/modules/Hosts.py b/bin/modules/Hosts.py index fce5595e..979e6680 100755 --- a/bin/modules/Hosts.py +++ b/bin/modules/Hosts.py @@ -55,7 +55,7 @@ class Hosts(AbstractModule): # if mimetype.split('/')[0] == "text": content = item.get_content() - hosts = self.regex_findall(self.host_regex, item.get_id(), content) + hosts = self.regex_findall(self.host_regex, item.get_id(), content, r_set=True) if hosts: print(f'{len(hosts)} host {item.get_id()}') for host in hosts: From d6d67f6a4c1ced317b57ea56a50abffe40f25248 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 9 Jan 2024 14:31:55 +0100 Subject: [PATCH 191/238] chg: [hosts] filter onion --- bin/modules/Hosts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/modules/Hosts.py b/bin/modules/Hosts.py index 979e6680..488e7acf 100755 --- a/bin/modules/Hosts.py +++ b/bin/modules/Hosts.py @@ -60,7 +60,8 @@ class Hosts(AbstractModule): print(f'{len(hosts)} host {item.get_id()}') for host in hosts: # print(host) - self.add_message_to_queue(message=str(host), queue='Host') + if not host.endswith('.onion'): + self.add_message_to_queue(message=str(host), queue='Host') if __name__ == '__main__': From 5418f47e9a2aa12f9e35760769eef3a5af3cc698 Mon Sep 17 00:00:00 2001 From: niclas Date: Mon, 15 Jan 2024 08:26:34 +0100 Subject: [PATCH 192/238] Initial commit --- other_installers/LXD/INSTALL.sh | 89 +++++++++++++++++++++++++++++++++ other_installers/LXD/README.md | 2 + 2 files changed, 91 insertions(+) create mode 100644 other_installers/LXD/INSTALL.sh create mode 100644 other_installers/LXD/README.md diff --git a/other_installers/LXD/INSTALL.sh b/other_installers/LXD/INSTALL.sh new file mode 100644 index 00000000..d795336a --- /dev/null +++ b/other_installers/LXD/INSTALL.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +setVars() { + GREEN='\033[0;32m' + YELLOW='\033[1;33m' + BLUE='\033[0;34m' + RED='\033[0;31m' + NC='\033[0m' # No Color + + PROJECT_NAME=$(generateName "AIL") + STORAGE_POOL_NAME=$(generateName "AIL") + NETWORK_NAME=$(generateName "AIL") + NETWORK_NAME=${NETWORK_NAME:0:14} + + UBUNTU="ubuntu:22.04" + + AIL_CONTAINER=$(generateName "AIL") +} + +error() { + echo -e "${RED}ERROR: $1${NC}" +} + +warn() { + echo -e "${YELLOW}WARNING: $1${NC}" +} + +info() { + echo -e "${BLUE}INFO: $1${NC}" +} + +success() { + echo -e "${GREEN}SUCCESS: $1${NC}" +} + +err() { + local parent_lineno="$1" + local message="$2" + local code="${3:-1}" + + if [[ -n "$message" ]] ; then + error "Line ${parent_lineno}: ${message}: exiting with status ${code}" + else + error "Line ${parent_lineno}: exiting with status ${code}" + fi + + deleteLXDProject "$PROJECT_NAME" + lxc storage delete "$APP_STORAGE" + lxc storage delete "$DB_STORAGE" + lxc network delete "$NETWORK_NAME" + exit "${code}" +} + +generateName(){ + local name="$1" + echo "${name}-$(date +%Y%m%d%H%M%S)" +} + +setupLXD(){ + lxc project create "$PROJECT_NAME" + lxc project switch "$PROJECT_NAME" + lxc storage create "$STORAGE_POOL_NAME" "dir" + lxc network create "$NETWORK_NAME" +} + +waitForContainer() { + local container_name="$1" + + sleep 3 + while true; do + status=$(lxc list --format=json | jq -e --arg name "$container_name" '.[] | select(.name == $name) | .status') + if [ "$status" = "\"Running\"" ]; then + echo -e "${BLUE}$container_name ${GREEN}is running.${NC}" + break + fi + echo "Waiting for $container_name container to start." + sleep 5 + done +} + +interrupt() { + warn "Script interrupted by user. Delete project and exit ..." + deleteLXDProject "$PROJECT_NAME" + lxc storage delete "$APP_STORAGE" + lxc storage delete "$DB_STORAGE" + lxc network delete "$NETWORK_NAME" + exit 130 +} + diff --git a/other_installers/LXD/README.md b/other_installers/LXD/README.md new file mode 100644 index 00000000..fb0f9110 --- /dev/null +++ b/other_installers/LXD/README.md @@ -0,0 +1,2 @@ +# AIL-framework-LXD +This installer is based on the [LXD](https://linuxcontainers.org/lxd/introduction/) container manager and can be used to install AIL on Linux. From 1c46bb4296e21c93e7c977bebafe3e77a6e6815c Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 15 Jan 2024 14:17:15 +0100 Subject: [PATCH 193/238] chg: [Language] replace pycld3 by gcld3 + clean text before language detection --- bin/lib/Language.py | 123 ++++++++++++++++++++++++++++++++---- bin/lib/objects/Items.py | 49 +++----------- bin/lib/objects/Messages.py | 52 --------------- bin/modules/Languages.py | 3 +- requirements.txt | 2 +- 5 files changed, 123 insertions(+), 106 deletions(-) diff --git a/bin/lib/Language.py b/bin/lib/Language.py index 12ad8843..1b8eed2b 100755 --- a/bin/lib/Language.py +++ b/bin/lib/Language.py @@ -2,9 +2,11 @@ # -*-coding:UTF-8 -* import os +import re import sys +import html2text -import cld3 +import gcld3 from libretranslatepy import LibreTranslateAPI sys.path.append(os.environ['AIL_BIN']) @@ -259,6 +261,91 @@ class LanguageDetector: def get_translator_instance(): return TRANSLATOR_URL +def _get_html2text(content, ignore_links=False): + h = html2text.HTML2Text() + h.ignore_links = ignore_links + h.ignore_images = ignore_links + return h.handle(content) + +def _clean_text_to_translate(content, html=False, keys_blocks=True): + if html: + content = _get_html2text(content, ignore_links=True) + + # REMOVE URLS + regex = r'\b(?:http://|https://)?(?:[a-zA-Z\d-]{,63}(?:\.[a-zA-Z\d-]{,63})+)(?:\:[0-9]+)*(?:/(?:$|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*\b' + url_regex = re.compile(regex) + urls = url_regex.findall(content) + urls = sorted(urls, key=len, reverse=True) + for url in urls: + content = content.replace(url, '') + + # REMOVE PGP Blocks + if keys_blocks: + regex_pgp_public_blocs = r'-----BEGIN PGP PUBLIC KEY BLOCK-----[\s\S]+?-----END PGP PUBLIC KEY BLOCK-----' + regex_pgp_signature = r'-----BEGIN PGP SIGNATURE-----[\s\S]+?-----END PGP SIGNATURE-----' + regex_pgp_message = r'-----BEGIN PGP MESSAGE-----[\s\S]+?-----END PGP MESSAGE-----' + re.compile(regex_pgp_public_blocs) + re.compile(regex_pgp_signature) + re.compile(regex_pgp_message) + res = re.findall(regex_pgp_public_blocs, content) + for it in res: + content = content.replace(it, '') + res = re.findall(regex_pgp_signature, content) + for it in res: + content = content.replace(it, '') + res = re.findall(regex_pgp_message, content) + for it in res: + content = content.replace(it, '') + return content + + +class LanguagesDetector: + + def __init__(self, nb_langs=3, min_proportion=0.2, min_probability=0.7, min_len=0): + self.lt = LibreTranslateAPI(get_translator_instance()) + try: + self.lt.languages() + except Exception: + self.lt = None + self.detector = gcld3.NNetLanguageIdentifier(min_num_bytes=0, max_num_bytes=1000) + self.nb_langs = nb_langs + self.min_proportion = min_proportion + self.min_probability = min_probability + self.min_len = min_len + + def detect_gcld3(self, content): + languages = [] + content = _clean_text_to_translate(content, html=True) + if self.min_len > 0: + if len(content) < self.min_len: + return languages + for lang in self.detector.FindTopNMostFreqLangs(content, num_langs=self.nb_langs): + if lang.proportion >= self.min_proportion and lang.probability >= self.min_probability and lang.is_reliable: + languages.append(lang.language) + return languages + + def detect_libretranslate(self, content): + languages = [] + try: + # [{"confidence": 0.6, "language": "en"}] + resp = self.lt.detect(content) + except: # TODO ERROR MESSAGE + resp = [] + if resp: + for language in resp: + if language.confidence >= self.min_probability: + languages.append(language) + return languages + + def detect(self, content): + # gcld3 + if len(content) >= 200 or not self.lt: + language = self.detect_gcld3(content) + # libretranslate + else: + language = self.detect_libretranslate(content) + return language + class LanguageTranslator: def __init__(self): @@ -273,9 +360,15 @@ class LanguageTranslator: print(e) return languages - def detect_cld3(self, content): - for lang in cld3.get_frequent_languages(content, num_langs=1): - return lang.language + def detect_gcld3(self, content): + content = _clean_text_to_translate(content, html=True) + detector = gcld3.NNetLanguageIdentifier(min_num_bytes=0, max_num_bytes=1000) + lang = detector.FindLanguage(content) + # print(lang.language) + # print(lang.is_reliable) + # print(lang.proportion) + # print(lang.probability) + return lang.language def detect_libretranslate(self, content): try: @@ -285,10 +378,10 @@ class LanguageTranslator: if language: return language[0].get('language') - def detect(self, content): # TODO replace by gcld3 - # cld3 + def detect(self, content): + # gcld3 if len(content) >= 200: - language = self.detect_cld3(content) + language = self.detect_gcld3(content) # libretranslate else: language = self.detect_libretranslate(content) @@ -313,18 +406,22 @@ class LanguageTranslator: translation = None return translation -try: - LIST_LANGUAGES = LanguageTranslator().languages() -except Exception as e: - print(e) - LIST_LANGUAGES = [] +LIST_LANGUAGES = [] def get_translation_languages(): + global LIST_LANGUAGES + if not LIST_LANGUAGES: + try: + LIST_LANGUAGES = LanguageTranslator().languages() + except Exception as e: + print(e) + LIST_LANGUAGES = [] return LIST_LANGUAGES if __name__ == '__main__': - t_content = '' + # t_content = '' langg = LanguageTranslator() + # langg = LanguagesDetector() # lang.translate(t_content, source='ru') langg.languages() diff --git a/bin/lib/objects/Items.py b/bin/lib/objects/Items.py index d8888fa0..d29fc521 100755 --- a/bin/lib/objects/Items.py +++ b/bin/lib/objects/Items.py @@ -7,7 +7,6 @@ import magic import os import re import sys -import cld3 import html2text from io import BytesIO @@ -23,6 +22,7 @@ from lib.ail_core import get_ail_uuid, rreplace from lib.objects.abstract_object import AbstractObject from lib.ConfigLoader import ConfigLoader from lib import item_basic +from lib.Language import LanguagesDetector from lib.data_retention_engine import update_obj_date, get_obj_date_first from packages import Date @@ -338,21 +338,10 @@ class Item(AbstractObject): nb_line += 1 return {'nb': nb_line, 'max_length': max_length} + # TODO RENAME ME def get_languages(self, min_len=600, num_langs=3, min_proportion=0.2, min_probability=0.7): - all_languages = [] - ## CLEAN CONTENT ## - content = self.get_html2text_content(ignore_links=True) - content = remove_all_urls_from_content(self.id, item_content=content) ########################################## - # REMOVE USELESS SPACE - content = ' '.join(content.split()) - #- CLEAN CONTENT -# - #print(content) - #print(len(content)) - if len(content) >= min_len: # # TODO: # FIXME: check num langs limit - for lang in cld3.get_frequent_languages(content, num_langs=num_langs): - if lang.proportion >= min_proportion and lang.probability >= min_probability and lang.is_reliable: - all_languages.append(lang) - return all_languages + ld = LanguagesDetector(nb_langs=num_langs, min_proportion=min_proportion, min_probability=min_probability, min_len=min_len) + return ld.detect(self.get_content()) def get_mimetype(self, content=None): if not content: @@ -677,24 +666,6 @@ def remove_all_urls_from_content(item_id, item_content=None): return item_content -def get_item_languages(item_id, min_len=600, num_langs=3, min_proportion=0.2, min_probability=0.7): - all_languages = [] - - ## CLEAN CONTENT ## - content = get_item_content_html2text(item_id, ignore_links=True) - content = remove_all_urls_from_content(item_id, item_content=content) - - # REMOVE USELESS SPACE - content = ' '.join(content.split()) - #- CLEAN CONTENT -# - - #print(content) - #print(len(content)) - if len(content) >= min_len: - for lang in cld3.get_frequent_languages(content, num_langs=num_langs): - if lang.proportion >= min_proportion and lang.probability >= min_probability and lang.is_reliable: - all_languages.append(lang) - return all_languages # API # def get_item(request_dict): @@ -945,13 +916,13 @@ def create_item(obj_id, obj_metadata, io_content): # delete_item(child_id) -if __name__ == '__main__': +# if __name__ == '__main__': # content = 'test file content' # duplicates = {'tests/2020/01/02/test.gz': [{'algo':'ssdeep', 'similarity':75}, {'algo':'tlsh', 'similarity':45}]} # -# item = Item('tests/2020/01/02/test_save.gz') + # item = Item('tests/2020/01/02/test_save.gz') # item.create(content, _save=False) - filters = {'date_from': '20230101', 'date_to': '20230501', 'sources': ['crawled', 'submitted'], 'start': ':submitted/2023/04/28/submitted_2b3dd861-a75d-48e4-8cec-6108d41450da.gz'} - gen = get_all_items_objects(filters=filters) - for obj_id in gen: - print(obj_id.id) +# filters = {'date_from': '20230101', 'date_to': '20230501', 'sources': ['crawled', 'submitted'], 'start': ':submitted/2023/04/28/submitted_2b3dd861-a75d-48e4-8cec-6108d41450da.gz'} +# gen = get_all_items_objects(filters=filters) +# for obj_id in gen: +# print(obj_id.id) diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index e8d422fd..2655c2ee 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -4,8 +4,6 @@ import os import re import sys -import cld3 -import html2text from datetime import datetime @@ -184,14 +182,6 @@ class Message(AbstractObject): """ return self._set_field('translated', translation) # translation by hash ??? -> avoid translating multiple time - def get_html2text_content(self, content=None, ignore_links=False): - if not content: - content = self.get_content() - h = html2text.HTML2Text() - h.ignore_links = ignore_links - h.ignore_images = ignore_links - return h.handle(content) - # def get_ail_2_ail_payload(self): # payload = {'raw': self.get_gzip_content(b64=True)} # return payload @@ -287,48 +277,6 @@ class Message(AbstractObject): # meta['encoding'] = None return meta - def _languages_cleaner(self, content=None): - if not content: - content = self.get_content() - # REMOVE URLS - regex = r'\b(?:http://|https://)?(?:[a-zA-Z\d-]{,63}(?:\.[a-zA-Z\d-]{,63})+)(?:\:[0-9]+)*(?:/(?:$|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*\b' - url_regex = re.compile(regex) - urls = url_regex.findall(content) - urls = sorted(urls, key=len, reverse=True) - for url in urls: - content = content.replace(url, '') - # REMOVE PGP Blocks - regex_pgp_public_blocs = r'-----BEGIN PGP PUBLIC KEY BLOCK-----[\s\S]+?-----END PGP PUBLIC KEY BLOCK-----' - regex_pgp_signature = r'-----BEGIN PGP SIGNATURE-----[\s\S]+?-----END PGP SIGNATURE-----' - regex_pgp_message = r'-----BEGIN PGP MESSAGE-----[\s\S]+?-----END PGP MESSAGE-----' - re.compile(regex_pgp_public_blocs) - re.compile(regex_pgp_signature) - re.compile(regex_pgp_message) - res = re.findall(regex_pgp_public_blocs, content) - for it in res: - content = content.replace(it, '') - res = re.findall(regex_pgp_signature, content) - for it in res: - content = content.replace(it, '') - res = re.findall(regex_pgp_message, content) - for it in res: - content = content.replace(it, '') - return content - - def detect_languages(self, min_len=600, num_langs=3, min_proportion=0.2, min_probability=0.7): - languages = [] - ## CLEAN CONTENT ## - content = self.get_html2text_content(ignore_links=True) - content = self._languages_cleaner(content=content) - # REMOVE USELESS SPACE - content = ' '.join(content.split()) - # - CLEAN CONTENT - # - if len(content) >= min_len: - for lang in cld3.get_frequent_languages(content, num_langs=num_langs): - if lang.proportion >= min_proportion and lang.probability >= min_probability and lang.is_reliable: - languages.append(lang) - return languages - # def translate(self, content=None): # TODO translation plugin # # TODO get text language # if not content: diff --git a/bin/modules/Languages.py b/bin/modules/Languages.py index 69e490a2..e1ce560a 100755 --- a/bin/modules/Languages.py +++ b/bin/modules/Languages.py @@ -31,7 +31,8 @@ class Languages(AbstractModule): if obj.is_crawled(): domain = Domain(obj.get_domain()) for lang in obj.get_languages(min_probability=0.8): - domain.add_language(lang.language) + print(lang) + domain.add_language(lang) if __name__ == '__main__': diff --git a/requirements.txt b/requirements.txt index 7e550eee..6db1d3dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -42,7 +42,7 @@ scrapy>2.0.0 scrapy-splash>=0.7.2 # Languages -pycld3>0.20 +gcld3 libretranslatepy #Graph From 5cf81f82913b4546f323e7d27a432bf88fb61ff4 Mon Sep 17 00:00:00 2001 From: niclas Date: Tue, 16 Jan 2024 08:44:32 +0100 Subject: [PATCH 194/238] add [install] container setup ail + lacus --- other_installers/LXD/INSTALL.sh | 303 +++++++++++++++++++++++++++++++- other_installers/LXD/README.md | 24 ++- 2 files changed, 320 insertions(+), 7 deletions(-) diff --git a/other_installers/LXD/INSTALL.sh b/other_installers/LXD/INSTALL.sh index d795336a..1c265c4f 100644 --- a/other_installers/LXD/INSTALL.sh +++ b/other_installers/LXD/INSTALL.sh @@ -7,14 +7,19 @@ setVars() { RED='\033[0;31m' NC='\033[0m' # No Color - PROJECT_NAME=$(generateName "AIL") STORAGE_POOL_NAME=$(generateName "AIL") NETWORK_NAME=$(generateName "AIL") NETWORK_NAME=${NETWORK_NAME:0:14} + PROFILE=$(generateName "AIL") UBUNTU="ubuntu:22.04" +} - AIL_CONTAINER=$(generateName "AIL") +setDefaults(){ + default_ail_project=$(generateName "AIL") + default_ail_name=$(generateName "AIL") + default_lacus="Yes" + default_lacus_name=$(generateName "LACUS") } error() { @@ -59,8 +64,24 @@ generateName(){ setupLXD(){ lxc project create "$PROJECT_NAME" lxc project switch "$PROJECT_NAME" - lxc storage create "$STORAGE_POOL_NAME" "dir" - lxc network create "$NETWORK_NAME" + + if checkRessourceExist "storage" "$STORAGE_POOL_NAME"; then + error "Storage '$STORAGE_POOL_NAME' already exists." + exit 1 + fi + lxc storage create "$STORAGE_POOL_NAME" zfs source="$PARTITION_NAME" + + if checkRessourceExist "network" "$NETWORK_NAME"; then + error "Network '$NETWORK_NAME' already exists." + fi + lxc network create "$NETWORK_NAME" --type=bridge + + if checkRessourceExist "profile" "$PROFILE"; then + error "Profile '$PROFILE' already exists." + fi + lxc profile create "$PROFILE" + lxc profile device add "$PROFILE" root disk path="/" pool="$STORAGE_POOL_NAME" + lxc profile device add "$PROFILE" eth0 nic name=eth0 network="$NETWORK_NAME" } waitForContainer() { @@ -81,9 +102,279 @@ waitForContainer() { interrupt() { warn "Script interrupted by user. Delete project and exit ..." deleteLXDProject "$PROJECT_NAME" - lxc storage delete "$APP_STORAGE" - lxc storage delete "$DB_STORAGE" lxc network delete "$NETWORK_NAME" exit 130 } +deleteLXDProject(){ + local project="$1" + + echo "Starting cleanup ..." + echo "Deleting container in project" + for container in $(lxc query "/1.0/containers?recursion=1&project=${project}" | jq .[].name -r); do + lxc delete --project "${project}" -f "${container}" + done + + echo "Deleting images in project" + for image in $(lxc query "/1.0/images?recursion=1&project=${project}" | jq .[].fingerprint -r); do + lxc image delete --project "${project}" "${image}" + done + + echo "Deleting profiles in project" + for profile in $(lxc query "/1.0/profiles?recursion=1&project=${project}" | jq .[].name -r); do + if [ "${profile}" = "default" ]; then + printf 'config: {}\ndevices: {}' | lxc profile edit --project "${project}" default + continue + fi + lxc profile delete --project "${project}" "${profile}" + done + + echo "Deleting project" + lxc project delete "${project}" +} + +createAILContainer(){ + lxc launch $UBUNTU "$AIL_CONTAINER" --profile "$PROFILE" + waitForContainer "$AIL_CONTAINER" + lxc exec "$AIL_CONTAINER" -- sed -i "/#\$nrconf{restart} = 'i';/s/.*/\$nrconf{restart} = 'a';/" /etc/needrestart/needrestart.conf + lxc exec "$AIL_CONTAINER" -- apt update + lxc exec "$AIL_CONTAINER" -- apt upgrade -y + lxc exec "$AIL_CONTAINER" -- useradd -m -s /bin/bash ail + if lxc exec "$AIL_CONTAINER" -- id ail; then + lxc exec "$AIL_CONTAINER" -- usermod -aG sudo ail + success "User ail created." + else + error "User ail not created." + exit 1 + fi + lxc exec "$AIL_CONTAINER" -- bash -c "echo 'ail ALL=(ALL) NOPASSWD: ALL' | sudo tee /etc/sudoers.d/ail" + lxc exec "$AIL_CONTAINER" --cwd=/home/ail -- sudo -u ail bash -c "git clone https://github.com/ail-project/ail-framework.git" + lxc exec "$AIL_CONTAINER" --cwd=/home/ail/ail-framework -- sudo -u ail bash -c "./installing_deps.sh" + lxc exec "$AIL_CONTAINER" -- sed -i '/^\[Flask\]/,/^\[/ s/host = 127\.0\.0\.1/host = 0.0.0.0/' /home/ail/ail-framework/configs/core.cfg + lxc exec "$AIL_CONTAINER" --cwd=/home/ail/ail-framework/bin -- sudo -u ail bash -c "./LAUNCH.sh -l" + lxc exec "$AIL_CONTAINER" -- sed -i "/^\$nrconf{restart} = 'a';/s/.*/#\$nrconf{restart} = 'i';/" /etc/needrestart/needrestart.conf +} + +createLacusContainer(){ + lxc launch $UBUNTU "$LACUS_CONTAINER" --profile "$PROFILE" + waitForContainer "$LACUS_CONTAINER" + lxc exec "$LACUS_CONTAINER" -- sed -i "/#\$nrconf{restart} = 'i';/s/.*/\$nrconf{restart} = 'a';/" /etc/needrestart/needrestart.conf + lxc exec "$LACUS_CONTAINER" -- apt update + lxc exec "$LACUS_CONTAINER" -- apt upgrade -y + lxc exec "$LACUS_CONTAINER" -- apt install pipx -y + lxc exec "$LACUS_CONTAINER" -- pipx install poetry + lxc exec "$LACUS_CONTAINER" -- pipx ensurepath + lxc exec "$LACUS_CONTAINER" -- apt install build-essential tcl -y + lxc exec "$LACUS_CONTAINER" -- git clone https://github.com/redis/redis.git + lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- git checkout 7.2 + lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- make + lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- make test + lxc exec "$LACUS_CONTAINER" -- git clone https://github.com/ail-project/lacus.git + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- /root/.local/bin/poetry install + AIL_VENV_PATH=$(lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "/root/.local/bin/poetry env info -p") + # lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "source ${AIL_VENV_PATH}/bin/activate" + # lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- /root/.local/bin/poetry shell + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "source ${AIL_VENV_PATH}/bin/activate && playwright install-deps" + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "echo LACUS_HOME=/root/lacus >> .env" + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "export PATH='/root/.local/bin:$PATH' && yes | /root/.local/bin/poetry run update --init" + lxc exec "$LACUS_CONTAINER" -- sed -i "/^\$nrconf{restart} = 'a';/s/.*/#\$nrconf{restart} = 'i';/" /etc/needrestart/needrestart.conf +} + +nonInteractiveConfig(){ + VALID_ARGS=$(getopt -o h --long help,production,project:ail-name:,no-lacus,lacus-name: -- "$@") + if [[ $? -ne 0 ]]; then + exit 1; + fi + + eval set -- "$VALID_ARGS" + while [ $# -gt 0 ]; do + case "$1" in + -h | --help) + usage + exit 0 + ;; + --project) + ail_project=$2 + shift 2 + ;; + --ail-name) + ail_name=$2 + shift 2 + ;; + --no-lacus) + lacus="N" + shift + ;; + --lacus-name) + lacus_name=$2 + shift 2 + ;; + *) + break + ;; + esac + done + + # Set global values + PROJECT_NAME=${ail_project:-$default_ail_project} + AIL_CONTAINER=${ail_name:-$default_ail_name} + lacus=${lacus:-$default_lacus} + LACUS=$(echo "$lacus" | grep -iE '^y(es)?$' > /dev/null && echo true || echo false) + LACUS_CONTAINER=${lacus_name:-$default_lacus_name} +} + +validateArgs(){ + # Check Names + local names=("$PROJECT_NAME" "$AIL_CONTAINER") + for i in "${names[@]}"; do + if ! checkNamingConvention "$i"; then + exit 1 + fi + done + + if $LACUS && ! checkNamingConvention "$LACUS_CONTAINER"; then + exit 1 + fi + + # Check for Project + if checkRessourceExist "project" "$PROJECT_NAME"; then + error "Project '$PROJECT_NAME' already exists." + exit 1 + fi + + # Check Container Names + local containers=("$AIL_CONTAINER") + + declare -A name_counts + for name in "${containers[@]}"; do + ((name_counts["$name"]++)) + done + + if $LACUS;then + ((name_counts["$LACUS_CONTAINER"]++)) + fi + + for name in "${!name_counts[@]}"; do + if ((name_counts["$name"] >= 2)); then + error "At least two container have the same name: $name" + exit 1 + fi + done +} + +checkRessourceExist() { + local resource_type="$1" + local resource_name="$2" + + case "$resource_type" in + "container") + lxc info "$resource_name" &>/dev/null + ;; + "image") + lxc image list --format=json | jq -e --arg alias "$resource_name" '.[] | select(.aliases[].name == $alias) | .fingerprint' &>/dev/null + ;; + "project") + lxc project list --format=json | jq -e --arg name "$resource_name" '.[] | select(.name == $name) | .name' &>/dev/null + ;; + "storage") + lxc storage list --format=json | jq -e --arg name "$resource_name" '.[] | select(.name == $name) | .name' &>/dev/null + ;; + "network") + lxc network list --format=json | jq -e --arg name "$resource_name" '.[] | select(.name == $name) | .name' &>/dev/null + ;; + "profile") + lxc profile list --format=json | jq -e --arg name "$resource_name" '.[] | select(.name == $name) | .name' &>/dev/null + ;; + esac + + return $? +} + +checkNamingConvention(){ + local input="$1" + local pattern="^[a-zA-Z0-9-]+$" + + if ! [[ "$input" =~ $pattern ]]; then + error "Invalid Name $input. Please use only alphanumeric characters and hyphens." + return 1 + fi + return 0 +} + +usage() { + echo "Usage: $0 [OPTIONS]" + echo + echo "Options:" + echo " -h, --help Display this help message and exit." + echo " --project Specify the project name." + echo " --ail-name Specify the AIL container name." + echo " --no-lacus Do not create Lacus container." + echo " --lacus-name Specify the Lacus container name." + echo " -i, --interactive Run the script in interactive mode." + echo + echo "This script sets up LXD containers for AIL and optionally Lacus." + echo "It creates a new LXD project, and configures the network and storage." + echo "Then it launches and configures the specified containers." + echo + echo "Examples:" + echo " $0 --project myProject --ail-name ailContainer" + echo " $0 --interactive" +} + +# ------------------ MAIN ------------------ + +setDefaults + +# Check if interactive mode +INTERACTIVE=false +for arg in "$@"; do + if [[ $arg == "-i" ]] || [[ $arg == "--interactive" ]]; then + INTERACTIVE=true + break + fi +done + +if [ "$INTERACTIVE" = true ]; then + interactiveConfig +else + nonInteractiveConfig "$@" +fi + +validateArgs +setVars + +trap 'interrupt' INT +trap 'err ${LINENO}' ERR + +info "Setup LXD Project" +setupLXD + +info "Create AIL Container" +createAILContainer + +if $LACUS; then + info "Create Lacus Container" + createLacusContainer +fi + +# Print info +ail_ip=$(lxc list $AIL_CONTAINER --format=json | jq -r '.[0].state.network.eth0.addresses[] | select(.family=="inet").address') +if $LACUS; then + lacus_ip=$(lxc list $LACUS_CONTAINER --format=json | jq -r '.[0].state.network.eth0.addresses[] | select(.family=="inet").address') +fi +ail_email=$(lxc exec $AIL_CONTAINER -- bash -c "grep '^email=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") +ail_password=$(lxc exec $AIL_CONTAINER -- bash -c "grep '^password=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") +ail_API_Key=$(lxc exec $AIL_CONTAINER -- bash -c "grep '^API_Key=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") + +echo "--------------------------------------------------------------------------------------------" +echo -e "${BLUE}AIL ${NC}is up and running on $ail_ip" +echo "--------------------------------------------------------------------------------------------" +echo -e "${BLUE}AIL ${NC}credentials:" +echo -e "Email: ${GREEN}$ail_email${NC}" +echo -e "Password: ${GREEN}$ail_password${NC}" +echo -e "API Key: ${GREEN}$ail_API_Key${NC}" +echo "--------------------------------------------------------------------------------------------" +if $LACUS; then + echo -e "${BLUE}Lacus ${NC}is up and running on $lacus_ip" +fi +echo "--------------------------------------------------------------------------------------------" diff --git a/other_installers/LXD/README.md b/other_installers/LXD/README.md index fb0f9110..28af09ab 100644 --- a/other_installers/LXD/README.md +++ b/other_installers/LXD/README.md @@ -1,2 +1,24 @@ # AIL-framework-LXD -This installer is based on the [LXD](https://linuxcontainers.org/lxd/introduction/) container manager and can be used to install AIL on Linux. +This installer is based on the [LXD](https://canonical.com/lxd) container manager and can be used to install AIL on Linux. It also supports the installation of [Lacus](https://github.com/ail-project/lacus) a crawler for the AIL framework. + +## Requirements +- [LXD](https://canonical.com/lxd) 5.19 +- jq 1.6 + +## Usage +Make sure you have all the requirements installed on your system. + +### Interactive mode +Run the INSTALL.sh script with the --interactive flag to enter the interactive mode, which guides you through the configuration process: +```bash +bash INSTALL.sh --interactive +``` + +### Non-interactive mode +If you want to install AIL without the interactive mode, you can use the following command: +```bash +bash INSTALL.sh [OPTIONS] +``` + +## Configuration +If you installed Lacus, you can configure AIL to use it as a crawler. For further information, please refer to the [HOWTO](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md) From 39aa74aa5242e1796759a84cb3d6647ec299664e Mon Sep 17 00:00:00 2001 From: niclas Date: Tue, 16 Jan 2024 11:41:40 +0100 Subject: [PATCH 195/238] Add [install] install tor on lacus --- other_installers/LXD/INSTALL.sh | 28 +++++++++++++++------- other_installers/LXD/systemd/lacus.service | 19 +++++++++++++++ 2 files changed, 38 insertions(+), 9 deletions(-) create mode 100644 other_installers/LXD/systemd/lacus.service diff --git a/other_installers/LXD/INSTALL.sh b/other_installers/LXD/INSTALL.sh index 1c265c4f..80204def 100644 --- a/other_installers/LXD/INSTALL.sh +++ b/other_installers/LXD/INSTALL.sh @@ -172,12 +172,23 @@ createLacusContainer(){ lxc exec "$LACUS_CONTAINER" -- git clone https://github.com/ail-project/lacus.git lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- /root/.local/bin/poetry install AIL_VENV_PATH=$(lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "/root/.local/bin/poetry env info -p") - # lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "source ${AIL_VENV_PATH}/bin/activate" - # lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- /root/.local/bin/poetry shell lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "source ${AIL_VENV_PATH}/bin/activate && playwright install-deps" lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "echo LACUS_HOME=/root/lacus >> .env" - lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "export PATH='/root/.local/bin:$PATH' && yes | /root/.local/bin/poetry run update --init" + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "export PATH='/root/.local/bin:$PATH' && echo 'no' | /root/.local/bin/poetry run update --init" + # Install Tor + lxc exec "$LACUS_CONTAINER" -- apt install apt-transport-https -y + lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list" + lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb-src https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list" + lxc exec "$LACUS_CONTAINER" -- bash -c "wget -qO- https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc | gpg --dearmor | tee /usr/share/keyrings/tor-archive-keyring.gpg > /dev/null" + lxc exec "$LACUS_CONTAINER" -- apt update + lxc exec "$LACUS_CONTAINER" -- apt install tor deb.torproject.org-keyring -y lxc exec "$LACUS_CONTAINER" -- sed -i "/^\$nrconf{restart} = 'a';/s/.*/#\$nrconf{restart} = 'i';/" /etc/needrestart/needrestart.conf + # Start Lacus + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- cp ./confg/logging.json.sample ./confg/logging.json + lxc file push ./systemd/lacus.service "$LACUS_CONTAINER"/etc/systemd/system/lacus.service + lxc exec "$LACUS_CONTAINER" -- systemctl daemon-reload + lxc exec "$LACUS_CONTAINER" -- systemctl enable lacus.service + lxc exec "$LACUS_CONTAINER" -- systemctl start lacus.service } nonInteractiveConfig(){ @@ -358,14 +369,13 @@ if $LACUS; then fi # Print info -ail_ip=$(lxc list $AIL_CONTAINER --format=json | jq -r '.[0].state.network.eth0.addresses[] | select(.family=="inet").address') +ail_ip=$(lxc list "$AIL_CONTAINER" --format=json | jq -r '.[0].state.network.eth0.addresses[] | select(.family=="inet").address') +ail_email=$(lxc exec "$AIL_CONTAINER" -- bash -c "grep '^email=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") +ail_password=$(lxc exec "$AIL_CONTAINER" -- bash -c "grep '^password=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") +ail_API_Key=$(lxc exec "$AIL_CONTAINER" -- bash -c "grep '^API_Key=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") if $LACUS; then - lacus_ip=$(lxc list $LACUS_CONTAINER --format=json | jq -r '.[0].state.network.eth0.addresses[] | select(.family=="inet").address') + lacus_ip=$(lxc list "$LACUS_CONTAINER" --format=json | jq -r '.[0].state.network.eth0.addresses[] | select(.family=="inet").address') fi -ail_email=$(lxc exec $AIL_CONTAINER -- bash -c "grep '^email=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") -ail_password=$(lxc exec $AIL_CONTAINER -- bash -c "grep '^password=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") -ail_API_Key=$(lxc exec $AIL_CONTAINER -- bash -c "grep '^API_Key=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") - echo "--------------------------------------------------------------------------------------------" echo -e "${BLUE}AIL ${NC}is up and running on $ail_ip" echo "--------------------------------------------------------------------------------------------" diff --git a/other_installers/LXD/systemd/lacus.service b/other_installers/LXD/systemd/lacus.service new file mode 100644 index 00000000..d6f67142 --- /dev/null +++ b/other_installers/LXD/systemd/lacus.service @@ -0,0 +1,19 @@ +[Unit] +Description=lacus service +After=network.target + +[Service] +User=root +Group=root +Type=forking +WorkingDirectory=/root/lacus +Environment="PATH=/root/.local/bin/poetry:/usr/bin" +Environment="LACUS_HOME=/root/lacus" +ExecStart=/bin/bash -c "exec /root/.local/bin/poetry run start" +ExecStop=/bin/bash -c "exec /root/.local/bin/poetry run stop" +StandardOutput=append:/var/log/lacus_message.log +StandardError=append:/var/log/lacus_error.log + + +[Install] +WantedBy=multi-user.target \ No newline at end of file From f586baa0c59db1482ddd3be7ec76f23c6438e82c Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 16 Jan 2024 12:04:39 +0100 Subject: [PATCH 196/238] fix: [Language] fix language source --- bin/lib/Language.py | 10 +++-- bin/lib/chats_viewer.py | 3 ++ bin/lib/objects/Items.py | 37 ------------------- var/www/blueprints/chats_explorer.py | 6 +++ .../chats_explorer/block_translation.html | 2 +- 5 files changed, 17 insertions(+), 41 deletions(-) diff --git a/bin/lib/Language.py b/bin/lib/Language.py index 1b8eed2b..70d5b685 100755 --- a/bin/lib/Language.py +++ b/bin/lib/Language.py @@ -388,6 +388,8 @@ class LanguageTranslator: return language def translate(self, content, source=None, target="en"): # TODO source target + if target not in LIST_LANGUAGES: + return None translation = None if content: if not source: @@ -407,15 +409,17 @@ class LanguageTranslator: return translation -LIST_LANGUAGES = [] +LIST_LANGUAGES = {} def get_translation_languages(): global LIST_LANGUAGES if not LIST_LANGUAGES: try: - LIST_LANGUAGES = LanguageTranslator().languages() + LIST_LANGUAGES = {} + for lang in LanguageTranslator().languages(): + LIST_LANGUAGES[lang['iso']] = lang['language'] except Exception as e: print(e) - LIST_LANGUAGES = [] + LIST_LANGUAGES = {} return LIST_LANGUAGES diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index e1809f47..5b2e2da6 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -23,6 +23,7 @@ from lib.objects import ChatThreads from lib.objects import Messages from lib.objects import UsersAccount from lib.objects import Usernames +from lib import Language config_loader = ConfigLoader() r_db = config_loader.get_db_conn("Kvrocks_DB") @@ -341,6 +342,8 @@ def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, pa if meta['subchannels']: meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels']) else: + if translation_target not in Language.LIST_LANGUAGES: + translation_target = None meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page) return meta, 200 diff --git a/bin/lib/objects/Items.py b/bin/lib/objects/Items.py index d29fc521..7b79749a 100755 --- a/bin/lib/objects/Items.py +++ b/bin/lib/objects/Items.py @@ -629,43 +629,6 @@ def get_item_metadata(item_id, item_content=None): def get_item_content(item_id): return item_basic.get_item_content(item_id) -def get_item_content_html2text(item_id, item_content=None, ignore_links=False): - if not item_content: - item_content = get_item_content(item_id) - h = html2text.HTML2Text() - h.ignore_links = ignore_links - h.ignore_images = ignore_links - return h.handle(item_content) - -def remove_all_urls_from_content(item_id, item_content=None): - if not item_content: - item_content = get_item_content(item_id) - regex = r'\b(?:http://|https://)?(?:[a-zA-Z\d-]{,63}(?:\.[a-zA-Z\d-]{,63})+)(?:\:[0-9]+)*(?:/(?:$|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*\b' - url_regex = re.compile(regex) - urls = url_regex.findall(item_content) - urls = sorted(urls, key=len, reverse=True) - for url in urls: - item_content = item_content.replace(url, '') - - regex_pgp_public_blocs = r'-----BEGIN PGP PUBLIC KEY BLOCK-----[\s\S]+?-----END PGP PUBLIC KEY BLOCK-----' - regex_pgp_signature = r'-----BEGIN PGP SIGNATURE-----[\s\S]+?-----END PGP SIGNATURE-----' - regex_pgp_message = r'-----BEGIN PGP MESSAGE-----[\s\S]+?-----END PGP MESSAGE-----' - re.compile(regex_pgp_public_blocs) - re.compile(regex_pgp_signature) - re.compile(regex_pgp_message) - - res = re.findall(regex_pgp_public_blocs, item_content) - for it in res: - item_content = item_content.replace(it, '') - res = re.findall(regex_pgp_signature, item_content) - for it in res: - item_content = item_content.replace(it, '') - res = re.findall(regex_pgp_message, item_content) - for it in res: - item_content = item_content.replace(it, '') - - return item_content - # API # def get_item(request_dict): diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index 081df950..c24370fb 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -82,6 +82,8 @@ def chats_explorer_chat(): chat_id = request.args.get('id') instance_uuid = request.args.get('uuid') target = request.args.get('target') + if target == "Don't Translate": + target = None nb_messages = request.args.get('nb') page = request.args.get('page') chat = chats_viewer.api_get_chat(chat_id, instance_uuid, translation_target=target, nb=nb_messages, page=page) @@ -111,6 +113,8 @@ def objects_subchannel_messages(): subchannel_id = request.args.get('id') instance_uuid = request.args.get('uuid') target = request.args.get('target') + if target == "Don't Translate": + target = None nb_messages = request.args.get('nb') page = request.args.get('page') subchannel = chats_viewer.api_get_subchannel(subchannel_id, instance_uuid, translation_target=target, nb=nb_messages, page=page) @@ -128,6 +132,8 @@ def objects_thread_messages(): thread_id = request.args.get('id') instance_uuid = request.args.get('uuid') target = request.args.get('target') + if target == "Don't Translate": + target = None nb_messages = request.args.get('nb') page = request.args.get('page') thread = chats_viewer.api_get_thread(thread_id, instance_uuid, translation_target=target, nb=nb_messages, page=page) diff --git a/var/www/templates/chats_explorer/block_translation.html b/var/www/templates/chats_explorer/block_translation.html index 63d47b7c..6c027921 100644 --- a/var/www/templates/chats_explorer/block_translation.html +++ b/var/www/templates/chats_explorer/block_translation.html @@ -12,7 +12,7 @@ {% endif %} {% for language in translation_languages %} - + {% endfor %}
    From 82aec1bf354bf9e5ee760374d57d2568c7a45968 Mon Sep 17 00:00:00 2001 From: niclas Date: Tue, 16 Jan 2024 12:59:06 +0100 Subject: [PATCH 197/238] Fix [tor install] tor install on lacus --- other_installers/LXD/INSTALL.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/other_installers/LXD/INSTALL.sh b/other_installers/LXD/INSTALL.sh index 80204def..264b18fb 100644 --- a/other_installers/LXD/INSTALL.sh +++ b/other_installers/LXD/INSTALL.sh @@ -177,14 +177,14 @@ createLacusContainer(){ lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "export PATH='/root/.local/bin:$PATH' && echo 'no' | /root/.local/bin/poetry run update --init" # Install Tor lxc exec "$LACUS_CONTAINER" -- apt install apt-transport-https -y - lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list" - lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb-src https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list" + lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb [signed-by=/usr/share/keyrings/tor-archive-keyring.gpg] https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list" + lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb-src [signed-by=/usr/share/keyrings/tor-archive-keyring.gpg] https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list" lxc exec "$LACUS_CONTAINER" -- bash -c "wget -qO- https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc | gpg --dearmor | tee /usr/share/keyrings/tor-archive-keyring.gpg > /dev/null" lxc exec "$LACUS_CONTAINER" -- apt update lxc exec "$LACUS_CONTAINER" -- apt install tor deb.torproject.org-keyring -y lxc exec "$LACUS_CONTAINER" -- sed -i "/^\$nrconf{restart} = 'a';/s/.*/#\$nrconf{restart} = 'i';/" /etc/needrestart/needrestart.conf # Start Lacus - lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- cp ./confg/logging.json.sample ./confg/logging.json + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- cp ./config/logging.json.sample ./config/logging.json lxc file push ./systemd/lacus.service "$LACUS_CONTAINER"/etc/systemd/system/lacus.service lxc exec "$LACUS_CONTAINER" -- systemctl daemon-reload lxc exec "$LACUS_CONTAINER" -- systemctl enable lacus.service From 8159b10e5d26f384af1b49e96f985f734840951d Mon Sep 17 00:00:00 2001 From: niclas Date: Tue, 16 Jan 2024 14:21:37 +0100 Subject: [PATCH 198/238] Add [interactive install] interactive install --- other_installers/LXD/INSTALL.sh | 114 +++++++++++++++++++++++++++++--- 1 file changed, 106 insertions(+), 8 deletions(-) diff --git a/other_installers/LXD/INSTALL.sh b/other_installers/LXD/INSTALL.sh index 264b18fb..2c278a89 100644 --- a/other_installers/LXD/INSTALL.sh +++ b/other_installers/LXD/INSTALL.sh @@ -1,12 +1,12 @@ #!/bin/bash -setVars() { - GREEN='\033[0;32m' - YELLOW='\033[1;33m' - BLUE='\033[0;34m' - RED='\033[0;31m' - NC='\033[0m' # No Color +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +RED='\033[0;31m' +NC='\033[0m' # No Color +setVars() { STORAGE_POOL_NAME=$(generateName "AIL") NETWORK_NAME=$(generateName "AIL") NETWORK_NAME=${NETWORK_NAME:0:14} @@ -20,6 +20,7 @@ setDefaults(){ default_ail_name=$(generateName "AIL") default_lacus="Yes" default_lacus_name=$(generateName "LACUS") + default_partition="" } error() { @@ -69,7 +70,7 @@ setupLXD(){ error "Storage '$STORAGE_POOL_NAME' already exists." exit 1 fi - lxc storage create "$STORAGE_POOL_NAME" zfs source="$PARTITION_NAME" + lxc storage create "$STORAGE_POOL_NAME" zfs source="$PARTITION" if checkRessourceExist "network" "$NETWORK_NAME"; then error "Network '$NETWORK_NAME' already exists." @@ -191,8 +192,100 @@ createLacusContainer(){ lxc exec "$LACUS_CONTAINER" -- systemctl start lacus.service } +interactiveConfig(){ + echo + echo "################################################################################" + echo -e "# Welcome to the ${BLUE}AIL-framework-LXD${NC} Installer Script #" + echo "#------------------------------------------------------------------------------#" + echo -e "# This installer script will guide you through the installation process of #" + echo -e "# ${BLUE}AIL${NC} using LXD. #" + echo -e "# #" + echo "################################################################################" + echo + + declare -A nameCheckArray + + # Ask for LXD project name + while true; do + read -r -p "Name of the AIL LXD-project (default: $default_ail_project): " ail_project + PROJECT_NAME=${ail_project:-$default_ail_project} + if ! checkNamingConvention "$PROJECT_NAME"; then + continue + fi + if checkRessourceExist "project" "$PROJECT_NAME"; then + error "Project '$PROJECT_NAME' already exists." + continue + fi + break + done + + # Ask for AIL container name + while true; do + read -r -p "Name of the AIL container (default: $default_ail_name): " ail_name + AIL_CONTAINER=${ail_name:-$default_ail_name} + if [[ ${nameCheckArray[$AIL_CONTAINER]+_} ]]; then + error "Name '$AIL_CONTAINER' has already been used. Please choose a different name." + continue + fi + if ! checkNamingConvention "$AIL_CONTAINER"; then + continue + fi + nameCheckArray[$AIL_CONTAINER]=1 + break + done + + # Ask for Lacus installation + read -r -p "Do you want to install Lacus (y/n, default: $default_lacus): " lacus + lacus=${lacus:-$default_lacus} + LACUS=$(echo "$lacus" | grep -iE '^y(es)?$' > /dev/null && echo true || echo false) + if $LACUS; then + # Ask for LACUS container name + while true; do + read -r -p "Name of the Lacus container (default: $default_lacus_name): " lacus_name + LACUS_CONTAINER=${lacus_name:-$default_lacus_name} + if [[ ${nameCheckArray[$LACUS_CONTAINER]+_} ]]; then + error "Name '$LACUS_CONTAINER' has already been used. Please choose a different name." + continue + fi + if ! checkNamingConvention "$LACUS_CONTAINER"; then + continue + fi + nameCheckArray[$LACUS_CONTAINER]=1 + break + done + + fi + + # Ask for dedicated partitions + read -r -p "Dedicated partition for AIL LXD-project (leave blank if none): " partition + PARTITION=${partition:-$default_partition} + + # Output values set by the user + echo -e "\nValues set:" + echo "--------------------------------------------------------------------------------------------------------------------" + echo -e "PROJECT_NAME: ${GREEN}$PROJECT_NAME${NC}" + echo "--------------------------------------------------------------------------------------------------------------------" + echo -e "AIL_CONTAINER: ${GREEN}$AIL_CONTAINER${NC}" + echo "--------------------------------------------------------------------------------------------------------------------" + echo -e "LACUS: ${GREEN}$LACUS${NC}" + if $LACUS; then + echo -e "LACUS_CONTAINER: ${GREEN}$LACUS_CONTAINER${NC}" + echo "--------------------------------------------------------------------------------------------------------------------" + fi + echo -e "PARTITION: ${GREEN}$PARTITION${NC}" + echo "--------------------------------------------------------------------------------------------------------------------" + + # Ask for confirmation + read -r -p "Do you want to proceed with the installation? (y/n): " confirm + confirm=${confirm:-$default_confirm} + if [[ $confirm != "y" ]]; then + warn "Installation aborted." + exit 1 + fi +} + nonInteractiveConfig(){ - VALID_ARGS=$(getopt -o h --long help,production,project:ail-name:,no-lacus,lacus-name: -- "$@") + VALID_ARGS=$(getopt -o h --long help,production,project:ail-name:,no-lacus,lacus-name:,partition: -- "$@") if [[ $? -ne 0 ]]; then exit 1; fi @@ -204,6 +297,10 @@ nonInteractiveConfig(){ usage exit 0 ;; + --partition) + partition=$2 + shift 2 + ;; --project) ail_project=$2 shift 2 @@ -232,6 +329,7 @@ nonInteractiveConfig(){ lacus=${lacus:-$default_lacus} LACUS=$(echo "$lacus" | grep -iE '^y(es)?$' > /dev/null && echo true || echo false) LACUS_CONTAINER=${lacus_name:-$default_lacus_name} + PARTITION=${partition:-$default_partition} } validateArgs(){ From 45aae5569126e88685ec9676a335b8654d8bb4f8 Mon Sep 17 00:00:00 2001 From: niclas Date: Tue, 16 Jan 2024 14:35:18 +0100 Subject: [PATCH 199/238] Update [README] add installer options --- other_installers/LXD/README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/other_installers/LXD/README.md b/other_installers/LXD/README.md index 28af09ab..59c779cf 100644 --- a/other_installers/LXD/README.md +++ b/other_installers/LXD/README.md @@ -20,5 +20,16 @@ If you want to install AIL without the interactive mode, you can use the followi bash INSTALL.sh [OPTIONS] ``` +The following options are available: +| Flag | Default Value | Description | +| ------------------------------- | ----------------------- | ------------------------------------------------------------------------ | +| `-i`, `--interactive` | N/A | Activates an interactive installation process. | +| `--project ` | `AIL-` | Name of the LXD project for organizing and running the containers. | +| `--ail-name ` | `AIL-` | The name of the container responsible for running the AIL application. | +| `--no-lacus` | `false` | Determines whether to install the Lacus container. | +| `--lacus-name ` | `LACUS-` | The name of the container responsible for running the Lacus application. | +| `--partition ` | `` | Dedicated partition for LXD-project storage. | + + ## Configuration -If you installed Lacus, you can configure AIL to use it as a crawler. For further information, please refer to the [HOWTO](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md) +If you installed Lacus, you can configure AIL to use it as a crawler. For further information, please refer to the [HOWTO.md](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md) From edf0c4c454b4b736605d477d3b081ac396a503e5 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 16 Jan 2024 14:38:29 +0100 Subject: [PATCH 200/238] chg: [message] UI translate message object --- bin/lib/Language.py | 2 +- bin/lib/chats_viewer.py | 4 ++-- var/www/blueprints/chats_explorer.py | 6 +++++- var/www/templates/chats_explorer/ChatMessage.html | 3 +++ 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/bin/lib/Language.py b/bin/lib/Language.py index 70d5b685..2b9bcc9b 100755 --- a/bin/lib/Language.py +++ b/bin/lib/Language.py @@ -388,7 +388,7 @@ class LanguageTranslator: return language def translate(self, content, source=None, target="en"): # TODO source target - if target not in LIST_LANGUAGES: + if target not in get_translation_languages(): return None translation = None if content: diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 5b2e2da6..3c46db5f 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -394,11 +394,11 @@ def api_get_thread(thread_id, thread_instance_uuid, translation_target=None, nb= meta['messages'], meta['pagination'], meta['tags_messages'] = thread.get_messages(translation_target=translation_target, nb=nb, page=page) return meta, 200 -def api_get_message(message_id): +def api_get_message(message_id, translation_target=None): message = Messages.Message(message_id) if not message.exists(): return {"status": "error", "reason": "Unknown uuid"}, 404 - meta = message.get_meta({'chat', 'content', 'icon', 'images', 'link', 'parent', 'parent_meta', 'user-account'}) + meta = message.get_meta({'chat', 'content', 'files-names', 'icon', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'}, translation_target=translation_target) # if meta['chat']: # print(meta['chat']) # # meta['chat'] = diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index c24370fb..eb817da3 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -163,13 +163,17 @@ def chats_explorer_chat_participants(): @login_read_only def objects_message(): message_id = request.args.get('id') - message = chats_viewer.api_get_message(message_id) + target = request.args.get('target') + if target == "Don't Translate": + target = None + message = chats_viewer.api_get_message(message_id, translation_target=target) if message[1] != 200: return create_json_response(message[0], message[1]) else: message = message[0] languages = Language.get_translation_languages() return render_template('ChatMessage.html', meta=message, bootstrap_label=bootstrap_label, + translation_languages=languages, translation_target=target, modal_add_tags=Tag.get_modal_add_tags(message['id'], object_type='message')) @chats_explorer.route("/objects/user-account", methods=['GET']) diff --git a/var/www/templates/chats_explorer/ChatMessage.html b/var/www/templates/chats_explorer/ChatMessage.html index 5a129e66..1e314541 100644 --- a/var/www/templates/chats_explorer/ChatMessage.html +++ b/var/www/templates/chats_explorer/ChatMessage.html @@ -135,6 +135,9 @@ {% include 'objects/image/block_blur_img_slider.html' %} + {% with translate_url=url_for('chats_explorer.objects_message', id=meta['id']), obj_id=meta['id'] %} + {% include 'chats_explorer/block_translation.html' %} + {% endwith %}
    From 8087e46da4084548c2fa55b604946dae9aabe1bb Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 18 Jan 2024 10:54:18 +0100 Subject: [PATCH 201/238] fix: [repo] Point to the correct ail-project repo --- other_installers/ansible/roles/ail-host/tasks/main.yml | 2 +- other_installers/docker/README.md | 2 +- var/www/modules/settings/templates/settings_index.html | 4 ++-- var/www/templates/settings/settings_index.html | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/other_installers/ansible/roles/ail-host/tasks/main.yml b/other_installers/ansible/roles/ail-host/tasks/main.yml index af520fe4..47ada451 100644 --- a/other_installers/ansible/roles/ail-host/tasks/main.yml +++ b/other_installers/ansible/roles/ail-host/tasks/main.yml @@ -64,7 +64,7 @@ - name: Clone the AIL repository git: - repo: 'https://github.com/CIRCL/AIL-framework.git' + repo: 'https://github.com/ail-project/AIL-framework.git' dest: /opt/AIL-framework update: yes diff --git a/other_installers/docker/README.md b/other_installers/docker/README.md index 070f791c..c931bb65 100644 --- a/other_installers/docker/README.md +++ b/other_installers/docker/README.md @@ -15,7 +15,7 @@ curl https://get.docker.com | /bin/bash 2. Type these commands to build the Docker image: ```bash -git clone https://github.com/CIRCL/AIL-framework.git +git clone https://github.com/ail-project/AIL-framework.git cd AIL-framework cp -r ./other_installers/docker/Dockerfile ./other_installers/docker/docker_start.sh ./other_installers/docker/pystemon ./ cp ./configs/update.cfg.sample ./configs/update.cfg diff --git a/var/www/modules/settings/templates/settings_index.html b/var/www/modules/settings/templates/settings_index.html index fd608a64..b6a3fdf7 100644 --- a/var/www/modules/settings/templates/settings_index.html +++ b/var/www/modules/settings/templates/settings_index.html @@ -121,7 +121,7 @@

    New Version Available!

    A new version is available, new version: {{git_metadata['last_remote_tag']}}

    - Check last release note. + Check last release note.
    {%endif%} @@ -130,7 +130,7 @@

    New Update Available!

    A new update is available, new commit ID: {{git_metadata['last_remote_commit']}}

    - Check last commit content. + Check last commit content.
    {%endif%} diff --git a/var/www/templates/settings/settings_index.html b/var/www/templates/settings/settings_index.html index 59d3fed5..dbdd6843 100644 --- a/var/www/templates/settings/settings_index.html +++ b/var/www/templates/settings/settings_index.html @@ -121,7 +121,7 @@

    New Version Available!

    A new version is available, new version: {{git_metadata['last_remote_tag']}}

    - Check last release note. + Check last release note.
    {%endif%} @@ -130,7 +130,7 @@

    New Update Available!

    A new update is available, new commit ID: {{git_metadata['last_remote_commit']}}

    - Check last commit content. + Check last commit content.
    {%endif%} From 0449cc8d25faf8bcf52fdfc2b1907db07a0db76b Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 18 Jan 2024 11:19:40 +0100 Subject: [PATCH 202/238] chg: [UI] If version None do not show release note link --- var/www/modules/settings/templates/settings_index.html | 2 ++ var/www/templates/settings/settings_index.html | 2 ++ 2 files changed, 4 insertions(+) diff --git a/var/www/modules/settings/templates/settings_index.html b/var/www/modules/settings/templates/settings_index.html index b6a3fdf7..2db88d81 100644 --- a/var/www/modules/settings/templates/settings_index.html +++ b/var/www/modules/settings/templates/settings_index.html @@ -43,7 +43,9 @@ AIL Version + {%if ail_version != 'None'%} {{current_version}} (release note) + {%endif%} AIL Version + {%if ail_version != 'None'%} {{ail_version}} (release note) + {%endif%} Date: Thu, 18 Jan 2024 11:42:37 +0100 Subject: [PATCH 203/238] chg: [LAUNCH] Addded a "restart" option, killAll + launchAuto --- bin/LAUNCH.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 68c20ac7..05540d3c 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -69,6 +69,7 @@ function helptext { LAUNCH.sh [-l | --launchAuto] LAUNCH DB + Scripts [-k | --killAll] Kill DB + Scripts + [-kl | --killLaunch] Kill All & launchAuto [-ks | --killscript] Scripts [-u | --update] Update AIL [-ut | --thirdpartyUpdate] Update UI/Frontend @@ -695,6 +696,9 @@ while [ "$1" != "" ]; do ;; -k | --killAll ) killall; ;; + -kl | --killLaunch ) killall; + launch_all "automatic"; + ;; -ks | --killscript ) killscript; ;; -m | --menu ) menu_display; From 9cfd2306611d1d6b6a9d9c34af9bb75a864b5705 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 18 Jan 2024 11:53:13 +0100 Subject: [PATCH 204/238] chg: [LAUNCH] make sure reload works --- bin/LAUNCH.sh | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 05540d3c..58c17d87 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -31,15 +31,19 @@ export PATH=$AIL_KVROCKS:$PATH export PATH=$AIL_BIN:$PATH export PATH=$AIL_FLASK:$PATH -isredis=`screen -ls | egrep '[0-9]+.Redis_AIL' | cut -d. -f1` -isardb=`screen -ls | egrep '[0-9]+.ARDB_AIL' | cut -d. -f1` -iskvrocks=`screen -ls | egrep '[0-9]+.KVROCKS_AIL' | cut -d. -f1` -islogged=`screen -ls | egrep '[0-9]+.Logging_AIL' | cut -d. -f1` -is_ail_core=`screen -ls | egrep '[0-9]+.Core_AIL' | cut -d. -f1` -is_ail_2_ail=`screen -ls | egrep '[0-9]+.AIL_2_AIL' | cut -d. -f1` -isscripted=`screen -ls | egrep '[0-9]+.Script_AIL' | cut -d. -f1` -isflasked=`screen -ls | egrep '[0-9]+.Flask_AIL' | cut -d. -f1` -isfeeded=`screen -ls | egrep '[0-9]+.Feeder_Pystemon' | cut -d. -f1` +function isup { + isredis=`screen -ls | egrep '[0-9]+.Redis_AIL' | cut -d. -f1` + isardb=`screen -ls | egrep '[0-9]+.ARDB_AIL' | cut -d. -f1` + iskvrocks=`screen -ls | egrep '[0-9]+.KVROCKS_AIL' | cut -d. -f1` + islogged=`screen -ls | egrep '[0-9]+.Logging_AIL' | cut -d. -f1` + is_ail_core=`screen -ls | egrep '[0-9]+.Core_AIL' | cut -d. -f1` + is_ail_2_ail=`screen -ls | egrep '[0-9]+.AIL_2_AIL' | cut -d. -f1` + isscripted=`screen -ls | egrep '[0-9]+.Script_AIL' | cut -d. -f1` + isflasked=`screen -ls | egrep '[0-9]+.Flask_AIL' | cut -d. -f1` + isfeeded=`screen -ls | egrep '[0-9]+.Feeder_Pystemon' | cut -d. -f1` +} + +isup function helptext { echo -e $YELLOW" @@ -69,7 +73,7 @@ function helptext { LAUNCH.sh [-l | --launchAuto] LAUNCH DB + Scripts [-k | --killAll] Kill DB + Scripts - [-kl | --killLaunch] Kill All & launchAuto + [-kl | --killLaunch] Kill All & launchAuto [-ks | --killscript] Scripts [-u | --update] Update AIL [-ut | --thirdpartyUpdate] Update UI/Frontend @@ -697,6 +701,7 @@ while [ "$1" != "" ]; do -k | --killAll ) killall; ;; -kl | --killLaunch ) killall; + isup; launch_all "automatic"; ;; -ks | --killscript ) killscript; From 89365844ab310c28f83ec739d419d90e5dce9fb2 Mon Sep 17 00:00:00 2001 From: niclas Date: Fri, 19 Jan 2024 15:56:44 +0100 Subject: [PATCH 205/238] Add [autobuild] build script + systemd config --- other_installers/LXD/.gitignore | 4 + other_installers/LXD/build/ailbuilder.py | 134 ++++++ other_installers/LXD/build/build.sh | 389 ++++++++++++++++++ other_installers/LXD/build/conf/lacus.service | 19 + .../LXD/build/conf/sign.json.template | 7 + .../LXD/build/conf/tracker.json.template | 28 ++ .../LXD/build/systemd/ailbuilder.service | 13 + other_installers/LXD/build/systemd/setup.sh | 88 ++++ other_installers/LXD/build/systemd/update.sh | 36 ++ 9 files changed, 718 insertions(+) create mode 100644 other_installers/LXD/.gitignore create mode 100644 other_installers/LXD/build/ailbuilder.py create mode 100644 other_installers/LXD/build/build.sh create mode 100644 other_installers/LXD/build/conf/lacus.service create mode 100644 other_installers/LXD/build/conf/sign.json.template create mode 100644 other_installers/LXD/build/conf/tracker.json.template create mode 100644 other_installers/LXD/build/systemd/ailbuilder.service create mode 100644 other_installers/LXD/build/systemd/setup.sh create mode 100644 other_installers/LXD/build/systemd/update.sh diff --git a/other_installers/LXD/.gitignore b/other_installers/LXD/.gitignore new file mode 100644 index 00000000..8c8b5b17 --- /dev/null +++ b/other_installers/LXD/.gitignore @@ -0,0 +1,4 @@ +build/conf/sign.json +build/conf/tracker.json +images/ + diff --git a/other_installers/LXD/build/ailbuilder.py b/other_installers/LXD/build/ailbuilder.py new file mode 100644 index 00000000..3044c806 --- /dev/null +++ b/other_installers/LXD/build/ailbuilder.py @@ -0,0 +1,134 @@ +import json +import requests +import subprocess +import re +import os +from time import sleep +from typing import List, Optional +from pathlib import Path + +BUILD_PATH = "/opt/ailbuilder/build" + +class Repo: + """Base class for repository tracking and update checking.""" + + def __init__(self, id: str, args: List[str], name: str, outputdir: str) -> None: + self.id = id + self.args = args + self.name = name + self.outputdir = outputdir + self.last_seen_update = None + + def _check_for_new_update(self) -> bool: + latest_update = self._get_latest_update() + if latest_update and (latest_update != self.last_seen_update): + print(f"New update found for {self.id}") + self.last_seen_update = latest_update + return True + return False + + def _get_latest_update(self): + raise NotImplementedError + + def _save_state(self): + try: + with open(f'{BUILD_PATH}/systemd/state.json', 'r') as file: + states = json.load(file) + except FileNotFoundError: + states = {} + + states[self.id] = self.last_seen_update + + with open(f'{BUILD_PATH}/systemd/state.json', 'w') as file: + json.dump(states, file) + + def load_state(self): + try: + with open(f'{BUILD_PATH}/systemd/state.json', 'r') as file: + states = json.load(file) + except FileNotFoundError: + states = {} + + self.last_seen_update = states.get(self.id, None) + + def build(self) -> None: + if self._check_for_new_update(): + try: + cmd = [f'{BUILD_PATH}/build.sh'] + self.args + ["-o", self.outputdir] + print(f"Running {cmd}") + result = subprocess.run(cmd, check=False) + if result.returncode != 0: + print(f"Failed to run {cmd} for {self.id}") + return + most_recent_dir = max((d for d in Path(self.outputdir).iterdir() if d.is_dir()), key=os.path.getctime, default=None) + relative_path = most_recent_dir.relative_to(Path(self.outputdir)) + if os.path.exists(f"{self.outputdir}/latest_{self.name}"): + os.remove(f"{self.outputdir}/latest_{self.name}") + os.symlink(relative_path, f"{self.outputdir}/latest_{self.name}") + print(f"Created symlink {self.outputdir}/latest_{self.name} to {relative_path}") + self._save_state() + except Exception as e: + print(f"Failed to run {cmd} for {self.id}: {e}") + +class GitHub(Repo): + """Class for tracking GitHub repositories.""" + + def __init__(self, id: str, mode: str, args: List[str], name: str, outputdir: str) -> None: + super().__init__(id, args, name, outputdir) + self.mode = mode + + def _get_latest_update(self) -> Optional[str]: + print(f"Fetching {self.mode} for {self.id}") + url=f'https://api.github.com/repos/{self.id}/{self.mode}' + response = requests.get(url) + if response.status_code == 200: + return response.json()[0]['sha'] + else: + print(f"Failed to fetch {self.mode} for {self.id}") + return None + +class APT(Repo): + """Class for tracking APT packages.""" + + def __init__(self, id: str, args: List[str], name: str, outputdir: str) -> None: + super().__init__(id, args, name, outputdir) + + def _get_latest_update(self) -> Optional[str]: + try: + cmd = ["apt-cache", "policy", self.id] + print (f"Running {cmd}") + output = subprocess.check_output(cmd).decode('utf-8') + match = re.search(r'Candidate: (\S+)', output) + if match: + return match.group(1) + else: + return None + except: + return None + +def main(): + with open(f'{BUILD_PATH}/conf/tracker.json') as f: + config = json.load(f) + + repos = [] + for repo in config["github"]: + repos.append(GitHub(repo["id"], repo["mode"], repo["args"], repo["name"], config["outputdir"])) + + aptpkg = [] + for package in config["apt"]: + aptpkg.append(APT(package["id"], package["args"], package["name"], config["outputdir"])) + + for repo in repos + aptpkg: + if config["sign"]: + repo.args.append("-s") + repo.load_state() + + while True: + for repo in repos: + repo.build() + for package in aptpkg: + package.build() + sleep(config["check_interval"]) + +if __name__ == "__main__": + main() diff --git a/other_installers/LXD/build/build.sh b/other_installers/LXD/build/build.sh new file mode 100644 index 00000000..b785364e --- /dev/null +++ b/other_installers/LXD/build/build.sh @@ -0,0 +1,389 @@ +#!/bin/bash + +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +RED='\033[0;31m' +NC='\033[0m' # No Color + +setVars() { + DEPEDENCIES=("lxc" "jq") + PATH_TO_BUILD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + + PROJECT_NAME=$(generateName "AIL") + STORAGE_POOL_NAME=$(generateName "AIL") + NETWORK_NAME=$(generateName "AIL") + NETWORK_NAME=${NETWORK_NAME:0:14} + + UBUNTU="ubuntu:22.04" + + AIL_CONTAINER=$(generateName "AIL") + LACUS_CONTAINER=$(generateName "LACUS") + LACUS_SERVICE_FILE="$PATH_TO_BUILD/conf/lacus.service" +} + +setDefaults(){ + default_ail=false + default_ail_image="AIL" + default_lacus=false + default_lacus_image="Lacus" + default_outputdir="" + default_sign=false +} + +error() { + echo -e "${RED}ERROR: $1${NC}" +} + +warn() { + echo -e "${YELLOW}WARNING: $1${NC}" +} + +info() { + echo -e "${BLUE}INFO: $1${NC}" +} + +success() { + echo -e "${GREEN}SUCCESS: $1${NC}" +} + +err() { + local parent_lineno="$1" + local message="$2" + local code="${3:-1}" + + if [[ -n "$message" ]] ; then + error "Line ${parent_lineno}: ${message}: exiting with status ${code}" + else + error "Line ${parent_lineno}: exiting with status ${code}" + fi + + deleteLXDProject "$PROJECT_NAME" + lxc storage delete "$APP_STORAGE" + lxc storage delete "$DB_STORAGE" + lxc network delete "$NETWORK_NAME" + exit "${code}" +} + +generateName(){ + local name="$1" + echo "${name}-$(date +%Y%m%d%H%M%S)" +} + + +waitForContainer() { + local container_name="$1" + + sleep 3 + while true; do + status=$(lxc list --format=json | jq -e --arg name "$container_name" '.[] | select(.name == $name) | .status') + if [ "$status" = "\"Running\"" ]; then + echo -e "${BLUE}$container_name ${GREEN}is running.${NC}" + break + fi + echo "Waiting for $container_name container to start." + sleep 5 + done +} + +interrupt() { + warn "Script interrupted by user. Delete project and exit ..." + deleteLXDProject "$PROJECT_NAME" + lxc network delete "$NETWORK_NAME" + exit 130 +} + +cleanupProject(){ + local project="$1" + + info "Starting cleanup ..." + echo "Deleting container in project" + for container in $(lxc query "/1.0/containers?recursion=1&project=${project}" | jq .[].name -r); do + lxc delete --project "${project}" -f "${container}" + done + + echo "Deleting images in project" + for image in $(lxc query "/1.0/images?recursion=1&project=${project}" | jq .[].fingerprint -r); do + lxc image delete --project "${project}" "${image}" + done + + echo "Deleting project" + lxc project delete "${project}" +} + +cleanup(){ + cleanupProject "$PROJECT_NAME" + lxc storage delete "$STORAGE_POOL_NAME" + lxc network delete "$NETWORK_NAME" +} + +createAILContainer(){ + lxc launch $UBUNTU "$AIL_CONTAINER" -p default --storage "$STORAGE_POOL_NAME" --network "$NETWORK_NAME" + waitForContainer "$AIL_CONTAINER" + lxc exec "$AIL_CONTAINER" -- sed -i "/#\$nrconf{restart} = 'i';/s/.*/\$nrconf{restart} = 'a';/" /etc/needrestart/needrestart.conf + lxc exec "$AIL_CONTAINER" -- apt update + lxc exec "$AIL_CONTAINER" -- apt upgrade -y + lxc exec "$AIL_CONTAINER" -- useradd -m -s /bin/bash ail + if lxc exec "$AIL_CONTAINER" -- id ail; then + lxc exec "$AIL_CONTAINER" -- usermod -aG sudo ail + success "User ail created." + else + error "User ail not created." + exit 1 + fi + lxc exec "$AIL_CONTAINER" -- bash -c "echo 'ail ALL=(ALL) NOPASSWD: ALL' | sudo tee /etc/sudoers.d/ail" + lxc exec "$AIL_CONTAINER" --cwd=/home/ail -- sudo -u ail bash -c "git clone https://github.com/ail-project/ail-framework.git" + lxc exec "$AIL_CONTAINER" --cwd=/home/ail/ail-framework -- sudo -u ail bash -c "./installing_deps.sh" + lxc exec "$AIL_CONTAINER" -- sed -i '/^\[Flask\]/,/^\[/ s/host = 127\.0\.0\.1/host = 0.0.0.0/' /home/ail/ail-framework/configs/core.cfg + lxc exec "$AIL_CONTAINER" --cwd=/home/ail/ail-framework/bin -- sudo -u ail bash -c "./LAUNCH.sh -l" + lxc exec "$AIL_CONTAINER" -- sed -i "/^\$nrconf{restart} = 'a';/s/.*/#\$nrconf{restart} = 'i';/" /etc/needrestart/needrestart.conf +} + +createLacusContainer(){ + lxc launch $UBUNTU "$LACUS_CONTAINER" -p default --storage "$STORAGE_POOL_NAME" --network "$NETWORK_NAME" + waitForContainer "$LACUS_CONTAINER" + lxc exec "$LACUS_CONTAINER" -- sed -i "/#\$nrconf{restart} = 'i';/s/.*/\$nrconf{restart} = 'a';/" /etc/needrestart/needrestart.conf + lxc exec "$LACUS_CONTAINER" -- apt update + lxc exec "$LACUS_CONTAINER" -- apt upgrade -y + lxc exec "$LACUS_CONTAINER" -- apt install pipx -y + lxc exec "$LACUS_CONTAINER" -- pipx install poetry + lxc exec "$LACUS_CONTAINER" -- pipx ensurepath + lxc exec "$LACUS_CONTAINER" -- apt install build-essential tcl -y + lxc exec "$LACUS_CONTAINER" -- git clone https://github.com/redis/redis.git + lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- git checkout 7.2 + lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- make + lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- make test + lxc exec "$LACUS_CONTAINER" -- git clone https://github.com/ail-project/lacus.git + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- /root/.local/bin/poetry install + AIL_VENV_PATH=$(lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "/root/.local/bin/poetry env info -p") + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "source ${AIL_VENV_PATH}/bin/activate && playwright install-deps" + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "echo LACUS_HOME=/root/lacus >> .env" + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "export PATH='/root/.local/bin:$PATH' && echo 'no' | /root/.local/bin/poetry run update --init" + # Install Tor + lxc exec "$LACUS_CONTAINER" -- apt install apt-transport-https -y + lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb [signed-by=/usr/share/keyrings/tor-archive-keyring.gpg] https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list" + lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb-src [signed-by=/usr/share/keyrings/tor-archive-keyring.gpg] https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list" + lxc exec "$LACUS_CONTAINER" -- bash -c "wget -qO- https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc | gpg --dearmor | tee /usr/share/keyrings/tor-archive-keyring.gpg > /dev/null" + lxc exec "$LACUS_CONTAINER" -- apt update + lxc exec "$LACUS_CONTAINER" -- apt install tor deb.torproject.org-keyring -y + lxc exec "$LACUS_CONTAINER" -- sed -i "/^\$nrconf{restart} = 'a';/s/.*/#\$nrconf{restart} = 'i';/" /etc/needrestart/needrestart.conf + # Start Lacus + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- cp ./config/logging.json.sample ./config/logging.json + lxc file push "$LACUS_SERVICE_FILE" "$LACUS_CONTAINER"/etc/systemd/system/lacus.service + lxc exec "$LACUS_CONTAINER" -- systemctl daemon-reload + lxc exec "$LACUS_CONTAINER" -- systemctl enable lacus.service + lxc exec "$LACUS_CONTAINER" -- systemctl start lacus.service +} + +createLXDImage() { + local container="$1" + local image_name="$2" + local path_to_repo="$3" + local user="$4" + + local commit_id + local version + commit_id=$(getCommitID "$container" "$path_to_repo") + version=$(getVersion "$container" "$path_to_repo" "$user") + + lxc stop "$container" + lxc publish "$container" --alias "$image_name" + lxc image export "$image_name" "$OUTPUTDIR" + local file_name + file_name="${image_name}_${version}_${commit_id}.tar.gz" + pushd "$OUTPUTDIR" && mv -i "$(ls -t | head -n1)" "$file_name" + popd || { error "Failed to rename image file"; exit 1; } + sleep 2 + if $SIGN; then + sign "$file_name" + fi +} + +getCommitID() { + local container="$1" + local path_to_repo="$2" + local current_branch + current_branch=$(lxc exec "$container" -- cat "$path_to_repo"/.git/HEAD | awk '{print $2}') + local commit_id + commit_id=$(lxc exec "$container" -- cat "$path_to_repo"/.git/"$current_branch") + echo "$commit_id" +} + +getVersion() { + local container="$1" + local path_to_repo="$2" + local user="$3" + local version + version=$(lxc exec "$container" --cwd="$path_to_repo" -- sudo -u "$user" bash -c "git tag | sort -V | tail -n 1") + echo "$version" +} + +sign() { + if ! command -v gpg &> /dev/null; then + error "GPG is not installed. Please install it before running this script with signing." + exit 1 + fi + local file=$1 + SIGN_CONFIG_FILE="$PATH_TO_BUILD/conf/sign.json" + + if [[ ! -f "$SIGN_CONFIG_FILE" ]]; then + error "Config file not found: $SIGN_CONFIG_FILE" + exit 1 + fi + + GPG_KEY_ID=$(jq -r '.EMAIL' "$SIGN_CONFIG_FILE") + GPG_KEY_PASSPHRASE=$(jq -r '.PASSPHRASE' "$SIGN_CONFIG_FILE") + + # Check if the GPG key is available + if ! gpg --list-keys | grep -q "$GPG_KEY_ID"; then + warn "GPG key not found: $GPG_KEY_ID. Create new key." + # Setup GPG key + KEY_NAME=$(jq -r '.NAME' "$SIGN_CONFIG_FILE") + KEY_EMAIL=$(jq -r '.EMAIL' "$SIGN_CONFIG_FILE") + KEY_COMMENT=$(jq -r '.COMMENT' "$SIGN_CONFIG_FILE") + KEY_EXPIRE=$(jq -r '.EXPIRE_DATE' "$SIGN_CONFIG_FILE") + KEY_PASSPHRASE=$(jq -r '.PASSPHRASE' "$SIGN_CONFIG_FILE") + BATCH_FILE=$(mktemp -d)/batch + + cat > "$BATCH_FILE" < /dev/null; then + echo -e "${RED}Error: $dep is not installed.${NC}" + exit 1 + fi + done +} + +usage() { + echo "Usage: $0 [OPTIONS]" +} + +# ------------------ MAIN ------------------ +checkSoftwareDependencies "${DEPEDENCIES[@]}" +setVars +setDefaults + +VALID_ARGS=$(getopt -o ho:s --long help,outputdir:,sign,ail,lacus,ail-name:,lacus-name: -- "$@") +if [[ $? -ne 0 ]]; then + exit 1; +fi + +eval set -- "$VALID_ARGS" +while [ $# -gt 0 ]; do + case "$1" in + -h | --help) + usage + exit 0 + ;; + --ail) + ail=true + shift + ;; + --lacus) + lacus=true + shift + ;; + --ail-name) + ail_image=$2 + shift 2 + ;; + --lacus-name) + lacus_image=$2 + shift 2 + ;; + -o | --outputdir) + outputdir=$2 + shift 2 + ;; + -s | --sign) + sign=true + shift + ;; + *) + break + ;; + esac +done + +AIL=${ail:-$default_ail} +LACUS=${lacus:-$default_lacus} +AIL_IMAGE=${ail_image:-$default_ail_image} +LACUS_IMAGE=${lacus_image:-$default_lacus_image} +OUTPUTDIR=${outputdir:-$default_outputdir} +SIGN=${sign:-$default_sign} + +if [ ! -e "$OUTPUTDIR" ]; then + error "The specified directory does not exist." + exit 1 +fi + +if ! $AIL && ! $LACUS; then + error "No image specified!" + exit 1 +fi + +echo "----------------------------------------" +echo "Startting creating LXD images ..." +echo "----------------------------------------" + +trap cleanup EXIT + +lxc project create "$PROJECT_NAME" +lxc project switch "$PROJECT_NAME" +lxc storage create "$STORAGE_POOL_NAME" "dir" +lxc network create "$NETWORK_NAME" + +if $AIL; then + createAILContainer + createLXDImage "$AIL_CONTAINER" "$AIL_IMAGE" "/home/ail/ail-framework" "ail" +fi + +if $LACUS; then + createLacusContainer + createLXDImage "$LACUS_CONTAINER" "$LACUS_IMAGE" "/root/lacus" "root" +fi + +echo "----------------------------------------" +echo "Build script finished." +echo "----------------------------------------" \ No newline at end of file diff --git a/other_installers/LXD/build/conf/lacus.service b/other_installers/LXD/build/conf/lacus.service new file mode 100644 index 00000000..d6f67142 --- /dev/null +++ b/other_installers/LXD/build/conf/lacus.service @@ -0,0 +1,19 @@ +[Unit] +Description=lacus service +After=network.target + +[Service] +User=root +Group=root +Type=forking +WorkingDirectory=/root/lacus +Environment="PATH=/root/.local/bin/poetry:/usr/bin" +Environment="LACUS_HOME=/root/lacus" +ExecStart=/bin/bash -c "exec /root/.local/bin/poetry run start" +ExecStop=/bin/bash -c "exec /root/.local/bin/poetry run stop" +StandardOutput=append:/var/log/lacus_message.log +StandardError=append:/var/log/lacus_error.log + + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/other_installers/LXD/build/conf/sign.json.template b/other_installers/LXD/build/conf/sign.json.template new file mode 100644 index 00000000..e3554a77 --- /dev/null +++ b/other_installers/LXD/build/conf/sign.json.template @@ -0,0 +1,7 @@ +{ + "NAME": "admin", + "EMAIL": "admin@admin.test", + "COMMENT": "Key for signing images", + "EXPIRE_DATE": 0, + "PASSPHRASE": "admin" +} \ No newline at end of file diff --git a/other_installers/LXD/build/conf/tracker.json.template b/other_installers/LXD/build/conf/tracker.json.template new file mode 100644 index 00000000..ea97ceb0 --- /dev/null +++ b/other_installers/LXD/build/conf/tracker.json.template @@ -0,0 +1,28 @@ +{ + "check_interval": 600, + "outputdir": "/opt/ailbuilder/images", + "sign": true, + "github": [ + { + "name": "AIL", + "id": "ail-project/ail-framework", + "mode": "commits", + "args": [ + "--ail", + "--ail-name", + "AIL" + ] + }, + { + "name": "Lacus", + "id": "ail-project/lacus", + "mode": "commits", + "args": [ + "--lacus", + "--lacus-name", + "Lacus" + ] + } + ], + "apt": [] +} \ No newline at end of file diff --git a/other_installers/LXD/build/systemd/ailbuilder.service b/other_installers/LXD/build/systemd/ailbuilder.service new file mode 100644 index 00000000..8935a2a7 --- /dev/null +++ b/other_installers/LXD/build/systemd/ailbuilder.service @@ -0,0 +1,13 @@ +[Unit] +Description=Service for building AIL and Lacus LXD images +After=network.target + +[Service] +Type=simple +User=ailbuilder +ExecStart=/usr/bin/python3 /opt/ailbuilder/build/ailbuilder.py +Restart=on-failure +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/other_installers/LXD/build/systemd/setup.sh b/other_installers/LXD/build/systemd/setup.sh new file mode 100644 index 00000000..42230a66 --- /dev/null +++ b/other_installers/LXD/build/systemd/setup.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +DIR="$(dirname "$0")" +SERVICE_FILE="ailbuilder.service" +SERVICE_PATH="/etc/systemd/system/" +AILBUILDER_PATH="/opt/ailbuilder" +BUILD_DIR="${DIR}/../../build" +BATCH_FILE="/tmp/key_batch" +SIGN_CONFIG_FILE="../conf/sign.json" + +log() { + echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*" +} + +echo "Start setting up ailbuilder service ..." + +if [[ $EUID -ne 0 ]]; then + log "This script must be run as root or with sudo privileges" + exit 1 +fi + +if [[ ! -d "$AILBUILDER_PATH" ]]; then + mkdir -p "$AILBUILDER_PATH"/images || { log "Failed to create directory $AILBUILDER_PATH"; exit 1; } +fi + +if [[ -d "$BUILD_DIR" ]]; then + cp -r "$BUILD_DIR" "$AILBUILDER_PATH/" || { log "Failed to copy build directory"; exit 1; } +else + log "Build directory $BUILD_DIR does not exist" + exit 1 +fi + +# Create user if it doesn't exist +if ! id "ailbuilder" &>/dev/null; then + useradd -r -s /bin/false ailbuilder || { log "Failed to create user ailbuilder"; exit 1; } +fi + +# Set ownership and permissions +chown -R ailbuilder: "$AILBUILDER_PATH" || { log "Failed to change ownership"; exit 1; } +chmod -R u+x "$AILBUILDER_PATH/build/"*.py || { log "Failed to set execute permission on scripts"; exit 1; } +chmod -R u+x "$AILBUILDER_PATH/build/"*.sh || { log "Failed to set execute permission on scripts"; exit 1; } +chmod -R u+w "$AILBUILDER_PATH/images/" || { log "Failed to set execute permission on images dir"; exit 1; } + +# Add user to lxd group +sudo usermod -aG lxd ailbuilder || { log "Failed to add user ailbuilder to lxd group"; exit 1; } +mkdir -p /home/ailbuilder || { log "Failed to create directory /home/ailbuilder"; exit 1; } +chown -R ailbuilder: "/home/ailbuilder" || { log "Failed to change ownership"; exit 1; } +chmod -R u+w "/home/ailbuilder" || { log "Failed to set execute permission on home dir"; exit 1; } + +# Setup GPG key +KEY_NAME=$(jq -r '.NAME' "$SIGN_CONFIG_FILE") +KEY_EMAIL=$(jq -r '.EMAIL' "$SIGN_CONFIG_FILE") +KEY_COMMENT=$(jq -r '.COMMENT' "$SIGN_CONFIG_FILE") +KEY_EXPIRE=$(jq -r '.EXPIRE_DATE' "$SIGN_CONFIG_FILE") +KEY_PASSPHRASE=$(jq -r '.PASSPHRASE' "$SIGN_CONFIG_FILE") + +if ! sudo -u ailbuilder bash -c "gpg --list-keys | grep -q $KEY_EMAIL"; then + cat > "$BATCH_FILE" < Date: Fri, 26 Jan 2024 15:31:32 +0100 Subject: [PATCH 206/238] fix: [crawler] log UNKNOWN timeout --- bin/crawlers/Crawler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index 0d6cb67c..c1ba0e0c 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -146,6 +146,7 @@ class Crawler(AbstractModule): task = capture.get_task() task.reset() capture.delete() + self.logger.warning(f'capture UNKNOWN Timeout, {task.uuid} Send back in queue') else: capture.update(status) else: From 699453f079a8e54b531ed8ba2e72a1c7377a20bd Mon Sep 17 00:00:00 2001 From: terrtia Date: Fri, 26 Jan 2024 15:42:46 +0100 Subject: [PATCH 207/238] chg: [relationships] add relationship engine + WIP relationships between forwarded messages/chats --- bin/importer/FeederImporter.py | 6 +- bin/importer/feeders/abstract_chats_feeder.py | 54 +- bin/lib/objects/Chats.py | 3 + bin/lib/objects/Messages.py | 33 +- bin/lib/objects/abstract_object.py | 17 + bin/lib/objects/ail_objects.py | 19 + bin/lib/relationships_engine.py | 111 +++ configs/6383.conf | 1 + configs/core.cfg.sample | 5 + var/www/blueprints/correlation.py | 63 +- .../correlation/metadata_card_chat.html | 78 ++ .../correlation/show_correlation.html | 9 + .../correlation/show_relationship.html | 719 ++++++++++++++++++ 13 files changed, 1097 insertions(+), 21 deletions(-) create mode 100755 bin/lib/relationships_engine.py create mode 100644 var/www/templates/correlation/metadata_card_chat.html create mode 100644 var/www/templates/correlation/show_relationship.html diff --git a/bin/importer/FeederImporter.py b/bin/importer/FeederImporter.py index 881cb8a9..34ccca8c 100755 --- a/bin/importer/FeederImporter.py +++ b/bin/importer/FeederImporter.py @@ -100,14 +100,15 @@ class FeederImporter(AbstractImporter): else: objs = set() - objs.add(data_obj) + if data_obj: + objs.add(data_obj) for obj in objs: if obj.type == 'item': # object save on disk as file (Items) gzip64_content = feeder.get_gzip64_content() return obj, f'{feeder_name} {gzip64_content}' else: # Messages save on DB - if obj.exists(): + if obj.exists() and obj.type != 'chat': return obj, f'{feeder_name}' @@ -136,4 +137,5 @@ class FeederModuleImporter(AbstractModule): # Launch Importer if __name__ == '__main__': module = FeederModuleImporter() + # module.debug = True module.run() diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py index 1dadb970..6b8f1041 100755 --- a/bin/importer/feeders/abstract_chats_feeder.py +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -93,7 +93,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC): return self.json_data['meta'].get('reactions', []) def get_message_timestamp(self): - return self.json_data['meta']['date']['timestamp'] # TODO CREATE DEFAULT TIMESTAMP + if not self.json_data['meta'].get('date'): + return None + else: + return self.json_data['meta']['date']['timestamp'] # if self.json_data['meta'].get('date'): # date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp']) # date = date.strftime('%Y/%m/%d') @@ -115,17 +118,29 @@ class AbstractChatFeeder(DefaultFeeder, ABC): def get_message_reply_id(self): return self.json_data['meta'].get('reply_to', {}).get('message_id') + def get_message_forward(self): + return self.json_data['meta'].get('forward') + def get_message_content(self): decoded = base64.standard_b64decode(self.json_data['data']) return _gunzip_bytes_obj(decoded) - def get_obj(self): # TODO handle others objects -> images, pdf, ... + def get_obj(self): #### TIMESTAMP #### timestamp = self.get_message_timestamp() #### Create Object ID #### chat_id = self.get_chat_id() - message_id = self.get_message_id() + try: + message_id = self.get_message_id() + except KeyError: + if chat_id: + self.obj = Chat(chat_id, self.get_chat_instance_uuid()) + return self.obj + else: + self.obj = None + return None + thread_id = self.get_thread_id() # channel id # thread id @@ -236,7 +251,10 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # # ADD NEW MESSAGE REF (used by discord) def process_sender(self, new_objs, obj, date, timestamp): - meta = self.json_data['meta']['sender'] + meta = self.json_data['meta'].get('sender') + if not meta: + return None + user_account = UsersAccount.UserAccount(meta['id'], self.get_chat_instance_uuid()) # date stat + correlation @@ -286,8 +304,6 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # REPLY reply_id = self.get_message_reply_id() - # TODO Translation - print(self.obj.type) # TODO FILES + FILES REF @@ -295,7 +311,7 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # get object by meta object type if self.obj.type == 'message': # Content - obj = Messages.create(self.obj.id, self.get_message_content()) # TODO translation + obj = Messages.create(self.obj.id, self.get_message_content()) # FILENAME media_name = self.get_media_name() @@ -305,7 +321,8 @@ class AbstractChatFeeder(DefaultFeeder, ABC): for reaction in self.get_reactions(): obj.add_reaction(reaction['reaction'], int(reaction['count'])) - + elif self.obj.type == 'chat': + pass else: chat_id = self.get_chat_id() thread_id = self.get_thread_id() @@ -341,12 +358,29 @@ class AbstractChatFeeder(DefaultFeeder, ABC): # CHAT chat_objs = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id) + # Message forward + # if self.get_json_meta().get('forward'): + # forward_from = self.get_message_forward() + # print('-----------------------------------------------------------') + # print(forward_from) + # if forward_from: + # forward_from_type = forward_from['from']['type'] + # if forward_from_type == 'channel' or forward_from_type == 'chat': + # chat_forward_id = forward_from['from']['id'] + # chat_forward = Chat(chat_forward_id, self.get_chat_instance_uuid()) + # if chat_forward.exists(): + # for chat_obj in chat_objs: + # if chat_obj.type == 'chat': + # chat_forward.add_relationship(chat_obj.get_global_id(), 'forward') + # # chat_forward.add_relationship(obj.get_global_id(), 'forward') + # SENDER # TODO HANDLE NULL SENDER user_account = self.process_sender(new_objs, obj, date, timestamp) + if user_account: # UserAccount---ChatObjects - for obj_chat in chat_objs: - user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id) + for obj_chat in chat_objs: + user_account.add_correlation(obj_chat.type, obj_chat.get_subtype(r_str=True), obj_chat.id) # if chat: # TODO Chat---Username correlation ??? # # Chat---Username => need to handle members and participants diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index 040c3ea5..e0dffd9d 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -76,6 +76,7 @@ class Chat(AbstractChatObject): meta['tags'] = self.get_tags(r_list=True) if 'icon' in options: meta['icon'] = self.get_icon() + meta['img'] = meta['icon'] if 'info' in options: meta['info'] = self.get_info() if 'participants' in options: @@ -93,6 +94,8 @@ class Chat(AbstractChatObject): if 'threads' in options: meta['threads'] = self.get_threads() print(meta['threads']) + if 'tags_safe' in options: + meta['tags_safe'] = self.is_tags_safe(meta['tags']) return meta def get_misp_object(self): diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 2655c2ee..659047be 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -131,7 +131,7 @@ class Message(AbstractObject): if meta: _, user_account_subtype, user_account_id = user_account.split(':', 3) user_account = UsersAccount.UserAccount(user_account_id, user_account_subtype).get_meta(options={'icon', 'username', 'username_meta'}) - return user_account + return user_account def get_files_names(self): names = [] @@ -148,15 +148,32 @@ class Message(AbstractObject): def add_reaction(self, reactions, nb_reaction): r_object.hset(f'meta:reactions:{self.type}::{self.id}', reactions, nb_reaction) - # Update value on import - # reply to -> parent ? - # reply/comment - > children ? + # Interactions between users -> use replies # nb views - # reactions - # nb fowards - # room ??? - # message from channel ??? + # MENTIONS -> Messages + Chats + # # relationship -> mention - Chat -> Chat + # - Message -> Chat + # - Message -> Message ??? fetch mentioned messages + # FORWARDS + # TODO Create forward CHAT -> message + # message (is forwarded) -> message (is forwarded from) ??? + # # TODO get source message timestamp + # + # # is forwarded + # # forwarded from -> check if relationship + # # nb forwarded -> scard relationship + # + # Messages -> CHATS -> NB forwarded + # CHAT -> NB forwarded by chats -> NB messages -> parse full set ???? + # + # + # + # + # + # + # show users chats # message media + # flag is deleted -> event or missing from feeder pass ??? def get_translation(self, content=None, source=None, target='fr'): """ diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index d651761f..86eacc44 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -24,6 +24,7 @@ from lib.ConfigLoader import ConfigLoader from lib import Duplicate from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations +from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers logging.config.dictConfig(ail_logger.get_config(name='ail')) @@ -284,6 +285,22 @@ class AbstractObject(ABC): ## -Correlation- ## + ## Relationship ## + + def get_nb_relationships(self, filter=[]): + return get_obj_nb_relationships(self.get_global_id()) + + def add_relationship(self, obj2_global_id, relationship, source=True): + # is source + if source: + print(self.get_global_id(), obj2_global_id, relationship) + add_obj_relationship(self.get_global_id(), obj2_global_id, relationship) + # is target + else: + add_obj_relationship(obj2_global_id, self.get_global_id(), relationship) + + ## -Relationship- ## + ## Parent ## def is_parent(self): diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 1b01c2c1..15717c78 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -10,6 +10,7 @@ sys.path.append(os.environ['AIL_BIN']) from lib.ConfigLoader import ConfigLoader from lib.ail_core import get_all_objects, get_object_all_subtypes from lib import correlations_engine +from lib import relationships_engine from lib import btc_ail from lib import Tag @@ -468,6 +469,24 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_ # --- CORRELATION --- # +def get_obj_nb_relationships(obj_type, subtype, obj_id, filter_types=[]): + obj = get_object(obj_type, subtype, obj_id) + return obj.get_nb_relationships(filter=filter_types) + +def get_relationships_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, + objs_hidden=set(), + flask_context=False): + obj_global_id = get_obj_global_id(obj_type, subtype, obj_id) + nodes, links, meta = relationships_engine.get_relationship_graph(obj_global_id, + filter_types=filter_types, + max_nodes=max_nodes, level=level, + objs_hidden=objs_hidden) + # print(meta) + meta['objs'] = list(meta['objs']) + return {"nodes": create_correlation_graph_nodes(nodes, obj_global_id, flask_context=flask_context), + "links": links, + "meta": meta} + # if __name__ == '__main__': # r = get_objects([{'lvl': 1, 'type': 'item', 'subtype': '', 'id': 'crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'}]) diff --git a/bin/lib/relationships_engine.py b/bin/lib/relationships_engine.py new file mode 100755 index 00000000..6791214a --- /dev/null +++ b/bin/lib/relationships_engine.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader + +config_loader = ConfigLoader() +r_rel = config_loader.get_db_conn("Kvrocks_Relationships") +config_loader = None + + +RELATIONSHIPS = { + "forward", + "mention" +} +def get_relationships(): + return RELATIONSHIPS + + +def get_obj_relationships_by_type(obj_global_id, relationship): + return r_rel.smembers(f'rel:{relationship}:{obj_global_id}') + +def get_obj_nb_relationships_by_type(obj_global_id, relationship): + return r_rel.scard(f'rel:{relationship}:{obj_global_id}') + +def get_obj_relationships(obj_global_id): + relationships = [] + for relationship in get_relationships(): + for rel in get_obj_relationships_by_type(obj_global_id, relationship): + meta = {'relationship': relationship} + direction, obj_id = rel.split(':', 1) + if direction == 'i': + meta['source'] = obj_id + meta['target'] = obj_global_id + else: + meta['target'] = obj_id + meta['source'] = obj_global_id + + if not obj_id.startswith('chat'): + continue + + meta['id'] = obj_id + # meta['direction'] = direction + relationships.append(meta) + return relationships + +def get_obj_nb_relationships(obj_global_id): + nb = {} + for relationship in get_relationships(): + nb[relationship] = get_obj_nb_relationships_by_type(obj_global_id, relationship) + return nb + + +# TODO Filter by obj type ??? +def add_obj_relationship(source, target, relationship): + r_rel.sadd(f'rel:{relationship}:{source}', f'o:{target}') + r_rel.sadd(f'rel:{relationship}:{target}', f'i:{source}') + # r_rel.sadd(f'rels:{source}', relationship) + # r_rel.sadd(f'rels:{target}', relationship) + + +def get_relationship_graph(obj_global_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set()): + links = [] + nodes = set() + meta = {'complete': True, 'objs': set()} + done = set() + done_link = set() + + _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link) + return nodes, links, meta + +def _get_relationship_graph(obj_global_id, links, nodes, meta, level, max_nodes, filter_types=[], objs_hidden=set(), done=set(), done_link=set()): + meta['objs'].add(obj_global_id) + nodes.add(obj_global_id) + + for rel in get_obj_relationships(obj_global_id): + meta['objs'].add(rel['id']) + + if rel['id'] in done: + continue + + if len(nodes) > max_nodes != 0: + meta['complete'] = False + break + + nodes.add(rel['id']) + + str_link = f"{rel['source']}{rel['target']}{rel['relationship']}" + if str_link not in done_link: + links.append({"source": rel['source'], "target": rel['target'], "relationship": rel['relationship']}) + done_link.add(str_link) + + if level > 0: + next_level = level - 1 + + _get_relationship_graph(rel['id'], links, nodes, meta, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, done=done, done_link=done_link) + + # done.add(rel['id']) + + +if __name__ == '__main__': + source = '' + target = '' + add_obj_relationship(source, target, 'forward') + # print(get_obj_relationships(source)) diff --git a/configs/6383.conf b/configs/6383.conf index a06d4e69..dfc1f205 100644 --- a/configs/6383.conf +++ b/configs/6383.conf @@ -664,6 +664,7 @@ namespace.db ail_datas namespace.dup ail_dups namespace.obj ail_objs namespace.tl ail_tls +namespace.rel ail_rels namespace.stat ail_stats namespace.tag ail_tags namespace.track ail_trackers diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index 0e0b900f..f93ec08a 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -196,6 +196,11 @@ host = localhost port = 6383 password = ail_objs +[Kvrocks_Relationships] +host = localhost +port = 6383 +password = ail_rels + [Kvrocks_Timeline] host = localhost port = 6383 diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index 4cf9ce53..2b81d96f 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -203,7 +203,7 @@ def get_description(): return Response(json.dumps({"status": "error", "reason": "404 Not Found"}, indent=2, sort_keys=True), mimetype='application/json'), 404 # object exist else: - res = ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags', 'tags_safe'}, + res = ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'icon', 'tags', 'tags_safe'}, flask_context=True) if 'tags' in res: res['tags'] = list(res['tags']) @@ -292,3 +292,64 @@ def correlation_tags_add(): max_nodes=nb_max, hidden=hidden, hidden_str=",".join(hidden), filter=",".join(filter_types))) + +##################################################################################### + +@correlation.route('/relationships/graph_node_json') +@login_required +@login_read_only +def relationships_graph_node_json(): + obj_id = request.args.get('id') + subtype = request.args.get('subtype') + obj_type = request.args.get('type') + max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) + level = sanitise_level(request.args.get('level')) + + json_graph = ail_objects.get_relationships_graph_node(obj_type, subtype, obj_id, max_nodes=max_nodes, level=level, flask_context=True) + return jsonify(json_graph) + + +@correlation.route('/relationship/show', methods=['GET', 'POST']) +@login_required +@login_read_only +def show_relationship(): + if request.method == 'POST': + object_type = request.form.get('obj_type') + subtype = request.form.get('subtype') + obj_id = request.form.get('obj_id') + max_nodes = request.form.get('max_nb_nodes_in') + level = sanitise_level(request.form.get('level')) + + # redirect to keep history and bookmark + return redirect(url_for('correlation.show_relationship', type=object_type, subtype=subtype, id=obj_id, + max_nodes=max_nodes, level=level)) + + # request.method == 'GET' + else: + obj_type = request.args.get('type') + subtype = request.args.get('subtype', '') + obj_id = request.args.get('id') + max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) + level = sanitise_level(request.args.get('level')) + + # check if obj_id exist + if not ail_objects.exists_obj(obj_type, subtype, obj_id): + return abort(404) + # object exist + else: # TODO remove old dict key + dict_object = {"type": obj_type, + "id": obj_id, + "object_type": obj_type, + "max_nodes": max_nodes, "level": level, + "correlation_id": obj_id, + "metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags', 'info', 'icon', 'username'}, flask_context=True), + "nb_relation": ail_objects.get_obj_nb_relationships(obj_type, subtype, obj_id) + } + if subtype: + dict_object["subtype"] = subtype + dict_object["metadata"]['type_id'] = subtype + else: + dict_object["subtype"] = '' + dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id) + return render_template("show_relationship.html", dict_object=dict_object, bootstrap_label=bootstrap_label, + tags_selector_data=Tag.get_tags_selector_data()) diff --git a/var/www/templates/correlation/metadata_card_chat.html b/var/www/templates/correlation/metadata_card_chat.html new file mode 100644 index 00000000..4e672e1a --- /dev/null +++ b/var/www/templates/correlation/metadata_card_chat.html @@ -0,0 +1,78 @@ + + + +{#{% with modal_add_tags=dict_object['metadata_card']['add_tags_modal']%}#} +{# {% include 'modals/add_tags.html' %}#} +{#{% endwith %}#} + +{% include 'modals/edit_tag.html' %} + +
    +
    +

    {{ dict_object["correlation_id"] }}

    + {{ dict_object }} +
    {{ dict_object["correlation_id"] }}
    +
      +
    • +
      +
      + + + + + + + + + + + + + + + + + +
      Object typeFirst seenLast seenNb seen
      + + + + {{ dict_object["metadata"]["icon"]["icon"] }} + + + {{ dict_object["object_type"] }} + {{ dict_object["metadata"]['first_seen'] }}{{ dict_object["metadata"]['last_seen'] }}{{ dict_object["metadata"]['nb_seen'] }}
      +
      +
      +
      +
      +
      +
      • + +
    • +
      +
      + Tags: + {% for tag in dict_object["metadata"]['tags'] %} + + {% endfor %} + +
      +
      • +
    + + {% with obj_type='cookie-name', obj_id=dict_object['correlation_id'], obj_subtype='' %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + +
    +
    + diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index cda58f1c..4005e13c 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -541,6 +541,15 @@ d3.json(url) .on("drag", dragged) .on("end", drag_end)); + /* + node.append("image") + .attr("xlink:href", "https://circl.lu/assets/images/circl-logo.png") + .attr("height", 20) + .attr("width", 20) + .attr("x", -10) + .attr("y", -10); + + */ node.append("circle") .attr("r", function(d) { diff --git a/var/www/templates/correlation/show_relationship.html b/var/www/templates/correlation/show_relationship.html new file mode 100644 index 00000000..bff41724 --- /dev/null +++ b/var/www/templates/correlation/show_relationship.html @@ -0,0 +1,719 @@ + + + + + + + AIL - framework + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + + {% if dict_object["object_type"] == "pgp" %} + {% include 'correlation/metadata_card_pgp.html' %} + {% elif dict_object["object_type"] == "cryptocurrency" %} + {% include 'correlation/metadata_card_cryptocurrency.html' %} + {% elif dict_object["object_type"] == "username" %} + {% include 'correlation/metadata_card_username.html' %} + {% elif dict_object["object_type"] == "decoded" %} + {% include 'correlation/metadata_card_decoded.html' %} + {% elif dict_object["object_type"] == "chat" %} + {% include 'correlation/metadata_card_chat.html' %} + {% elif dict_object["object_type"] == "cve" %} + {% include 'correlation/metadata_card_cve.html' %} + {% elif dict_object["object_type"] == "domain" %} + {% include 'correlation/metadata_card_domain.html' %} + {% elif dict_object["object_type"] == "screenshot" %} + {% include 'correlation/metadata_card_screenshot.html' %} + {% elif dict_object["object_type"] == "title" %} + {% include 'correlation/metadata_card_title.html' %} + {% elif dict_object["object_type"] == "cookie-name" %} + {% include 'correlation/metadata_card_cookie_name.html' %} + {% elif dict_object["object_type"] == "etag" %} + {% include 'correlation/metadata_card_etag.html' %} + {% elif dict_object["object_type"] == "hhhash" %} + {% include 'correlation/metadata_card_hhhash.html' %} + {% elif dict_object["object_type"] == "item" %} + {% include 'correlation/metadata_card_item.html' %} + {% endif %} + +
    +
    + +
    +
    + Graph + +
    +
    +
    +
    + +
    +
    + +
    +
    + +
    +
    +
    +
    +     +{# #} +{# {% if dict_object["object_type"] != "" %}#} +{# {% with obj_type=dict_object["object_type"], obj_id=dict_object["correlation_id"], obj_subtype=dict_object["metadata"]["type_id"],obj_lvl=1%}#} +{# {% include 'import_export/block_add_user_object_to_export.html' %}#} +{# {% endwith %}#} +{# {% endif %}#} +{# #} + + + + + + +
    +  Graph Incomplete, Max Nodes Reached. +
    +
    +
    +
    + + Loading... +
    +
    +
    +
    +
    + +{#

    Press H on an object / node to hide it.

    #} +{# {% if dict_object["hidden"] %}#} +{#
    Hidden objects:
    #} +{# {% for obj_hidden in dict_object["hidden"] %}#} +{# {{ obj_hidden }}
    #} +{# {% endfor %}#} +{# {% endif %}#} + +
    + +
    + +
    +
    + +
      +
    • Relationship
      • +
      + + + + + +{#
    • #} +{#
      #} +{# #} +{# #} +{#
      #} +{#
      #} +{# #} +{# #} +{#
      #} +{##} +{#
      • #} +
    • + +
      + + +
      + + +
      • +
    • + +
      + + +
      + +
      + +
      + + +
      • +
      +
    + +
      +
      • +
    • +

      Double click on a node to open this object

      + + + + + + Current Object
      +

      +
      • +
    +
      +
    • Direct Relationships
      • +
    • + {% for relationship in dict_object['nb_relation'] %} +
      +
      + {{ relationship }} +
      +
      + {{ dict_object['nb_relation'][relationship] }} +
      +
      + {% endfor %} +
      • +
    + +
    +
    +
    +
    + +
    +
    + + {% include 'correlation/legend_graph_correlation.html' %} + +
    +
    + + + +
    +
    +

    Tags All Objects

    +
    +
    +
    + + + + + + + + {% include 'tags/block_tags_selector.html' %} + +
    +
    +
    + +
    +
    +
    + + + + + + + + + + From 74e41017a1afb0157eeadcbcc38f1377c06c9191 Mon Sep 17 00:00:00 2001 From: terrtia Date: Fri, 26 Jan 2024 15:55:19 +0100 Subject: [PATCH 208/238] chg: [v5.3] add v5.3 update --- update/v5.3/Update.py | 24 ++++++++++++++++++++++++ update/v5.3/Update.sh | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100755 update/v5.3/Update.py create mode 100755 update/v5.3/Update.sh diff --git a/update/v5.3/Update.py b/update/v5.3/Update.py new file mode 100755 index 00000000..20eed0af --- /dev/null +++ b/update/v5.3/Update.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys + +sys.path.append(os.environ['AIL_HOME']) +################################## +# Import Project packages +################################## +from update.bin.ail_updater import AIL_Updater +from lib import ail_updates + +class Updater(AIL_Updater): + """default Updater.""" + + def __init__(self, version): + super(Updater, self).__init__(version) + + +if __name__ == '__main__': + updater = Updater('v5.3') + updater.run_update() + diff --git a/update/v5.3/Update.sh b/update/v5.3/Update.sh new file mode 100755 index 00000000..1e040200 --- /dev/null +++ b/update/v5.3/Update.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; + +export PATH=$AIL_HOME:$PATH +export PATH=$AIL_REDIS:$PATH +export PATH=$AIL_BIN:$PATH +export PATH=$AIL_FLASK:$PATH + +GREEN="\\033[1;32m" +DEFAULT="\\033[0;39m" + +echo -e $GREEN"Shutting down AIL ..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -ks +wait + +# SUBMODULES # +git submodule update + +echo "" +echo -e $GREEN"Updating python packages ..."$DEFAULT +echo "" +pip install -U gcld3 +pip install -U libretranslatepy +pip install -U xxhash +pip install -U DomainClassifier + +echo "" +echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT +echo "" +python ${AIL_HOME}/update/v5.3/Update.py +wait +echo "" +echo "" + +exit 0 From 61bccecdab60adb7e20c574bd90ac3c10efd4f5a Mon Sep 17 00:00:00 2001 From: terrtia Date: Fri, 26 Jan 2024 16:06:42 +0100 Subject: [PATCH 209/238] chg: [chats] show NB messages by chat --- bin/lib/chats_viewer.py | 2 +- bin/lib/objects/Chats.py | 2 ++ var/www/blueprints/chats_explorer.py | 2 +- var/www/templates/chats_explorer/chat_instance.html | 4 +++- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 3c46db5f..69b913f4 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -178,7 +178,7 @@ class ChatServiceInstance: if 'chats' in options: meta['chats'] = [] for chat_id in self.get_chats(): - meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'created_at', 'icon', 'nb_subchannels'})) + meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'created_at', 'icon', 'nb_subchannels', 'nb_messages'})) return meta def get_nb_chats(self): diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index e0dffd9d..746f0dea 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -83,6 +83,8 @@ class Chat(AbstractChatObject): meta['participants'] = self.get_participants() if 'nb_participants' in options: meta['nb_participants'] = self.get_nb_participants() + if 'nb_messages' in options: + meta['nb_messages'] = self.get_nb_messages() if 'username' in options: meta['username'] = self.get_username() if 'subchannels' in options: diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index eb817da3..38d6c413 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -63,7 +63,7 @@ def chats_explorer_networks(): return render_template('chats_networks.html', protocol=protocol, networks=networks) -@chats_explorer.route("chats/explorer/instance", methods=['GET']) +@chats_explorer.route("chats/explorer/instances", methods=['GET']) @login_required @login_read_only def chats_explorer_instance(): diff --git a/var/www/templates/chats_explorer/chat_instance.html b/var/www/templates/chats_explorer/chat_instance.html index 557ade0a..0f5e25ed 100644 --- a/var/www/templates/chats_explorer/chat_instance.html +++ b/var/www/templates/chats_explorer/chat_instance.html @@ -70,7 +70,8 @@ Created at First Seen Last Seen - NB SubChannels + SubChannels + Messages @@ -94,6 +95,7 @@ {% endif %} {{ chat['nb_subchannels'] }} + {{ chat['nb_messages'] }} {% endfor %} From 6a24c58c8be15ed8b27d0a86f05fe2fd8b41e3a1 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 29 Jan 2024 10:30:53 +0100 Subject: [PATCH 210/238] fix: [heatmap] fix tooltip position --- var/www/static/js/d3/heatmap_week_hour.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/var/www/static/js/d3/heatmap_week_hour.js b/var/www/static/js/d3/heatmap_week_hour.js index f41e8985..4696a052 100644 --- a/var/www/static/js/d3/heatmap_week_hour.js +++ b/var/www/static/js/d3/heatmap_week_hour.js @@ -50,7 +50,12 @@ const create_heatmap_week_hour = (container_id, data, options) => { .style("stroke", "black") //.style("stroke-opacity", 1) + var xPosition = d3.mouse(this)[0] + margin.left; + var yPosition = d3.mouse(this)[1] + margin.top + window.scrollY + 100; + tooltip.html(d.date + " " + d.hour + "-" + (d.hour + 1) + "h: " + d.count + " messages") + .style("left", xPosition + "px") + .style("top", yPosition + "px"); } const mouseleave = function(d) { tooltip.style("opacity", 0) From 6363a4f1cfb18e036c1773c3bde4f8759aa194a9 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 29 Jan 2024 10:52:18 +0100 Subject: [PATCH 211/238] fix: [chat view] fix created_at + filter --- bin/lib/chats_viewer.py | 6 +----- bin/lib/objects/ChatSubChannels.py | 5 +++-- bin/lib/objects/Chats.py | 1 - .../chats_explorer/block_obj_time_search.html | 10 +++++----- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 69b913f4..a8346bac 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -280,7 +280,6 @@ def create_chat_service_instance(protocol, network=None, address=None): ####################################################################################### def get_obj_chat(chat_type, chat_subtype, chat_id): - print(chat_type, chat_subtype, chat_id) if chat_type == 'chat': return Chats.Chat(chat_id, chat_subtype) elif chat_type == 'chat-subchannel': @@ -305,7 +304,7 @@ def get_subchannels_meta_from_global_id(subchannels): for sub in subchannels: _, instance_uuid, sub_id = sub.split(':', 2) subchannel = ChatSubChannels.ChatSubChannel(sub_id, instance_uuid) - meta.append(subchannel.get_meta({'nb_messages'})) + meta.append(subchannel.get_meta({'nb_messages', 'created_at', 'icon'})) return meta def get_chat_meta_from_global_id(chat_global_id): @@ -399,9 +398,6 @@ def api_get_message(message_id, translation_target=None): if not message.exists(): return {"status": "error", "reason": "Unknown uuid"}, 404 meta = message.get_meta({'chat', 'content', 'files-names', 'icon', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'}, translation_target=translation_target) - # if meta['chat']: - # print(meta['chat']) - # # meta['chat'] = return meta, 200 def api_get_user_account(user_id, instance_uuid): diff --git a/bin/lib/objects/ChatSubChannels.py b/bin/lib/objects/ChatSubChannels.py index 7a799240..ef343baa 100755 --- a/bin/lib/objects/ChatSubChannels.py +++ b/bin/lib/objects/ChatSubChannels.py @@ -82,8 +82,9 @@ class ChatSubChannel(AbstractChatObject): meta['name'] = self.get_name() if 'chat' in options: meta['chat'] = self.get_chat() - if 'img' in options: - meta['img'] = self.get_img() + if 'icon' in options: + meta['icon'] = self.get_icon() + meta['img'] = meta['icon'] if 'nb_messages' in options: meta['nb_messages'] = self.get_nb_messages() if 'created_at' in options: diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index 746f0dea..c894cc26 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -95,7 +95,6 @@ class Chat(AbstractChatObject): meta['created_at'] = self.get_created_at(date=True) if 'threads' in options: meta['threads'] = self.get_threads() - print(meta['threads']) if 'tags_safe' in options: meta['tags_safe'] = self.is_tags_safe(meta['tags']) return meta diff --git a/var/www/templates/chats_explorer/block_obj_time_search.html b/var/www/templates/chats_explorer/block_obj_time_search.html index 515a8ea7..60401d42 100644 --- a/var/www/templates/chats_explorer/block_obj_time_search.html +++ b/var/www/templates/chats_explorer/block_obj_time_search.html @@ -39,9 +39,9 @@
    - - - +{##} +{##} +{##} From 896b411eafbb437039c3319c0c6621d7a7a2fc1a Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 29 Jan 2024 14:36:53 +0100 Subject: [PATCH 212/238] chg: [translation] translate chats info, users info and subchannels names --- bin/lib/Language.py | 20 +++++++++++++++++ bin/lib/chats_viewer.py | 16 +++++++------- bin/lib/objects/ChatSubChannels.py | 4 +++- bin/lib/objects/Chats.py | 4 +++- bin/lib/objects/Messages.py | 3 ++- bin/lib/objects/UsersAccount.py | 14 +++++++++++- bin/lib/objects/abstract_object.py | 11 ++++++++++ var/www/blueprints/chats_explorer.py | 9 ++++++-- .../chats_explorer/SubChannelMessages.html | 3 +++ .../chats_explorer/chat_instance.html | 2 +- .../templates/chats_explorer/chat_viewer.html | 22 ++++++++++++++----- .../chats_explorer/user_account.html | 10 +++++++-- 12 files changed, 95 insertions(+), 23 deletions(-) diff --git a/bin/lib/Language.py b/bin/lib/Language.py index 2b9bcc9b..1fee96f7 100755 --- a/bin/lib/Language.py +++ b/bin/lib/Language.py @@ -16,6 +16,7 @@ sys.path.append(os.environ['AIL_BIN']) from lib.ConfigLoader import ConfigLoader config_loader = ConfigLoader() +r_cache = config_loader.get_redis_conn("Redis_Cache") TRANSLATOR_URL = config_loader.get_config_str('Translation', 'libretranslate') config_loader = None @@ -298,6 +299,25 @@ def _clean_text_to_translate(content, html=False, keys_blocks=True): content = content.replace(it, '') return content +#### AIL Objects #### + +def get_obj_translation(obj_global_id, content, field='', source=None, target='en'): + """ + Returns translated content + """ + translation = r_cache.get(f'translation:{target}:{obj_global_id}:{field}') + if translation: + # DEBUG + # print('cache') + # r_cache.expire(f'translation:{target}:{obj_global_id}:{field}', 0) + return translation + translation = LanguageTranslator().translate(content, source=source, target=target) + if translation: + r_cache.set(f'translation:{target}:{obj_global_id}:{field}', translation) + r_cache.expire(f'translation:{target}:{obj_global_id}:{field}', 300) + return translation + +## --AIL Objects-- ## class LanguagesDetector: diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index a8346bac..797a9ed8 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -299,12 +299,12 @@ def get_obj_chat_meta(obj_chat, new_options=set()): options.add(option) return obj_chat.get_meta(options=options) -def get_subchannels_meta_from_global_id(subchannels): +def get_subchannels_meta_from_global_id(subchannels, translation_target=None): meta = [] for sub in subchannels: _, instance_uuid, sub_id = sub.split(':', 2) subchannel = ChatSubChannels.ChatSubChannel(sub_id, instance_uuid) - meta.append(subchannel.get_meta({'nb_messages', 'created_at', 'icon'})) + meta.append(subchannel.get_meta({'nb_messages', 'created_at', 'icon', 'translation'}, translation_target=translation_target)) return meta def get_chat_meta_from_global_id(chat_global_id): @@ -335,13 +335,13 @@ def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, pa chat = Chats.Chat(chat_id, chat_instance_uuid) if not chat.exists(): return {"status": "error", "reason": "Unknown chat"}, 404 - meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'username'}) + meta = chat.get_meta({'created_at', 'icon', 'info', 'nb_participants', 'subchannels', 'threads', 'translation', 'username'}, translation_target=translation_target) if meta['username']: meta['username'] = get_username_meta_from_global_id(meta['username']) if meta['subchannels']: - meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels']) + meta['subchannels'] = get_subchannels_meta_from_global_id(meta['subchannels'], translation_target=translation_target) else: - if translation_target not in Language.LIST_LANGUAGES: + if translation_target not in Language.get_translation_languages(): translation_target = None meta['messages'], meta['pagination'], meta['tags_messages'] = chat.get_messages(translation_target=translation_target, nb=nb, page=page) return meta, 200 @@ -373,7 +373,7 @@ def api_get_subchannel(chat_id, chat_instance_uuid, translation_target=None, nb= subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid) if not subchannel.exists(): return {"status": "error", "reason": "Unknown subchannel"}, 404 - meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages', 'nb_participants', 'threads'}) + meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages', 'nb_participants', 'threads', 'translation'}, translation_target=translation_target) if meta['chat']: meta['chat'] = get_chat_meta_from_global_id(meta['chat']) if meta.get('threads'): @@ -400,11 +400,11 @@ def api_get_message(message_id, translation_target=None): meta = message.get_meta({'chat', 'content', 'files-names', 'icon', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'thread', 'translation', 'user-account'}, translation_target=translation_target) return meta, 200 -def api_get_user_account(user_id, instance_uuid): +def api_get_user_account(user_id, instance_uuid, translation_target=None): user_account = UsersAccount.UserAccount(user_id, instance_uuid) if not user_account.exists(): return {"status": "error", "reason": "Unknown user-account"}, 404 - meta = user_account.get_meta({'chats', 'icon', 'info', 'subchannels', 'threads', 'username', 'username_meta'}) + meta = user_account.get_meta({'chats', 'icon', 'info', 'subchannels', 'threads', 'translation', 'username', 'username_meta'}, translation_target=translation_target) return meta, 200 # # # # # # # # # # LATER diff --git a/bin/lib/objects/ChatSubChannels.py b/bin/lib/objects/ChatSubChannels.py index ef343baa..331c29ea 100755 --- a/bin/lib/objects/ChatSubChannels.py +++ b/bin/lib/objects/ChatSubChannels.py @@ -76,7 +76,7 @@ class ChatSubChannel(AbstractChatObject): # TODO TIME LAST MESSAGES - def get_meta(self, options=set()): + def get_meta(self, options=set(), translation_target=None): meta = self._get_meta(options=options) meta['tags'] = self.get_tags(r_list=True) meta['name'] = self.get_name() @@ -95,6 +95,8 @@ class ChatSubChannel(AbstractChatObject): meta['participants'] = self.get_participants() if 'nb_participants' in options: meta['nb_participants'] = self.get_nb_participants() + if 'translation' in options and translation_target: + meta['translation_name'] = self.translate(meta['name'], field='name', target=translation_target) return meta def get_misp_object(self): diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index c894cc26..dde776b0 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -70,7 +70,7 @@ class Chat(AbstractChatObject): icon = '\uf086' return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5} - def get_meta(self, options=set()): + def get_meta(self, options=set(), translation_target=None): meta = self._get_meta(options=options) meta['name'] = self.get_name() meta['tags'] = self.get_tags(r_list=True) @@ -79,6 +79,8 @@ class Chat(AbstractChatObject): meta['img'] = meta['icon'] if 'info' in options: meta['info'] = self.get_info() + if 'translation' in options and translation_target: + meta['translation_info'] = self.translate(meta['info'], field='info', target=translation_target) if 'participants' in options: meta['participants'] = self.get_participants() if 'nb_participants' in options: diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 659047be..a88eb6da 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -179,6 +179,7 @@ class Message(AbstractObject): """ Returns translated content """ + # return self._get_field('translated') global_id = self.get_global_id() translation = r_cache.get(f'translation:{target}:{global_id}') @@ -289,7 +290,7 @@ class Message(AbstractObject): if 'reactions' in options: meta['reactions'] = self.get_reactions() if 'translation' in options and translation_target: - meta['translation'] = self.get_translation(content=meta.get('content'), target=translation_target) + meta['translation'] = self.translate(content=meta.get('content'), target=translation_target) # meta['encoding'] = None return meta diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py index 2148697a..27dbf30c 100755 --- a/bin/lib/objects/UsersAccount.py +++ b/bin/lib/objects/UsersAccount.py @@ -5,6 +5,7 @@ import os import sys # import re +# from datetime import datetime from flask import url_for from pymisp import MISPObject @@ -88,6 +89,13 @@ class UserAccount(AbstractSubtypeObject): def set_info(self, info): return self._set_field('info', info) + # def get_created_at(self, date=False): + # created_at = self._get_field('created_at') + # if date and created_at: + # created_at = datetime.fromtimestamp(float(created_at)) + # created_at = created_at.isoformat(' ') + # return created_at + # TODO MESSAGES: # 1) ALL MESSAGES + NB # 2) ALL MESSAGES TIMESTAMP @@ -122,7 +130,7 @@ class UserAccount(AbstractSubtypeObject): def get_messages_by_chat_obj(self, chat_obj): return self.get_correlation_iter_obj(chat_obj, 'message') - def get_meta(self, options=set()): # TODO Username timeline + def get_meta(self, options=set(), translation_target=None): # TODO Username timeline meta = self._get_meta(options=options) meta['id'] = self.id meta['subtype'] = self.subtype @@ -141,6 +149,10 @@ class UserAccount(AbstractSubtypeObject): meta['icon'] = self.get_icon() if 'info' in options: meta['info'] = self.get_info() + if 'translation' in options and translation_target: + meta['translation_info'] = self.translate(meta['info'], field='info', target=translation_target) + # if 'created_at': + # meta['created_at'] = self.get_created_at(date=True) if 'chats' in options: meta['chats'] = self.get_chats() if 'subchannels' in options: diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index 86eacc44..64697b1e 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -25,6 +25,7 @@ from lib import Duplicate from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type, get_obj_inter_correlation from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations from lib.relationships_engine import get_obj_nb_relationships, add_obj_relationship +from lib.Language import get_obj_translation from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers logging.config.dictConfig(ail_logger.get_config(name='ail')) @@ -301,6 +302,16 @@ class AbstractObject(ABC): ## -Relationship- ## + ## Translation ## + + def translate(self, content=None, field='', source=None, target='en'): + global_id = self.get_global_id() + if not content: + content = self.get_content() + return get_obj_translation(global_id, content, field=field, source=source, target=target) + + ## -Translation- ## + ## Parent ## def is_parent(self): diff --git a/var/www/blueprints/chats_explorer.py b/var/www/blueprints/chats_explorer.py index 38d6c413..ef385b44 100644 --- a/var/www/blueprints/chats_explorer.py +++ b/var/www/blueprints/chats_explorer.py @@ -182,9 +182,14 @@ def objects_message(): def objects_user_account(): instance_uuid = request.args.get('subtype') user_id = request.args.get('id') - user_account = chats_viewer.api_get_user_account(user_id, instance_uuid) + target = request.args.get('target') + if target == "Don't Translate": + target = None + user_account = chats_viewer.api_get_user_account(user_id, instance_uuid, translation_target=target) if user_account[1] != 200: return create_json_response(user_account[0], user_account[1]) else: user_account = user_account[0] - return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label) \ No newline at end of file + languages = Language.get_translation_languages() + return render_template('user_account.html', meta=user_account, bootstrap_label=bootstrap_label, + translation_languages=languages, translation_target=target) diff --git a/var/www/templates/chats_explorer/SubChannelMessages.html b/var/www/templates/chats_explorer/SubChannelMessages.html index 858a1159..5eb6153b 100644 --- a/var/www/templates/chats_explorer/SubChannelMessages.html +++ b/var/www/templates/chats_explorer/SubChannelMessages.html @@ -75,6 +75,9 @@ {{ subchannel['name'] }} + {% if subchannel['translation_name'] %} +
    {{ subchannel['translation_name'] }}
    + {% endif %} {{ subchannel["created_at"] }} diff --git a/var/www/templates/chats_explorer/chat_instance.html b/var/www/templates/chats_explorer/chat_instance.html index 0f5e25ed..70524e40 100644 --- a/var/www/templates/chats_explorer/chat_instance.html +++ b/var/www/templates/chats_explorer/chat_instance.html @@ -71,7 +71,7 @@ First Seen Last Seen SubChannels - Messages + diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html index 64f14295..0506d444 100644 --- a/var/www/templates/chats_explorer/chat_viewer.html +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -100,6 +100,10 @@ {% if chat['info'] %} 
  • {{ chat['info'] }}
    + {% if chat['translation_info'] %} +
    + 
    {{ chat['translation_info'] }}
    + {% endif %}
    • {% endif %} @@ -112,8 +116,12 @@ {{ tag }} {{ chat['tags_messages'][tag] }} {% endfor %} + {% with translate_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), obj_id=chat['id'], pagination=chat['pagination'] %} + {% include 'chats_explorer/block_translation.html' %} + {% endwith %} + {% if chat['subchannels'] %} -

    Sub-Channels:

    +

    Sub-Channels:

    @@ -123,7 +131,7 @@ - + @@ -132,7 +140,12 @@ - + - @@ -56,7 +55,6 @@ - + {%if ail_version is not none %} + {%else%} + + {%endif%} + {%if ail_version is not none %} + {%else%} + + {%endif%} Date: Fri, 2 Feb 2024 11:15:08 +0100 Subject: [PATCH 222/238] chg: [tags] add Tag class --- bin/lib/Tag.py | 142 +++++++++++++++++- bin/lib/objects/UsersAccount.py | 5 +- .../chats_explorer/chat_participants.html | 56 +------ 3 files changed, 146 insertions(+), 57 deletions(-) diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index 6d1f91a9..9140a583 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -685,7 +685,7 @@ def confirm_tag(tag, obj): def update_tag_global_by_obj_type(tag, obj_type, subtype=''): tag_deleted = False if obj_type == 'item' or obj_type == 'message': - if not r_tags.exists(f'tag_metadata:{tag}'): + if not r_tags.exists(f'tag_metadata:{tag}'): # TODO FIXME ################################################################# tag_deleted = True else: if not r_tags.exists(f'{obj_type}:{subtype}:{tag}'): @@ -1200,12 +1200,17 @@ def get_enabled_tags_with_synonyms_ui(): # TYPE -> taxonomy/galaxy/custom +# TODO GET OBJ Types class Tag: def __int__(self, name: str, local=False): # TODO Get first seen by object, obj='item self.name = name self.local = local + # TODO + def exists(self): + pass + def is_local(self): return self.local @@ -1216,7 +1221,11 @@ class Tag: else: return 'taxonomy' + def is_taxonomy(self): + return not self.local and self.is_galaxy() + def is_galaxy(self): + return not self.local and self.name.startswith('misp-galaxy:') def get_first_seen(self, r_int=False): first_seen = r_tags.hget(f'meta:tag:{self.name}', 'first_seen') @@ -1227,6 +1236,9 @@ class Tag: first_seen = 99999999 return first_seen + def set_first_seen(self, first_seen): + return r_tags.hget(f'meta:tag:{self.name}', 'first_seen', int(first_seen)) + def get_last_seen(self, r_int=False): last_seen = r_tags.hget(f'meta:tag:{self.name}', 'last_seen') # 'last_seen:object' -> only if date or daterange if r_int: @@ -1236,6 +1248,9 @@ class Tag: last_seen = 0 return last_seen + def set_last_seen(self, last_seen): + return r_tags.hset(f'meta:tag:{self.name}', 'last_seen', int(last_seen) + def get_color(self): color = r_tags.hget(f'meta:tag:{self.name}', 'color') if not color: @@ -1258,6 +1273,131 @@ class Tag: 'local': self.is_local()} return meta + def update_obj_type_first_seen(self, obj_type, first_seen, last_seen): # TODO SUBTYPE ################################## + if int(first_seen) > int(last_seen): + raise Exception(f'INVALID first_seen/last_seen, {first_seen}/{last_seen}') + + for date in Date.get_daterange(first_seen, last_seen): + date = int(date) + if date == last_seen: + if r_tags.scard(f'{obj_type}::{self.name}:{first_seen}') > 0: + r_tags.hset(f'tag_metadata:{self.name}', 'first_seen', first_seen) + else: + r_tags.hdel(f'tag_metadata:{self.name}', 'first_seen') # TODO SUBTYPE + r_tags.hdel(f'tag_metadata:{self.name}', 'last_seen') # TODO SUBTYPE + r_tags.srem(f'list_tags:{obj_type}', self.name) # TODO SUBTYPE + + elif r_tags.scard(f'{obj_type}::{self.name}:{first_seen}') > 0: + r_tags.hset(f'tag_metadata:{self.name}', 'first_seen', first_seen) # TODO METADATA OBJECT NAME + + + def update_obj_type_last_seen(self, obj_type, first_seen, last_seen): # TODO SUBTYPE ################################## + if int(first_seen) > int(last_seen): + raise Exception(f'INVALID first_seen/last_seen, {first_seen}/{last_seen}') + + for date in Date.get_daterange(first_seen, last_seen).reverse(): + date = int(date) + if date == last_seen: + if r_tags.scard(f'{obj_type}::{self.name}:{last_seen}') > 0: + r_tags.hset(f'tag_metadata:{self.name}', 'last_seen', last_seen) + else: + r_tags.hdel(f'tag_metadata:{self.name}', 'first_seen') # TODO SUBTYPE + r_tags.hdel(f'tag_metadata:{self.name}', 'last_seen') # TODO SUBTYPE + r_tags.srem(f'list_tags:{obj_type}', self.name) # TODO SUBTYPE + + elif r_tags.scard(f'{obj_type}::{self.name}:{last_seen}') > 0: + r_tags.hset(f'tag_metadata:{self.name}', 'last_seen', last_seen) # TODO METADATA OBJECT NAME + + # TODO + # TODO Update First seen and last seen + # TODO SUBTYPE CHATS ?????????????? + def update_obj_type_date(self, obj_type, date, op='add', first_seen=None, last_seen=None): + date = int(date) + if not first_seen: + first_seen = self.get_first_seen(r_int=True) + if not last_seen: + last_seen = self.get_last_seen(r_int=True) + + # Add tag + if op == 'add': + if date < first_seen: + self.set_first_seen(date) + if date > last_seen: + self.set_last_seen(date) + + # Delete tag + else: + if date == first_seen and date == last_seen: + + # TODO OBJECTS ############################################################################################## + if r_tags.scard(f'{obj_type}::{self.name}:{first_seen}') < 1: ####################### TODO OBJ SUBTYPE ??????????????????? + r_tags.hdel(f'tag_metadata:{self.name}', 'first_seen') + r_tags.hdel(f'tag_metadata:{self.name}', 'last_seen') + # TODO CHECK IF DELETE FULL TAG LIST ############################ + + elif date == first_seen: + if r_tags.scard(f'{obj_type}::{self.name}:{first_seen}') < 1: + if int(last_seen) >= int(first_seen): + self.update_obj_type_first_seen(obj_type, first_seen, last_seen) # TODO OBJ_TYPE + + elif date == last_seen: + if r_tags.scard(f'{obj_type}::{self.name}:{last_seen}') < 1: + if int(last_seen) >= int(first_seen): + self.update_obj_type_last_seen(obj_type, first_seen, last_seen) # TODO OBJ_TYPE + + # STATS + nb = r_tags.hincrby(f'daily_tags:{date}', self.name, -1) + if nb < 1: + r_tags.hdel(f'daily_tags:{date}', self.name) + + # TODO -> CHECK IF TAG EXISTS + UPDATE FIRST SEEN/LAST SEEN + def update(self, date=None): + pass + + # TODO CHANGE ME TO SUB FUNCTION ##### add_object_tag(tag, obj_type, obj_id, subtype='') + def add(self, obj_type, subtype, obj_id): + if subtype is None: + subtype = '' + + if r_tags.sadd(f'tag:{obj_type}:{subtype}:{obj_id}', self.name) == 1: + r_tags.sadd('list_tags', self.name) + r_tags.sadd(f'list_tags:{obj_type}', self.name) + if subtype: + r_tags.sadd(f'list_tags:{obj_type}:{subtype}', self.name) + + if obj_type == 'item': + date = item_basic.get_item_date(obj_id) + + # add domain tag + if item_basic.is_crawled(obj_id) and self.name != 'infoleak:submission="crawler"' and self.name != 'infoleak:submission="manual"': + domain = item_basic.get_item_domain(obj_id) + self.add('domain', '', domain) + elif obj_type == 'message': + timestamp = obj_id.split('/')[1] + date = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y%m%d') + else: + date = None + + if date: + r_tags.sadd(f'{obj_type}:{subtype}:{self.name}:{date}', obj_id) + update_tag_metadata(self.name, date) + else: + r_tags.sadd(f'{obj_type}:{subtype}:{self.name}', obj_id) + + # TODO REPLACE ME BY DATE TAGS ???? + # STATS BY TYPE ??? + # DAILY STATS + r_tags.hincrby(f'daily_tags:{datetime.date.today().strftime("%Y%m%d")}', self.name, 1) + + + # TODO CREATE FUNCTION GET OBJECT DATE + def remove(self, obj_type, subtype, obj_id): + # TODO CHECK IN ALL OBJECT TO DELETE + pass + + def delete(self): + pass + #### TAG AUTO PUSH #### diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py index 27dbf30c..5777bd17 100755 --- a/bin/lib/objects/UsersAccount.py +++ b/bin/lib/objects/UsersAccount.py @@ -128,7 +128,10 @@ class UserAccount(AbstractSubtypeObject): self._get_timeline_username().add_timestamp(timestamp, username_global_id) def get_messages_by_chat_obj(self, chat_obj): - return self.get_correlation_iter_obj(chat_obj, 'message') + messages = [] + for mess in self.get_correlation_iter_obj(chat_obj, 'message'): + messages.append(f'message:{mess}') + return messages def get_meta(self, options=set(), translation_target=None): # TODO Username timeline meta = self._get_meta(options=options) diff --git a/var/www/templates/chats_explorer/chat_participants.html b/var/www/templates/chats_explorer/chat_participants.html index b8e367b0..bb732e56 100644 --- a/var/www/templates/chats_explorer/chat_participants.html +++ b/var/www/templates/chats_explorer/chat_participants.html @@ -31,61 +31,7 @@
    -{#
    #} TODO CHAT abstract metadata -{##} -{#
    #} -{#

    {% if chat['username'] %}{{ chat["username"]["id"] }} {% else %} {{ chat['name'] }}{% endif %} :

    #} -{# {% if chat['icon'] %}#} -{#
    {{ chat['id'] }}
    #} -{# {% endif %}#} -{#
      #} -{#
    • #} -{#
    Created at First Seen Last SeenNB Messages
    {{ meta['id'] }} {{ meta['name'] }} + {{ meta['name'] }} + {% if meta['translation_name'] %} +
    {{ meta['translation_name'] }}
    + {% endif %} +     		{{ meta['id'] }} {{ meta['created_at'] }} @@ -161,9 +174,6 @@ {% include 'objects/image/block_blur_img_slider.html' %} - {% with translate_url=url_for('chats_explorer.chats_explorer_chat', uuid=chat['subtype']), obj_id=chat['id'], pagination=chat['pagination'] %} - {% include 'chats_explorer/block_translation.html' %} - {% endwith %} {% with obj_subtype=chat['subtype'], obj_id=chat['id'], url_endpoint=url_for("chats_explorer.chats_explorer_chat"), nb=chat['pagination']['nb'] %} {% set date_from=chat['first_seen'] %} {% set date_to=chat['last_seen'] %} diff --git a/var/www/templates/chats_explorer/user_account.html b/var/www/templates/chats_explorer/user_account.html index ee8d4e88..7d869733 100644 --- a/var/www/templates/chats_explorer/user_account.html +++ b/var/www/templates/chats_explorer/user_account.html @@ -46,7 +46,6 @@
    username IDCreated at First Seen Last Seen NB Chats
    {{ meta['username']['id'] }} {{ meta['id'] }}{{ meta['created_at'] }} {% if meta['first_seen'] %} {{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }} @@ -74,6 +72,10 @@ {% if meta['info'] %} 
  • {{ meta['info'] }}
    + {% if meta['translation_info'] %} +
    + 
    {{ meta['translation_info'] }}
    + {% endif %}
    • {% endif %} @@ -100,6 +102,10 @@ + {% with translate_url=url_for('chats_explorer.objects_user_account', subtype=meta['subtype']), obj_id=meta['id'] %} + {% include 'chats_explorer/block_translation.html' %} + {% endwith %} + {# {% if meta['subchannels'] %}#} {#

    Sub-Channels:

    #} From a10119fb6a5d4179b666c678ed476e517b01cf24 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 29 Jan 2024 16:41:59 +0100 Subject: [PATCH 213/238] chg: [kvrocks] j -4 install + update to latest version --- bin/LAUNCH.sh | 5 +- configs/6383.conf | 430 ++++++++++++++++++++++++++++++++---------- installing_deps.sh | 2 +- update/v5.3/Update.sh | 13 +- 4 files changed, 345 insertions(+), 105 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 68c20ac7..862a49a0 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -602,7 +602,7 @@ function launch_all { function menu_display { - options=("Redis" "Ardb" "Kvrocks" "Logs" "Scripts" "Flask" "Killall" "Update" "Update-config" "Update-thirdparty") + options=("Redis" "Kvrocks" "Logs" "Scripts" "Flask" "Killall" "Update" "Update-config" "Update-thirdparty") menu() { echo "What do you want to Launch?:" @@ -630,9 +630,6 @@ function menu_display { Redis) launch_redis; ;; - Ardb) - launch_ardb; - ;; Kvrocks) launch_kvrocks; ;; diff --git a/configs/6383.conf b/configs/6383.conf index dfc1f205..0b889dbe 100644 --- a/configs/6383.conf +++ b/configs/6383.conf @@ -1,14 +1,14 @@ ################################ GENERAL ##################################### -# By default kvrocks listens for connections from all the network interfaces -# available on the server. It is possible to listen to just one or multiple -# interfaces using the "bind" configuration directive, followed by one or -# more IP addresses. +# By default kvrocks listens for connections from localhost interface. +# It is possible to listen to just one or multiple interfaces using +# the "bind" configuration directive, followed by one or more IP addresses. # # Examples: # # bind 192.168.1.100 10.0.0.1 # bind 127.0.0.1 ::1 +# bind 0.0.0.0 bind 127.0.0.1 # Unix socket. @@ -26,32 +26,52 @@ port 6383 # Close the connection after a client is idle for N seconds (0 to disable) timeout 0 -# The number of worker's threads, increase or decrease it would effect the performance. +# The number of worker's threads, increase or decrease would affect the performance. workers 8 -# By default kvrocks does not run as a daemon. Use 'yes' if you need it. -# Note that kvrocks will write a pid file in /var/run/kvrocks.pid when daemonized. +# By default, kvrocks does not run as a daemon. Use 'yes' if you need it. +# Note that kvrocks will write a PID file in /var/run/kvrocks.pid when daemonized daemonize no -# Kvrocks implements cluster solution that is similar with redis cluster solution. +# Kvrocks implements the cluster solution that is similar to the Redis cluster solution. # You can get cluster information by CLUSTER NODES|SLOTS|INFO command, it also is -# adapted to redis-cli, redis-benchmark, redis cluster SDK and redis cluster proxy. -# But kvrocks doesn't support to communicate with each others, so you must set +# adapted to redis-cli, redis-benchmark, Redis cluster SDK, and Redis cluster proxy. +# But kvrocks doesn't support communicating with each other, so you must set # cluster topology by CLUSTER SETNODES|SETNODEID commands, more details: #219. # # PLEASE NOTE: # If you enable cluster, kvrocks will encode key with its slot id calculated by -# CRC16 and modulo 16384, endoding key with its slot id makes it efficient to -# migrate keys based on slot. So if you enabled at first time, cluster mode must +# CRC16 and modulo 16384, encoding key with its slot id makes it efficient to +# migrate keys based on the slot. So if you enabled at first time, cluster mode must # not be disabled after restarting, and vice versa. That is to say, data is not # compatible between standalone mode with cluster mode, you must migrate data # if you want to change mode, otherwise, kvrocks will make data corrupt. # # Default: no + cluster-enabled no +# By default, namespaces are stored in the configuration file and won't be replicated +# to replicas. This option allows to change this behavior, so that namespaces are also +# propagated to slaves. Note that: +# 1) it won't replicate the 'masterauth' to prevent breaking master/replica replication +# 2) it will overwrite replica's namespace with master's namespace, so be careful of in-using namespaces +# 3) cannot switch off the namespace replication once it's enabled +# +# Default: no +repl-namespace-enabled no + +# Persist the cluster nodes topology in local file($dir/nodes.conf). This configuration +# takes effect only if the cluster mode was enabled. +# +# If yes, it will try to load the cluster topology from the local file when starting, +# and dump the cluster nodes into the file if it was changed. +# +# Default: yes +persist-cluster-nodes-enabled yes + # Set the max number of connected clients at the same time. By default -# this limit is set to 10000 clients, however if the server is not +# this limit is set to 10000 clients. However, if the server is not # able to configure the process file limit to allow for the specified limit # the max number of allowed clients is set to the current file limit # @@ -71,18 +91,17 @@ maxclients 10000 # 150k passwords per second against a good box. This means that you should # use a very strong password otherwise it will be very easy to break. # -# requirepass foobared requirepass ail # If the master is password protected (using the "masterauth" configuration # directive below) it is possible to tell the slave to authenticate before -# starting the replication synchronization process, otherwise the master will +# starting the replication synchronization process. Otherwise, the master will # refuse the slave request. # # masterauth foobared # Master-Salve replication would check db name is matched. if not, the slave should -# refuse to sync the db from master. Don't use default value, set the db-name to identify +# refuse to sync the db from master. Don't use the default value, set the db-name to identify # the cluster. db-name change.me.db @@ -98,7 +117,22 @@ dir DATA_KVROCKS # # log-dir stdout -# When running daemonized, kvrocks writes a pid file in ${CONFIG_DIR}/kvrocks.pid by +# Log level +# Possible values: info, warning, error, fatal +# Default: info +log-level info + +# You can configure log-retention-days to control whether to enable the log cleaner +# and the maximum retention days that the INFO level logs will be kept. +# +# if set to -1, that means to disable the log cleaner. +# if set to 0, all previous INFO level logs will be immediately removed. +# if set to between 0 to INT_MAX, that means it will retent latest N(log-retention-days) day logs. + +# By default the log-retention-days is -1. +log-retention-days -1 + +# When running in daemonize mode, kvrocks writes a PID file in ${CONFIG_DIR}/kvrocks.pid by # default. You can specify a custom pid file location here. # pidfile /var/run/kvrocks.pid pidfile DATA_KVROCKS/kvrocks.pid @@ -146,7 +180,7 @@ tcp-backlog 511 master-use-repl-port no # Currently, master only checks sequence number when replica asks for PSYNC, -# that is not enough since they may have different replication history even +# that is not enough since they may have different replication histories even # the replica asking sequence is in the range of the master current WAL. # # We design 'Replication Sequence ID' PSYNC, we add unique replication id for @@ -180,11 +214,11 @@ use-rsid-psync no # is still in progress, the slave can act in two different ways: # # 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will -# still reply to client requests, possibly with out of date data, or the +# still reply to client requests, possibly with out-of-date data, or the # data set may just be empty if this is the first synchronization. # # 2) if slave-serve-stale-data is set to 'no' the slave will reply with -# an error "SYNC with master in progress" to all the kind of commands +# an error "SYNC with master in progress" to all kinds of commands # but to INFO and SLAVEOF. # slave-serve-stale-data yes @@ -203,6 +237,35 @@ slave-serve-stale-data yes # Default: no slave-empty-db-before-fullsync no +# A Kvrocks master is able to list the address and port of the attached +# replicas in different ways. For example the "INFO replication" section +# offers this information, which is used, among other tools, by +# Redis Sentinel in order to discover replica instances. +# Another place where this info is available is in the output of the +# "ROLE" command of a master. +# +# The listed IP address and port normally reported by a replica is +# obtained in the following way: +# +# IP: The address is auto detected by checking the peer address +# of the socket used by the replica to connect with the master. +# +# Port: The port is communicated by the replica during the replication +# handshake, and is normally the port that the replica is using to +# listen for connections. +# +# However when port forwarding or Network Address Translation (NAT) is +# used, the replica may actually be reachable via different IP and port +# pairs. The following two options can be used by a replica in order to +# report to its master a specific set of IP and port, so that both INFO +# and ROLE will report those values. +# +# There is no need to use both the options if you need to override just +# the port or the IP address. +# +# replica-announce-ip 5.5.5.5 +# replica-announce-port 1234 + # If replicas need full synchronization with master, master need to create # checkpoint for feeding replicas, and replicas also stage a checkpoint of # the master. If we also keep the backup, it maybe occupy extra disk space. @@ -212,7 +275,7 @@ slave-empty-db-before-fullsync no # Default: no purge-backup-on-fullsync no -# The maximum allowed rate (in MB/s) that should be used by Replication. +# The maximum allowed rate (in MB/s) that should be used by replication. # If the rate exceeds max-replication-mb, replication will slow down. # Default: 0 (i.e. no limit) max-replication-mb 0 @@ -220,8 +283,8 @@ max-replication-mb 0 # The maximum allowed aggregated write rate of flush and compaction (in MB/s). # If the rate exceeds max-io-mb, io will slow down. # 0 is no limit -# Default: 500 -max-io-mb 500 +# Default: 0 +max-io-mb 0 # The maximum allowed space (in GB) that should be used by RocksDB. # If the total size of the SST files exceeds max_allowed_space, writes to RocksDB will fail. @@ -231,7 +294,7 @@ max-db-size 0 # The maximum backup to keep, server cron would run every minutes to check the num of current # backup, and purge the old backup if exceed the max backup num to keep. If max-backup-to-keep -# is 0, no backup would be keep. But now, we only support 0 or 1. +# is 0, no backup would be kept. But now, we only support 0 or 1. max-backup-to-keep 1 # The maximum hours to keep the backup. If max-backup-keep-hours is 0, wouldn't purge any backup. @@ -243,6 +306,115 @@ max-backup-keep-hours 24 # Default: 16 max-bitmap-to-string-mb 16 +# Whether to enable SCAN-like cursor compatible with Redis. +# If enabled, the cursor will be unsigned 64-bit integers. +# If disabled, the cursor will be a string. +# Default: no +redis-cursor-compatible yes + +# Whether to enable the RESP3 protocol. +# NOTICE: RESP3 is still under development, don't enable it in production environment. +# +# Default: no +# resp3-enabled no + +# Maximum nesting depth allowed when parsing and serializing +# JSON documents while using JSON commands like JSON.SET. +# Default: 1024 +json-max-nesting-depth 1024 + +# The underlying storage format of JSON data type +# NOTE: This option only affects newly written/updated key-values +# The CBOR format may reduce the storage size and speed up JSON commands +# Available values: json, cbor +# Default: json +json-storage-format json + +################################## TLS ################################### + +# By default, TLS/SSL is disabled, i.e. `tls-port` is set to 0. +# To enable it, `tls-port` can be used to define TLS-listening ports. +# tls-port 0 + +# Configure a X.509 certificate and private key to use for authenticating the +# server to connected clients, masters or cluster peers. +# These files should be PEM formatted. +# +# tls-cert-file kvrocks.crt +# tls-key-file kvrocks.key + +# If the key file is encrypted using a passphrase, it can be included here +# as well. +# +# tls-key-file-pass secret + +# Configure a CA certificate(s) bundle or directory to authenticate TLS/SSL +# clients and peers. Kvrocks requires an explicit configuration of at least one +# of these, and will not implicitly use the system wide configuration. +# +# tls-ca-cert-file ca.crt +# tls-ca-cert-dir /etc/ssl/certs + +# By default, clients on a TLS port are required +# to authenticate using valid client side certificates. +# +# If "no" is specified, client certificates are not required and not accepted. +# If "optional" is specified, client certificates are accepted and must be +# valid if provided, but are not required. +# +# tls-auth-clients no +# tls-auth-clients optional + +# By default, only TLSv1.2 and TLSv1.3 are enabled and it is highly recommended +# that older formally deprecated versions are kept disabled to reduce the attack surface. +# You can explicitly specify TLS versions to support. +# Allowed values are case insensitive and include "TLSv1", "TLSv1.1", "TLSv1.2", +# "TLSv1.3" (OpenSSL >= 1.1.1) or any combination. +# To enable only TLSv1.2 and TLSv1.3, use: +# +# tls-protocols "TLSv1.2 TLSv1.3" + +# Configure allowed ciphers. See the ciphers(1ssl) manpage for more information +# about the syntax of this string. +# +# Note: this configuration applies only to <= TLSv1.2. +# +# tls-ciphers DEFAULT:!MEDIUM + +# Configure allowed TLSv1.3 ciphersuites. See the ciphers(1ssl) manpage for more +# information about the syntax of this string, and specifically for TLSv1.3 +# ciphersuites. +# +# tls-ciphersuites TLS_CHACHA20_POLY1305_SHA256 + +# When choosing a cipher, use the server's preference instead of the client +# preference. By default, the server follows the client's preference. +# +# tls-prefer-server-ciphers yes + +# By default, TLS session caching is enabled to allow faster and less expensive +# reconnections by clients that support it. Use the following directive to disable +# caching. +# +# tls-session-caching no + +# Change the default number of TLS sessions cached. A zero value sets the cache +# to unlimited size. The default size is 20480. +# +# tls-session-cache-size 5000 + +# Change the default timeout of cached TLS sessions. The default timeout is 300 +# seconds. +# +# tls-session-cache-timeout 60 + +# By default, a replica does not attempt to establish a TLS connection +# with its master. +# +# Use the following directive to enable TLS on replication links. +# +# tls-replication yes + ################################## SLOW LOG ################################### # The Kvrocks Slow Log is a mechanism to log queries that exceeded a specified @@ -301,8 +473,8 @@ supervised no # Default: empty # profiling-sample-commands "" -# Ratio of the samples would be recorded. We simply use the rand to determine -# whether to record the sample or not. +# Ratio of the samples would be recorded. It is a number between 0 and 100. +# We simply use the rand to determine whether to record the sample or not. # # Default: 0 profiling-sample-ratio 0 @@ -331,15 +503,27 @@ profiling-sample-record-threshold-ms 100 # 0-7am every day. compaction-checker-range 0-7 -# Bgsave scheduler, auto bgsave at schedule time +# When the compaction checker is triggered, the db will periodically pick the SST file +# with the highest "deleted percentage" (i.e. the percentage of deleted keys in the SST +# file) to compact, in order to free disk space. +# However, if a specific SST file was created more than "force-compact-file-age" seconds +# ago, and its percentage of deleted keys is higher than +# "force-compact-file-min-deleted-percentage", it will be forcely compacted as well. + +# Default: 172800 seconds; Range: [60, INT64_MAX]; +# force-compact-file-age 172800 +# Default: 10 %; Range: [1, 100]; +# force-compact-file-min-deleted-percentage 10 + +# Bgsave scheduler, auto bgsave at scheduled time # time expression format is the same as crontab(currently only support * and int) # e.g. bgsave-cron 0 3 * * * 0 4 * * * -# would bgsave the db at 3am and 4am everyday +# would bgsave the db at 3am and 4am every day # Command renaming. # # It is possible to change the name of dangerous commands in a shared -# environment. For instance the KEYS command may be renamed into something +# environment. For instance, the KEYS command may be renamed into something # hard to guess so that it will still be available for internal-use tools # but not available for general clients. # @@ -352,39 +536,26 @@ compaction-checker-range 0-7 # # rename-command KEYS "" -# The key-value size may so be quite different in many scenes, and use 256MiB as SST file size -# may cause data loading(large index/filter block) ineffective when the key-value was too small. -# kvrocks supports user-defined SST file in config(rocksdb.target_file_size_base), -# but it still too trivial and inconvenient to adjust the different sizes for different instances. -# so we want to periodic auto-adjust the SST size in-flight with user avg key-value size. -# -# If enabled, kvrocks will auto resize rocksdb.target_file_size_base -# and rocksdb.write_buffer_size in-flight with user avg key-value size. -# Please see #118. -# -# Default: yes -auto-resize-block-and-sst yes - ################################ MIGRATE ##################################### # If the network bandwidth is completely consumed by the migration task, # it will affect the availability of kvrocks. To avoid this situation, -# migrate-speed is adpoted to limit the migrating speed. -# Migrating speed is limited by controling the duraiton between sending data, -# the duation is calculated by: 1000000 * migrate-pipeline-size / migrate-speed (us). +# migrate-speed is adopted to limit the migrating speed. +# Migrating speed is limited by controlling the duration between sending data, +# the duration is calculated by: 1000000 * migrate-pipeline-size / migrate-speed (us). # Value: [0,INT_MAX], 0 means no limit # # Default: 4096 migrate-speed 4096 -# In order to reduce data transimission times and improve the efficiency of data migration, +# In order to reduce data transmission times and improve the efficiency of data migration, # pipeline is adopted to send multiple data at once. Pipeline size can be set by this option. # Value: [1, INT_MAX], it can't be 0 # # Default: 16 migrate-pipeline-size 16 -# In order to reduce the write forbidden time during migrating slot, we will migrate the incremetal -# data sevral times to reduce the amount of incremetal data. Until the quantity of incremetal +# In order to reduce the write forbidden time during migrating slot, we will migrate the incremental +# data several times to reduce the amount of incremental data. Until the quantity of incremental # data is reduced to a certain threshold, slot will be forbidden write. The threshold is set by # this option. # Value: [1, INT_MAX], it can't be 0 @@ -394,22 +565,21 @@ migrate-sequence-gap 10000 ################################ ROCKSDB ##################################### -# Specify the capacity of metadata column family block cache. Larger block cache -# may make request faster while more keys would be cached. Max Size is 200*1024. -# Default: 2048MB -rocksdb.metadata_block_cache_size 2048 +# Specify the capacity of column family block cache. A larger block cache +# may make requests faster while more keys would be cached. Max Size is 400*1024. +# Default: 4096MB +rocksdb.block_cache_size 4096 -# Specify the capacity of subkey column family block cache. Larger block cache -# may make request faster while more keys would be cached. Max Size is 200*1024. -# Default: 2048MB -rocksdb.subkey_block_cache_size 2048 - -# Metadata column family and subkey column family will share a single block cache -# if set 'yes'. The capacity of shared block cache is -# metadata_block_cache_size + subkey_block_cache_size +# Specify the type of cache used in the block cache. +# Accept value: "lru", "hcc" +# "lru" stands for the cache with the LRU(Least Recently Used) replacement policy. # -# Default: yes -rocksdb.share_metadata_and_subkey_block_cache yes +# "hcc" stands for the Hyper Clock Cache, a lock-free cache alternative +# that offers much improved CPU efficiency vs. LRU cache under high parallel +# load or high contention. +# +# default lru +rocksdb.block_cache_type lru # A global cache for table-level rows in RocksDB. If almost always point # lookups, enlarging row cache may improve read performance. Otherwise, @@ -423,7 +593,7 @@ rocksdb.row_cache_size 0 # files opened are always kept open. You can estimate number of files based # on target_file_size_base and target_file_size_multiplier for level-based # compaction. For universal-style compaction, you can usually set it to -1. -# Default: 4096 +# Default: 8096 rocksdb.max_open_files 8096 # Amount of data to build up in memory (backed by an unsorted log @@ -442,7 +612,7 @@ rocksdb.max_open_files 8096 # default is 64MB rocksdb.write_buffer_size 64 -# Target file size for compaction, target file size for Leve N can be caculated +# Target file size for compaction, target file size for Level N can be calculated # by target_file_size_base * (target_file_size_multiplier ^ (L-1)) # # Default: 128MB @@ -457,20 +627,29 @@ rocksdb.target_file_size_base 128 # allowed. rocksdb.max_write_buffer_number 4 +# Maximum number of concurrent background jobs (compactions and flushes). +# For backwards compatibility we will set `max_background_jobs = +# max_background_compactions + max_background_flushes` in the case where user +# sets at least one of `max_background_compactions` or `max_background_flushes` +# (we replace -1 by 1 in case one option is unset). +rocksdb.max_background_jobs 4 + +# DEPRECATED: it is automatically decided based on the value of rocksdb.max_background_jobs # Maximum number of concurrent background compaction jobs, submitted to # the default LOW priority thread pool. -rocksdb.max_background_compactions 4 +rocksdb.max_background_compactions -1 +# DEPRECATED: it is automatically decided based on the value of rocksdb.max_background_jobs # Maximum number of concurrent background memtable flush jobs, submitted by # default to the HIGH priority thread pool. If the HIGH priority thread pool # is configured to have zero threads, flush jobs will share the LOW priority # thread pool with compaction jobs. -rocksdb.max_background_flushes 4 +rocksdb.max_background_flushes -1 # This value represents the maximum number of threads that will # concurrently perform a compaction job by breaking it into multiple, # smaller ones that are run simultaneously. -# Default: 2 (i.e. no subcompactions) +# Default: 2 rocksdb.max_sub_compactions 2 # In order to limit the size of WALs, RocksDB uses DBOptions::max_total_wal_size @@ -494,8 +673,8 @@ rocksdb.max_sub_compactions 2 # default is 512MB rocksdb.max_total_wal_size 512 -# We impl the repliction with rocksdb WAL, it would trigger full sync when the seq was out of range. -# wal_ttl_seconds and wal_size_limit_mb would affect how archived logswill be deleted. +# We implement the replication with rocksdb WAL, it would trigger full sync when the seq was out of range. +# wal_ttl_seconds and wal_size_limit_mb would affect how archived logs will be deleted. # If WAL_ttl_seconds is not 0, then WAL files will be checked every WAL_ttl_seconds / 2 and those that # are older than WAL_ttl_seconds will be deleted# # @@ -505,26 +684,26 @@ rocksdb.wal_ttl_seconds 10800 # If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0, # WAL files will be checked every 10 min and if total size is greater # then WAL_size_limit_MB, they will be deleted starting with the -# earliest until size_limit is met. All empty files will be deleted +# earliest until size_limit is met. All empty files will be deleted # Default: 16GB rocksdb.wal_size_limit_mb 16384 # Approximate size of user data packed per block. Note that the -# block size specified here corresponds to uncompressed data. The +# block size specified here corresponds to uncompressed data. The # actual size of the unit read from disk may be smaller if # compression is enabled. # -# Default: 4KB +# Default: 16KB rocksdb.block_size 16384 # Indicating if we'd put index/filter blocks to the block cache # -# Default: no +# Default: yes rocksdb.cache_index_and_filter_blocks yes # Specify the compression to use. Only compress level greater # than 2 to improve performance. -# Accept value: "no", "snappy" +# Accept value: "no", "snappy", "lz4", "zstd", "zlib" # default snappy rocksdb.compression snappy @@ -579,7 +758,7 @@ rocksdb.stats_dump_period_sec 0 # Default: no rocksdb.disable_auto_compactions no -# BlobDB(key-value separation) is essentially RocksDB for large-value use cases. +# BlobDB(key-value separation) is essentially RocksDB for large-value use cases. # Since 6.18.0, The new implementation is integrated into the RocksDB core. # When set, large values (blobs) are written to separate blob files, and only # pointers to them are stored in SST files. This can reduce write amplification @@ -608,7 +787,7 @@ rocksdb.blob_file_size 268435456 # Enables garbage collection of blobs. Valid blobs residing in blob files # older than a cutoff get relocated to new files as they are encountered # during compaction, which makes it possible to clean up blob files once -# they contain nothing but obsolete/garbage blobs. +# they contain nothing but obsolete/garbage blobs. # See also rocksdb.blob_garbage_collection_age_cutoff below. # # Default: yes @@ -623,16 +802,16 @@ rocksdb.enable_blob_garbage_collection yes rocksdb.blob_garbage_collection_age_cutoff 25 -# The purpose of following three options are to dynamically adjust the upper limit of -# the data that each layer can store according to the size of the different +# The purpose of the following three options are to dynamically adjust the upper limit of +# the data that each layer can store according to the size of the different # layers of the LSM. Enabling this option will bring some improvements in -# deletion efficiency and space amplification, but it will lose a certain +# deletion efficiency and space amplification, but it will lose a certain # amount of read performance. -# If you want know more details about Levels' Target Size, you can read RocksDB wiki: +# If you want to know more details about Levels' Target Size, you can read RocksDB wiki: # https://github.com/facebook/rocksdb/wiki/Leveled-Compaction#levels-target-size # -# Default: no -rocksdb.level_compaction_dynamic_level_bytes no +# Default: yes +rocksdb.level_compaction_dynamic_level_bytes yes # The total file size of level-1 sst. # @@ -641,39 +820,92 @@ rocksdb.max_bytes_for_level_base 268435456 # Multiplication factor for the total file size of L(n+1) layers. # This option is a double type number in RocksDB, but kvrocks is -# not support double data type number yet, so we use int data +# not support the double data type number yet, so we use integer # number instead of double currently. # # Default: 10 rocksdb.max_bytes_for_level_multiplier 10 +# This feature only takes effect in Iterators and MultiGet. +# If yes, RocksDB will try to read asynchronously and in parallel as much as possible to hide IO latency. +# In iterators, it will prefetch data asynchronously in the background for each file being iterated on. +# In MultiGet, it will read the necessary data blocks from those files in parallel as much as possible. + +# Default no +rocksdb.read_options.async_io no + +# If yes, the write will be flushed from the operating system +# buffer cache before the write is considered complete. +# If this flag is enabled, writes will be slower. +# If this flag is disabled, and the machine crashes, some recent +# rites may be lost. Note that if it is just the process that +# crashes (i.e., the machine does not reboot), no writes will be +# lost even if sync==false. +# +# Default: no +rocksdb.write_options.sync no + +# If yes, writes will not first go to the write ahead log, +# and the write may get lost after a crash. +# You must keep wal enabled if you use replication. +# +# Default: no +rocksdb.write_options.disable_wal no + +# If enabled and we need to wait or sleep for the write request, fails +# immediately. +# +# Default: no +rocksdb.write_options.no_slowdown no + +# If enabled, write requests are of lower priority if compaction is +# behind. In this case, no_slowdown = true, the request will be canceled +# immediately. Otherwise, it will be slowed down. +# The slowdown value is determined by RocksDB to guarantee +# it introduces minimum impacts to high priority writes. +# +# Default: no +rocksdb.write_options.low_pri no + +# If enabled, this writebatch will maintain the last insert positions of each +# memtable as hints in concurrent write. It can improve write performance +# in concurrent writes if keys in one writebatch are sequential. +# +# Default: no +rocksdb.write_options.memtable_insert_hint_per_batch no + + +# Support RocksDB auto-tune rate limiter for the background IO +# if enabled, Rate limiter will limit the compaction write if flush write is high +# Please see https://rocksdb.org/blog/2017/12/18/17-auto-tuned-rate-limiter.html +# +# Default: yes +rocksdb.rate_limiter_auto_tuned yes + +# Enable this option will schedule the deletion of obsolete files in a background thread +# on iterator destruction. It can reduce the latency if there are many files to be removed. +# see https://github.com/facebook/rocksdb/wiki/IO#avoid-blocking-io +# +# Default: yes +# rocksdb.avoid_unnecessary_blocking_io yes + ################################ NAMESPACE ##################################### # namespace.test change.me + +-# investigation -> db ???? +-# ail2ail -> a2a ???? + + backup-dir DATA_KVROCKS/backup -fullsync-recv-file-delay 0 log-dir DATA_KVROCKS -unixsocketperm 26 - - - - namespace.cor ail_correls namespace.crawl ail_crawlers namespace.db ail_datas namespace.dup ail_dups namespace.obj ail_objs -namespace.tl ail_tls namespace.rel ail_rels namespace.stat ail_stats namespace.tag ail_tags +namespace.tl ail_tls namespace.track ail_trackers - -# investigation -> db ???? -# ail2ail -> a2a ????? - - - - - - diff --git a/installing_deps.sh b/installing_deps.sh index e6f907a1..c681249b 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -88,7 +88,7 @@ DEFAULT_HOME=$(pwd) #### KVROCKS #### test ! -d kvrocks/ && git clone https://github.com/apache/incubator-kvrocks.git kvrocks pushd kvrocks -./x.py build +./x.py build -j 4 popd DEFAULT_KVROCKS_DATA=$DEFAULT_HOME/DATA_KVROCKS diff --git a/update/v5.3/Update.sh b/update/v5.3/Update.sh index 1e040200..534cd295 100755 --- a/update/v5.3/Update.sh +++ b/update/v5.3/Update.sh @@ -14,7 +14,7 @@ GREEN="\\033[1;32m" DEFAULT="\\033[0;39m" echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -ks +bash ${AIL_BIN}/LAUNCH.sh -k wait # SUBMODULES # @@ -28,6 +28,17 @@ pip install -U libretranslatepy pip install -U xxhash pip install -U DomainClassifier +echo "" +echo -e $GREEN"Updating KVROCKS ..."$DEFAULT +echo "" +pushd ${AIL_HOME}/kvrocks +git pull +./x.py build -j 4 +popd + +bash ${AIL_BIN}/LAUNCH.sh -lrv +bash ${AIL_BIN}/LAUNCH.sh -lkv + echo "" echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT echo "" From ce1e1f791347f435a609875543e6c30e049311a7 Mon Sep 17 00:00:00 2001 From: Niclas Dauster <83672752+NMD03@users.noreply.github.com> Date: Tue, 30 Jan 2024 09:15:50 +0100 Subject: [PATCH 214/238] Update README.md --- other_installers/LXD/README.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/other_installers/LXD/README.md b/other_installers/LXD/README.md index 59c779cf..2c504f9f 100644 --- a/other_installers/LXD/README.md +++ b/other_installers/LXD/README.md @@ -33,3 +33,36 @@ The following options are available: ## Configuration If you installed Lacus, you can configure AIL to use it as a crawler. For further information, please refer to the [HOWTO.md](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md) + +## Using Images to run AIL +If you want to use images to install AIL, you can download them from the ail-project [image website](https://images.ail-project.org/) + +After downloading the images, you can import them into LXD using the following command: +```bash +lxc image import --alias +``` +Now you can use the image to create a container: +```bash +lxc launch +``` + +To log into the container you need to know the automatically generated password. You can get it with the following command: +```bash +lxc exec -- bash -c "grep '^password=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2" +``` + +If you also want to use Lacus, you can do the same with the Lacus image. After that, you can configure AIL to use Lacus as a crawler. For further information, please refer to the [HOWTO.md](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md). + +## Building the images locally +If you want to build the images locally, you can use the `build.sh` script: +```bash +bash build.sh [OPTIONS] +``` +| Flag | Default Value | Description | +| ------------------------------- | ------------- | --------------------------------------------------------------------------------- | +| `--ail` | `false` | Activates the creation of the AIL container. | +| `--lacus` | `false` | Activates the creation of the Lacus container. | +| `--ail-name ` | `AIL` | Specifies the name of the AIL container. The default is a generic name "AIL". | +| `--lacus-name ` | `Lacus` | Specifies the name of the Lacus container. The default is a generic name "Lacus". | +| `-o`, `--outputdir ` | `` | Sets the output directory for the LXD image files. | +| `-s`, `--sign` | `false` | Enables the signing of the generated LXD image files. | From 2db8587d03642e16cfa8d468e9b35f4835c0c58b Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 30 Jan 2024 10:28:50 +0100 Subject: [PATCH 215/238] chg: [Hosts] improve perf + regex timeout + cache DNS results --- bin/modules/DomClassifier.py | 30 ++++++++++++++++++------------ bin/modules/Hosts.py | 36 +++++++++++++++++++----------------- 2 files changed, 37 insertions(+), 29 deletions(-) diff --git a/bin/modules/DomClassifier.py b/bin/modules/DomClassifier.py index 94cf53db..b4620ee2 100755 --- a/bin/modules/DomClassifier.py +++ b/bin/modules/DomClassifier.py @@ -41,7 +41,13 @@ class DomClassifier(AbstractModule): addr_dns = config_loader.get_config_str("DomClassifier", "dns") - self.c = DomainClassifier.domainclassifier.Extract(rawtext="", nameservers=[addr_dns]) + redis_host = config_loader.get_config_str('Redis_Cache', 'host') + redis_port = config_loader.get_config_int('Redis_Cache', 'port') + redis_db = config_loader.get_config_int('Redis_Cache', 'db') + self.dom_classifier = DomainClassifier.domainclassifier.Extract(rawtext="", nameservers=[addr_dns], + redis_host=redis_host, + redis_port=redis_port, redis_db=redis_db, + re_timeout=30) self.cc = config_loader.get_config_str("DomClassifier", "cc") self.cc_tld = config_loader.get_config_str("DomClassifier", "cc_tld") @@ -58,34 +64,34 @@ class DomClassifier(AbstractModule): item_source = item.get_source() try: - self.c.text(rawtext=host) - if not self.c.domain: + self.dom_classifier.text(rawtext=host) + if not self.dom_classifier.domain: return - print(self.c.domain) - self.c.validdomain(passive_dns=True, extended=False) - # self.logger.debug(self.c.vdomain) + print(self.dom_classifier.domain) + self.dom_classifier.validdomain(passive_dns=True, extended=False) + # self.logger.debug(self.dom_classifier.vdomain) - print(self.c.vdomain) + print(self.dom_classifier.vdomain) print() - if self.c.vdomain and d4.is_passive_dns_enabled(): - for dns_record in self.c.vdomain: + if self.dom_classifier.vdomain and d4.is_passive_dns_enabled(): + for dns_record in self.dom_classifier.vdomain: self.add_message_to_queue(obj=None, message=dns_record) if self.cc_tld: - localizeddomains = self.c.include(expression=self.cc_tld) + localizeddomains = self.dom_classifier.include(expression=self.cc_tld) if localizeddomains: print(localizeddomains) self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{item.get_id()}") if self.cc: - localizeddomains = self.c.localizedomain(cc=self.cc) + localizeddomains = self.dom_classifier.localizedomain(cc=self.cc) if localizeddomains: print(localizeddomains) self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{item.get_id()}") if r_result: - return self.c.vdomain + return self.dom_classifier.vdomain except IOError as err: self.redis_logger.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed") diff --git a/bin/modules/Hosts.py b/bin/modules/Hosts.py index 488e7acf..55670777 100755 --- a/bin/modules/Hosts.py +++ b/bin/modules/Hosts.py @@ -18,13 +18,14 @@ import os import re import sys +import DomainClassifier.domainclassifier + sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## from modules.abstract_module import AbstractModule from lib.ConfigLoader import ConfigLoader -from lib.objects.Items import Item class Hosts(AbstractModule): """ @@ -43,28 +44,29 @@ class Hosts(AbstractModule): # Waiting time in seconds between to message processed self.pending_seconds = 1 - self.host_regex = r'\b([a-zA-Z\d-]{,63}(?:\.[a-zA-Z\d-]{,63})+)\b' - re.compile(self.host_regex) - + redis_host = config_loader.get_config_str('Redis_Cache', 'host') + redis_port = config_loader.get_config_int('Redis_Cache', 'port') + redis_db = config_loader.get_config_int('Redis_Cache', 'db') + self.dom_classifier = DomainClassifier.domainclassifier.Extract(rawtext="", + redis_host=redis_host, + redis_port=redis_port, + redis_db=redis_db, + re_timeout=30) self.logger.info(f"Module: {self.module_name} Launched") def compute(self, message): - item = self.get_obj() + obj = self.get_obj() - # mimetype = item_basic.get_item_mimetype(item.get_id()) - # if mimetype.split('/')[0] == "text": - - content = item.get_content() - hosts = self.regex_findall(self.host_regex, item.get_id(), content, r_set=True) - if hosts: - print(f'{len(hosts)} host {item.get_id()}') - for host in hosts: - # print(host) - if not host.endswith('.onion'): - self.add_message_to_queue(message=str(host), queue='Host') + content = obj.get_content() + self.dom_classifier.text(content) + if self.dom_classifier.domain: + print(f'{len(self.dom_classifier.domain)} host {obj.get_id()}') + # print(self.dom_classifier.domain) + for domain in self.dom_classifier.domain: + if domain: + self.add_message_to_queue(message=domain, queue='Host') if __name__ == '__main__': - module = Hosts() module.run() From fbd7e2236afbf1e097c8256f97b921189b9186fc Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 30 Jan 2024 11:24:12 +0100 Subject: [PATCH 216/238] fix: [crawlers] fix errored capture start time --- bin/lib/crawlers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index f98af175..ed418432 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -1331,6 +1331,8 @@ class CrawlerCapture: start_time = self.get_task().get_start_time() if r_str: return start_time + elif not start_time: + return 0 else: start_time = datetime.strptime(start_time, "%Y/%m/%d - %H:%M.%S").timestamp() return int(start_time) From d1608e89e13a4a1df8e83f2c87d941de3147a5f3 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 30 Jan 2024 11:29:42 +0100 Subject: [PATCH 217/238] fix: [crawlers] fix errored capture queue --- bin/crawlers/Crawler.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index c1ba0e0c..b1a7d7cd 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -142,6 +142,12 @@ class Crawler(AbstractModule): return capture elif status == crawlers.CaptureStatus.UNKNOWN: capture_start = capture.get_start_time(r_str=False) + if capture_start == 0: + task = capture.get_task() + task.delete() + capture.delete() + self.logger.warning(f'capture UNKNOWN ERROR STATE, {task.uuid} Removed from queue') + return None if int(time.time()) - capture_start > 600: # TODO ADD in new crawler config task = capture.get_task() task.reset() From 194ae960fc6849786cb040f215f68115676d7e8a Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 30 Jan 2024 11:35:43 +0100 Subject: [PATCH 218/238] fix: [crawlers] fix capture return error code --- bin/crawlers/Crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index b1a7d7cd..1f46e938 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -161,7 +161,7 @@ class Crawler(AbstractModule): except ConnectionError: print(capture.uuid) - capture.update(self, -1) + capture.update(-1) self.refresh_lacus_status() time.sleep(self.pending_seconds) From 5fab2326e60ed81dc18e0cfd62fb40a3cec84770 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 30 Jan 2024 11:45:43 +0100 Subject: [PATCH 219/238] fix: [misp export] fix empty event on module start --- bin/exporter/MISPExporter.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/bin/exporter/MISPExporter.py b/bin/exporter/MISPExporter.py index 4a0d3a1c..1fd32045 100755 --- a/bin/exporter/MISPExporter.py +++ b/bin/exporter/MISPExporter.py @@ -319,11 +319,7 @@ class MISPExporterAutoDaily(MISPExporter): def __init__(self, url='', key='', ssl=False): super().__init__(url=url, key=key, ssl=ssl) - # create event if don't exists - try: - self.event_id = self.get_daily_event_id() - except MISPConnectionError: - self.event_id = - 1 + self.event_id = - 1 self.date = datetime.date.today() def export(self, obj, tag): @@ -345,6 +341,7 @@ class MISPExporterAutoDaily(MISPExporter): self.add_event_object(self.event_id, obj) except MISPConnectionError: + self.event_id = - 1 return -1 From e4f21f05cc250b62dd45834f3a894a7e18e01490 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 30 Jan 2024 14:31:09 +0100 Subject: [PATCH 220/238] fix: [D4] fix module cache --- bin/core/D4_client.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bin/core/D4_client.py b/bin/core/D4_client.py index 9c452912..8bd79ded 100755 --- a/bin/core/D4_client.py +++ b/bin/core/D4_client.py @@ -34,16 +34,20 @@ class D4Client(AbstractModule): self.d4_client = d4.create_d4_client() self.last_refresh = time.time() + self.last_config_check = time.time() # Send module state to logs self.logger.info(f'Module {self.module_name} initialized') def compute(self, dns_record): # Refresh D4 Client - if self.last_refresh < d4.get_config_last_update_time(): - self.d4_client = d4.create_d4_client() - self.last_refresh = time.time() - print('D4 Client: config updated') + if self.last_config_check < int(time.time()) - 30: + print('refresh rrrr') + if self.last_refresh < d4.get_config_last_update_time(): + self.d4_client = d4.create_d4_client() + self.last_refresh = time.time() + print('D4 Client: config updated') + self.last_config_check = time.time() if self.d4_client: # Send DNS Record to D4Server From deb89db8188960ee871ed0fff9c67344ecdb2f58 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 1 Feb 2024 09:58:21 +0100 Subject: [PATCH 221/238] fix: [ui] Do not show relNotes link if not on a tag --- var/www/modules/settings/templates/settings_index.html | 4 ++++ var/www/templates/settings/settings_index.html | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/var/www/modules/settings/templates/settings_index.html b/var/www/modules/settings/templates/settings_index.html index b6a3fdf7..4dbdce42 100644 --- a/var/www/modules/settings/templates/settings_index.html +++ b/var/www/modules/settings/templates/settings_index.html @@ -43,7 +43,11 @@
    AIL Version{{current_version}} (release note){{git_metadata['current_branch']}}
    AIL Version{{ail_version}} (release note){{git_metadata['current_branch']}}
    #} -{# #} -{# #} -{# #} -{# #} -{# #} -{# #} -{# #} -{# #} -{# #} -{# #} -{# #} -{# #} -{# #} -{# #} -{# #} -{# #} -{# #} -{# #} -{# #} -{# #} -{# #} -{# #} -{#
    NameIDCreated atFirst SeenLast SeenNB Sub-ChannelsParticipants
    {{ chat['name'] }}{{ chat['id'] }}{{ chat['created_at'] }}#} -{# {% if chat['first_seen'] %}#} -{# {{ chat['first_seen'][0:4] }}-{{ chat['first_seen'][4:6] }}-{{ chat['first_seen'][6:8] }}#} -{# {% endif %}#} -{# #} -{# {% if chat['last_seen'] %}#} -{# {{ chat['last_seen'][0:4] }}-{{ chat['last_seen'][4:6] }}-{{ chat['last_seen'][6:8] }}#} -{# {% endif %}#} -{# {{ chat['nb_subchannels'] }}#} -{# {{ chat['participants'] | length}}#} -{##} -{#
    #} -{# {% if chat['info'] %}#} -{#
  • #} -{# 
    {{ chat['info'] }}
    #} -{#
    • #} -{# {% endif %}#} -{# #} -{# #} -{##} -{#
    #} -{#
    #} - +
    {{ meta['type'] }} {{ meta['subtype'] }} {{ meta['id'] }}

    Participants:

    From 7295f7b32d5999b49050ed2221d5e8987ecbdd25 Mon Sep 17 00:00:00 2001 From: terrtia Date: Fri, 2 Feb 2024 11:42:10 +0100 Subject: [PATCH 223/238] chg: [LAUNCH] change restart flags --- bin/LAUNCH.sh | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index fe33720d..31761a94 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -31,19 +31,15 @@ export PATH=$AIL_KVROCKS:$PATH export PATH=$AIL_BIN:$PATH export PATH=$AIL_FLASK:$PATH -function isup { - isredis=`screen -ls | egrep '[0-9]+.Redis_AIL' | cut -d. -f1` - isardb=`screen -ls | egrep '[0-9]+.ARDB_AIL' | cut -d. -f1` - iskvrocks=`screen -ls | egrep '[0-9]+.KVROCKS_AIL' | cut -d. -f1` - islogged=`screen -ls | egrep '[0-9]+.Logging_AIL' | cut -d. -f1` - is_ail_core=`screen -ls | egrep '[0-9]+.Core_AIL' | cut -d. -f1` - is_ail_2_ail=`screen -ls | egrep '[0-9]+.AIL_2_AIL' | cut -d. -f1` - isscripted=`screen -ls | egrep '[0-9]+.Script_AIL' | cut -d. -f1` - isflasked=`screen -ls | egrep '[0-9]+.Flask_AIL' | cut -d. -f1` - isfeeded=`screen -ls | egrep '[0-9]+.Feeder_Pystemon' | cut -d. -f1` -} - -isup +isredis=`screen -ls | egrep '[0-9]+.Redis_AIL' | cut -d. -f1` +isardb=`screen -ls | egrep '[0-9]+.ARDB_AIL' | cut -d. -f1` +iskvrocks=`screen -ls | egrep '[0-9]+.KVROCKS_AIL' | cut -d. -f1` +islogged=`screen -ls | egrep '[0-9]+.Logging_AIL' | cut -d. -f1` +is_ail_core=`screen -ls | egrep '[0-9]+.Core_AIL' | cut -d. -f1` +is_ail_2_ail=`screen -ls | egrep '[0-9]+.AIL_2_AIL' | cut -d. -f1` +isscripted=`screen -ls | egrep '[0-9]+.Script_AIL' | cut -d. -f1` +isflasked=`screen -ls | egrep '[0-9]+.Flask_AIL' | cut -d. -f1` +isfeeded=`screen -ls | egrep '[0-9]+.Feeder_Pystemon' | cut -d. -f1` function helptext { echo -e $YELLOW" @@ -63,7 +59,6 @@ function helptext { - All the queuing modules. - All the processing modules. - All Redis in memory servers. - - All ARDB on disk servers. - All KVROCKS servers. "$DEFAULT" (Inside screen Daemons) @@ -73,7 +68,7 @@ function helptext { LAUNCH.sh [-l | --launchAuto] LAUNCH DB + Scripts [-k | --killAll] Kill DB + Scripts - [-kl | --killLaunch] Kill All & launchAuto + [-r | --restart] Restart [-ks | --killscript] Scripts [-u | --update] Update AIL [-ut | --thirdpartyUpdate] Update UI/Frontend @@ -697,8 +692,8 @@ while [ "$1" != "" ]; do ;; -k | --killAll ) killall; ;; - -kl | --killLaunch ) killall; - isup; + -r | --restart ) killall; + sleep 0.1; launch_all "automatic"; ;; -ks | --killscript ) killscript; From 67e5c5777d8c3e491d2053db4661fff84c3cf1d1 Mon Sep 17 00:00:00 2001 From: terrtia Date: Fri, 2 Feb 2024 11:50:13 +0100 Subject: [PATCH 224/238] fix: [docs + UI] fix repository links --- other_installers/ansible/roles/ail-host/tasks/main.yml | 2 +- other_installers/docker/README.md | 2 +- var/www/modules/settings/templates/settings_index.html | 4 ++-- var/www/templates/settings/settings_index.html | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/other_installers/ansible/roles/ail-host/tasks/main.yml b/other_installers/ansible/roles/ail-host/tasks/main.yml index 47ada451..c429a9f3 100644 --- a/other_installers/ansible/roles/ail-host/tasks/main.yml +++ b/other_installers/ansible/roles/ail-host/tasks/main.yml @@ -64,7 +64,7 @@ - name: Clone the AIL repository git: - repo: 'https://github.com/ail-project/AIL-framework.git' + repo: 'https://github.com/ail-project/ail-framework.git' dest: /opt/AIL-framework update: yes diff --git a/other_installers/docker/README.md b/other_installers/docker/README.md index c931bb65..018ba146 100644 --- a/other_installers/docker/README.md +++ b/other_installers/docker/README.md @@ -15,7 +15,7 @@ curl https://get.docker.com | /bin/bash 2. Type these commands to build the Docker image: ```bash -git clone https://github.com/ail-project/AIL-framework.git +git clone https://github.com/ail-project/ail-framework.git cd AIL-framework cp -r ./other_installers/docker/Dockerfile ./other_installers/docker/docker_start.sh ./other_installers/docker/pystemon ./ cp ./configs/update.cfg.sample ./configs/update.cfg diff --git a/var/www/modules/settings/templates/settings_index.html b/var/www/modules/settings/templates/settings_index.html index e267b706..af3d74c2 100644 --- a/var/www/modules/settings/templates/settings_index.html +++ b/var/www/modules/settings/templates/settings_index.html @@ -125,7 +125,7 @@

    New Version Available!

    A new version is available, new version: {{git_metadata['last_remote_tag']}}

    - Check last release note. + Check last release note. {%endif%} @@ -134,7 +134,7 @@

    New Update Available!

    A new update is available, new commit ID: {{git_metadata['last_remote_commit']}}

    - Check last commit content. + Check last commit content. {%endif%} diff --git a/var/www/templates/settings/settings_index.html b/var/www/templates/settings/settings_index.html index ecc2e9e3..9819d4ea 100644 --- a/var/www/templates/settings/settings_index.html +++ b/var/www/templates/settings/settings_index.html @@ -125,7 +125,7 @@

    New Version Available!

    A new version is available, new version: {{git_metadata['last_remote_tag']}}

    - Check last release note. + Check last release note. {%endif%} @@ -134,7 +134,7 @@

    New Update Available!

    A new update is available, new commit ID: {{git_metadata['last_remote_commit']}}

    - Check last commit content. + Check last commit content. {%endif%} From 1a2d1e41f577ced47d706a3498e96130dcff721b Mon Sep 17 00:00:00 2001 From: terrtia Date: Fri, 2 Feb 2024 11:53:25 +0100 Subject: [PATCH 225/238] fix: [tags] fix typo --- bin/lib/Tag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index 9140a583..588be36f 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -1249,7 +1249,7 @@ class Tag: return last_seen def set_last_seen(self, last_seen): - return r_tags.hset(f'meta:tag:{self.name}', 'last_seen', int(last_seen) + return r_tags.hset(f'meta:tag:{self.name}', 'last_seen', int(last_seen)) def get_color(self): color = r_tags.hget(f'meta:tag:{self.name}', 'color') From a7fd838329f421d55fc6e839bbfb0d87504df0c8 Mon Sep 17 00:00:00 2001 From: terrtia Date: Fri, 2 Feb 2024 14:41:57 +0100 Subject: [PATCH 226/238] fix: [tags] fix invalid tags --- bin/lib/Tag.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index 588be36f..1dacbfe8 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -1538,7 +1538,7 @@ def api_add_obj_tags(tags=[], galaxy_tags=[], object_id=None, object_type="item" # r_serv_metadata.srem('tag:{}'.format(object_id), tag) # r_tags.srem('{}:{}'.format(object_type, tag), object_id) -def delete_tag(object_type, tag, object_id, obj_date=None): ################################ # TODO: +def delete_tag(object_type, tag, object_id, obj_date=None): ################################ # TODO: REMOVE ME # tag exist if is_obj_tagged(object_id, tag): if not obj_date: @@ -1613,6 +1613,11 @@ def _fix_tag_obj_id(date_from): print(tag) new_tag = tag.split(';')[0] print(new_tag) + r_tags.hdel(f'tag_metadata:{tag}', 'first_seen') + r_tags.hdel(f'tag_metadata:{tag}', 'last_seen') + r_tags.srem(f'list_tags:{obj_type}', tag) + r_tags.srem(f'list_tags:{obj_type}:', tag) + r_tags.srem(f'list_tags', tag) raw = get_obj_by_tags(obj_type, [tag], nb_obj=500000, date_from=date_from, date_to=date_to) if raw.get('tagged_obj', []): for obj_id in raw['tagged_obj']: From b6eb6c901653d597629946bf752b985cd2544c3c Mon Sep 17 00:00:00 2001 From: terrtia Date: Fri, 2 Feb 2024 14:48:19 +0100 Subject: [PATCH 227/238] fix: [crawler] fix capture None domain name --- bin/crawlers/Crawler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index 1f46e938..9d1514e8 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -230,7 +230,12 @@ class Crawler(AbstractModule): domain = task.get_domain() print(domain) if not domain: - raise Exception(f'Error: domain {domain}') + if self.debug: + raise Exception(f'Error: domain {domain} - task {task.uuid} - capture {capture.uuid}') + else: + self.logger.critical(f'Error: domain {domain} - task {task.uuid} - capture {capture.uuid}') + print(f'Error: domain {domain}') + return None self.domain = Domain(domain) self.original_domain = Domain(domain) From ff59dcf81d9d4075fe48987290c147eaea93761f Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 5 Feb 2024 09:57:53 +0100 Subject: [PATCH 228/238] fix: [LAUNCH] fix ENV error message --- bin/LAUNCH.sh | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 31761a94..b38e6b64 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -20,7 +20,7 @@ if [ -e "${DIR}/AILENV/bin/python" ]; then export AIL_VENV=${AIL_HOME}/AILENV/ . ./AILENV/bin/activate else - echo "Please make sure you have a AIL-framework environment, au revoir" + echo "Please make sure AILENV is installed" exit 1 fi @@ -31,15 +31,17 @@ export PATH=$AIL_KVROCKS:$PATH export PATH=$AIL_BIN:$PATH export PATH=$AIL_FLASK:$PATH -isredis=`screen -ls | egrep '[0-9]+.Redis_AIL' | cut -d. -f1` -isardb=`screen -ls | egrep '[0-9]+.ARDB_AIL' | cut -d. -f1` -iskvrocks=`screen -ls | egrep '[0-9]+.KVROCKS_AIL' | cut -d. -f1` -islogged=`screen -ls | egrep '[0-9]+.Logging_AIL' | cut -d. -f1` -is_ail_core=`screen -ls | egrep '[0-9]+.Core_AIL' | cut -d. -f1` -is_ail_2_ail=`screen -ls | egrep '[0-9]+.AIL_2_AIL' | cut -d. -f1` -isscripted=`screen -ls | egrep '[0-9]+.Script_AIL' | cut -d. -f1` -isflasked=`screen -ls | egrep '[0-9]+.Flask_AIL' | cut -d. -f1` -isfeeded=`screen -ls | egrep '[0-9]+.Feeder_Pystemon' | cut -d. -f1` +function check_screens { + isredis=`screen -ls | egrep '[0-9]+.Redis_AIL' | cut -d. -f1` + isardb=`screen -ls | egrep '[0-9]+.ARDB_AIL' | cut -d. -f1` + iskvrocks=`screen -ls | egrep '[0-9]+.KVROCKS_AIL' | cut -d. -f1` + islogged=`screen -ls | egrep '[0-9]+.Logging_AIL' | cut -d. -f1` + is_ail_core=`screen -ls | egrep '[0-9]+.Core_AIL' | cut -d. -f1` + is_ail_2_ail=`screen -ls | egrep '[0-9]+.AIL_2_AIL' | cut -d. -f1` + isscripted=`screen -ls | egrep '[0-9]+.Script_AIL' | cut -d. -f1` + isflasked=`screen -ls | egrep '[0-9]+.Flask_AIL' | cut -d. -f1` + isfeeded=`screen -ls | egrep '[0-9]+.Feeder_Pystemon' | cut -d. -f1` +} function helptext { echo -e $YELLOW" @@ -671,7 +673,7 @@ function menu_display { } #echo "$@" - +check_screens; while [ "$1" != "" ]; do case $1 in -l | --launchAuto ) launch_all "automatic"; @@ -694,6 +696,7 @@ while [ "$1" != "" ]; do ;; -r | --restart ) killall; sleep 0.1; + check_screens; launch_all "automatic"; ;; -ks | --killscript ) killscript; From c1529b217d2b75d71128f6de3f2965eaba303dea Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 5 Feb 2024 11:09:41 +0100 Subject: [PATCH 229/238] fix: [LAUNCH] fix killall --- bin/LAUNCH.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index b38e6b64..d0af627e 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -692,7 +692,8 @@ while [ "$1" != "" ]; do ;; --set_kvrocks_namespaces ) set_kvrocks_namespaces; ;; - -k | --killAll ) killall; + -k | --killAll ) check_screens; + killall; ;; -r | --restart ) killall; sleep 0.1; From 335d94cf791ae9e4308bc2db530da03d42907fdd Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 5 Feb 2024 11:21:59 +0100 Subject: [PATCH 230/238] chg: [requirement] bump flask requirement --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6db1d3dd..6fc9e511 100644 --- a/requirements.txt +++ b/requirements.txt @@ -71,7 +71,7 @@ pylibinjection>=0.2.4 phonenumbers>8.12.1 # Web -flask==2.3.3 +flask>=2.3.3 flask-login bcrypt>3.1.6 From 99fedf98559e552c7082cdf1b7be639d062c4439 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 5 Feb 2024 11:32:49 +0100 Subject: [PATCH 231/238] fix: [LAUNCH] update screen status --- bin/LAUNCH.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index d0af627e..f9c487e0 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -676,13 +676,16 @@ function menu_display { check_screens; while [ "$1" != "" ]; do case $1 in - -l | --launchAuto ) launch_all "automatic"; + -l | --launchAuto ) check_screens; + launch_all "automatic"; ;; - -lr | --launchRedis ) launch_redis; + -lr | --launchRedis ) check_screens; + launch_redis; ;; -la | --launchARDB ) launch_ardb; ;; - -lk | --launchKVROCKS ) launch_kvrocks; + -lk | --launchKVROCKS ) check_screens; + launch_kvrocks; ;; -lrv | --launchRedisVerify ) launch_redis; wait_until_redis_is_ready; @@ -700,7 +703,8 @@ while [ "$1" != "" ]; do check_screens; launch_all "automatic"; ;; - -ks | --killscript ) killscript; + -ks | --killscript ) check_screens; + killscript; ;; -m | --menu ) menu_display; ;; From aa56e716314a8b78a28489ed63adbdcab0e913cc Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 5 Feb 2024 14:10:19 +0100 Subject: [PATCH 232/238] fix: [language] crawled items, force gcld3 detection --- bin/lib/Language.py | 4 ++-- bin/lib/objects/Items.py | 4 ++-- bin/modules/Languages.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/lib/Language.py b/bin/lib/Language.py index 1fee96f7..041b0169 100755 --- a/bin/lib/Language.py +++ b/bin/lib/Language.py @@ -357,9 +357,9 @@ class LanguagesDetector: languages.append(language) return languages - def detect(self, content): + def detect(self, content, force_gcld3=False): # gcld3 - if len(content) >= 200 or not self.lt: + if len(content) >= 200 or not self.lt or force_gcld3: language = self.detect_gcld3(content) # libretranslate else: diff --git a/bin/lib/objects/Items.py b/bin/lib/objects/Items.py index 7b79749a..8204017d 100755 --- a/bin/lib/objects/Items.py +++ b/bin/lib/objects/Items.py @@ -339,9 +339,9 @@ class Item(AbstractObject): return {'nb': nb_line, 'max_length': max_length} # TODO RENAME ME - def get_languages(self, min_len=600, num_langs=3, min_proportion=0.2, min_probability=0.7): + def get_languages(self, min_len=600, num_langs=3, min_proportion=0.2, min_probability=0.7, force_gcld3=False): ld = LanguagesDetector(nb_langs=num_langs, min_proportion=min_proportion, min_probability=min_probability, min_len=min_len) - return ld.detect(self.get_content()) + return ld.detect(self.get_content(), force_gcld3=force_gcld3) def get_mimetype(self, content=None): if not content: diff --git a/bin/modules/Languages.py b/bin/modules/Languages.py index e1ce560a..bff7b0ba 100755 --- a/bin/modules/Languages.py +++ b/bin/modules/Languages.py @@ -30,7 +30,7 @@ class Languages(AbstractModule): if obj.type == 'item': if obj.is_crawled(): domain = Domain(obj.get_domain()) - for lang in obj.get_languages(min_probability=0.8): + for lang in obj.get_languages(min_probability=0.8, force_gcld3=True): print(lang) domain.add_language(lang) From 4c1d058e6d3bdd947b4ce27d28f15c61a8dd8c2c Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 5 Feb 2024 14:19:42 +0100 Subject: [PATCH 233/238] fix: [language] catch libretranslate exception --- bin/lib/Language.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/lib/Language.py b/bin/lib/Language.py index 041b0169..8052bf28 100755 --- a/bin/lib/Language.py +++ b/bin/lib/Language.py @@ -349,9 +349,12 @@ class LanguagesDetector: try: # [{"confidence": 0.6, "language": "en"}] resp = self.lt.detect(content) - except: # TODO ERROR MESSAGE - resp = [] + except Exception as e: # TODO ERROR MESSAGE + raise Exception(f'libretranslate error: {e}') + # resp = [] if resp: + if isinstance(resp, dict): + raise Exception(f'libretranslate error {resp}') for language in resp: if language.confidence >= self.min_probability: languages.append(language) From d84bc14b623f9705fc39ba334de5359be688a2a7 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 5 Feb 2024 16:22:39 +0100 Subject: [PATCH 234/238] chg: [HOWTO] Libretranslate Chat translation --- HOWTO.md | 72 ++++++++++++++++++++++++-------------------------------- 1 file changed, 31 insertions(+), 41 deletions(-) diff --git a/HOWTO.md b/HOWTO.md index b0539d9d..1042b677 100644 --- a/HOWTO.md +++ b/HOWTO.md @@ -1,17 +1,16 @@ -# Feeding, adding new features and contributing +# Feeding, Adding new features and Contributing -## [Documentation AIL Importers](./doc/README.md#ail-importers) +## [AIL Importers](./doc/README.md#ail-importers) -[Documentation AIL Importers](./doc/README.md#ail-importers) +Refer to the [AIL Importers Documentation](./doc/README.md#ail-importers) -## How to feed the AIL framework +## Feeding Data to AIL AIL is an analysis tool, not a collector! However, if you want to collect some pastes and feed them to AIL, the procedure is described below. Nevertheless, moderate your queries! 1. [AIL Importers](./doc/README.md#ail-importers) - 2. ZMQ: Be a collaborator of CIRCL and ask to access our feed. It will be sent to the static IP you are using for AIL. ## How to create a new module @@ -19,22 +18,16 @@ However, if you want to collect some pastes and feed them to AIL, the procedure To add a new processing or analysis module to AIL, follow these steps: 1. Add your module name in [./configs/modules.cfg](./configs/modules.cfg) and subscribe to at least one module at minimum (Usually, `Item`). - 2. Use [./bin/modules/modules/TemplateModule.py](./bin/modules/modules/TemplateModule.py) as a sample module and create a new file in bin/modules with the module name used in the `modules.cfg` configuration. -## How to contribute a module +## Contributions -Feel free to fork the code, play with it, make some patches or add additional analysis modules. +Contributions are welcome! Fork the repository, experiment with the code, and submit your modules or patches through a pull request. -To contribute your module, feel free to pull your contribution. +## Crawler - -## Additional information - -### Crawler - -In AIL, you can crawl websites and Tor hidden services. Don't forget to review the proxy configuration of your Tor client and especially if you enabled the SOCKS5 proxy +AIL supports crawling of websites and Tor hidden services. Ensure your Tor client's proxy configuration is correct, especially the SOCKS5 proxy settings. ### Installation @@ -45,38 +38,35 @@ In AIL, you can crawl websites and Tor hidden services. Don't forget to review t 1. Lacus URL: In the web interface, go to `Crawlers` > `Settings` and click on the Edit button -![Splash Manager Config](./doc/screenshots/lacus_config.png?raw=true "AIL Lacus Config") +![AIL Crawler Config](./doc/screenshots/lacus_config.png?raw=true "AIL Lacus Config") -![Splash Manager Config](./doc/screenshots/lacus_config_edit.png?raw=true "AIL Lacus Config") +![AIL Crawler Config Edis](./doc/screenshots/lacus_config_edit.png?raw=true "AIL Lacus Config") -2. Launch AIL Crawlers: +2. Number of Crawlers: Choose the number of crawlers you want to launch -![Splash Manager Nb Crawlers Config](./doc/screenshots/crawler_nb_captures.png?raw=true "AIL Lacus Nb Crawlers Config") +![Crawler Manager Nb Crawlers Config](./doc/screenshots/crawler_nb_captures.png?raw=true "AIL Lacus Nb Crawlers Config") -![Splash Manager Nb Crawlers Config](./doc/screenshots/crawler_nb_captures_edit.png?raw=true "AIL Lacus Nb Crawlers Config") +![Crawler Manager Nb Crawlers Config](./doc/screenshots/crawler_nb_captures_edit.png?raw=true "AIL Lacus Nb Crawlers Config") +## Chats Translation with LibreTranslate -### Kvrocks Migration ---------------------- -**Important Note: -We are currently working on a [migration script](https://github.com/ail-project/ail-framework/blob/master/update/v5.0/DB_KVROCKS_MIGRATION.py) to facilitate the migration to Kvrocks. -** +Chats message can be translated using [libretranslate](https://github.com/LibreTranslate/LibreTranslate), an open-source self-hosted machine translation. -Please note that the current version of this migration script only supports migrating the database on the same server. -(If you plan to migrate to another server, we will provide additional instructions in this section once the migration script is completed) +### Installation: +1. Install LibreTranslate by running the following command: +```bash +pip install libretranslate +``` +2. Run libretranslate: +```bash +libretranslate +``` + +### Configuration: +To enable LibreTranslate for chat translation, edit the LibreTranslate URL in the [./configs/core.cfg](./configs/core.cfg) file under the [Translation] section. +``` +[Translation] +libretranslate = http://127.0.0.1:5000 +``` -To migrate your database to Kvrocks: -1. Launch ARDB and Kvrocks -2. Pull from remote - ```shell - git checkout master - git pull - ``` -3. Launch the migration script: - ```shell - git checkout master - git pull - cd update/v5.0 - ./DB_KVROCKS_MIGRATION.py - ``` From 88f30833c2e72704e56e9aa519b6ee2381fa8fc1 Mon Sep 17 00:00:00 2001 From: terrtia Date: Mon, 5 Feb 2024 16:34:20 +0100 Subject: [PATCH 235/238] chg: [doc] add discord/telegram chats JSON fields --- doc/README.md | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/doc/README.md b/doc/README.md index bd891289..694610c6 100644 --- a/doc/README.md +++ b/doc/README.md @@ -189,9 +189,119 @@ from GHArchive, collect and feed AIL - [ail-feeder-leak](https://github.com/ail-project/ail-feeder-leak): Automates the process of feeding files to AIL, using data chunking to handle large files. - [ail-feeder-atom-rss](https://github.com/ail-project/ail-feeder-atom-rss) Atom and RSS feeder for AIL. - [ail-feeder-jsonlogs](https://github.com/ail-project/ail-feeder-jsonlogs) Aggregate JSON log lines and pushes them to AIL. + +### AIL Chats Feeders List: - [ail-feeder-discord](https://github.com/ail-project/ail-feeder-discord) Discord Feeder. - [ail-feeder-telegram](https://github.com/ail-project/ail-feeder-telegram) Telegram Channels and User Feeder. +### Chats Message + +Overview of the JSON fields used by the Chat feeder. + +``` +{ + "data": "New NFT Scam available," + "meta": { + "chat": { + "date": { + "datestamp": "2023-01-10 08:19:16", + "timestamp": 1673870217.0, + "timezone": "UTC" + }, + "icon": "AAAAAAAA", + "id": 123456, + "info": "", + "name": "NFT legit", + "subchannel": { + "date": { + "datestamp": "2023-08-10 08:19:18", + "timestamp": 1691655558.0, + "timezone": "UTC" + }, + "id": 285, + "name": "Market" + }, + }, + "date": { + "datestamp": "2024-02-01 13:43:46", + "timestamp": 1707139999.0, + "timezone": "UTC" + }, + "id": 16, + "reply_to": { + "message_id": 12 + }, + "sender": { + "first_name": "nftmaster", + "icon": "AAAAAAAA", + "id": 5684, + "info": "best legit NFT vendor", + "username": "nft_best" + }, + "type": "message" + }, + "source": "ail_feeder_telegram", + "source-uuid": "9cde0855-248b-4439-b964-0495b9b2b8bb" +} +``` + +#### 1. "data" +- Content of the message. + +#### 2. "meta" +- Provides metadata about the message. + + ##### "type": + - Indicates the type of message. It can be either "message" or "image". + + ##### "id": + - The unique identifier of the message. + + ##### "date": + - Represents the timestamp of the message. + - "datestamp": The date in the format "YYYY-MM-DD HH:MM:SS". + - "timestamp": The timestamp representing the date and time. + - "timezone": The timezone in which the date and time are specified (e.g., "UTC"). + + ##### "reply_to": + - The unique identifier of a message to which this message is a reply (optional). + - "message_id": The unique identifier of the replied message. + + ##### "sender": + - Contains information about the sender of the message. + - "id": The unique identifier for the sender. + - "info": Additional information about the sender (optional). + - "username": The sender's username (optional). + - "firstname": The sender's firstname (optional). + - "lastname": The sender's lastname (optional). + - "phone": The sender's phone (optional). + + ##### "chat": + - Contains information about the chat where the message was sent. + - "date": The chat creation date. + - "datestamp": The date in the format "YYYY-MM-DD HH:MM:SS". + - "timestamp": The timestamp representing the date and time. + - "timezone": The timezone in which the date and time are specified (e.g., "UTC"). + - "icon": The icon associated with the chat (optional). + - "id": The unique identifier of the chat. + - "info": Chat description/info (optional). + - "name": The name of the chat. + - "username": The username of the chat (optional). + - "subchannel": If this message is posted in a subchannel within the chat (optional). + - "date": The subchannel creation date. + - "datestamp": The date in the format "YYYY-MM-DD HH:MM:SS". + - "timestamp": The timestamp representing the date and time. + - "timezone": The timezone in which the date and time are specified (e.g., "UTC"). + - "id": The unique identifier of the subchannel. + - "name": The name of the subchannel (optional). + +#### 3. "source" +- Indicates the feeder name. + +#### 4. "source-uuid" +- The UUID associated with the source. + + #### Example: Feeding AIL with Conti leaks ```python From 4168d07118635a0e307e68122a10f1abf4648db4 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 6 Feb 2024 11:13:45 +0100 Subject: [PATCH 236/238] fix: [chats] fix chats image importer --- bin/importer/feeders/Default.py | 3 ++- bin/lib/objects/ChatThreads.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/importer/feeders/Default.py b/bin/importer/feeders/Default.py index 25b3d13b..ca0861a1 100755 --- a/bin/importer/feeders/Default.py +++ b/bin/importer/feeders/Default.py @@ -63,7 +63,8 @@ class DefaultFeeder: return self.json_data.get('data') def get_obj_type(self): - return self.json_data.get('type', 'item') + meta = self.get_json_meta() + return meta.get('type', 'item') ## OVERWRITE ME ## def get_obj(self): diff --git a/bin/lib/objects/ChatThreads.py b/bin/lib/objects/ChatThreads.py index 0609faab..a2559cc4 100755 --- a/bin/lib/objects/ChatThreads.py +++ b/bin/lib/objects/ChatThreads.py @@ -106,7 +106,7 @@ def create(thread_id, chat_instance, chat_id, subchannel_id, message_id, contain new_thread_id = f'{chat_id}/{subchannel_id}/{thread_id}' thread = ChatThread(new_thread_id, chat_instance) - if not thread.exists(): + if not thread.is_children(): thread.create(container_obj, message_id) return thread From 38a918e48580500c18896e6b7af30e49976e1ea2 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 6 Feb 2024 11:56:39 +0100 Subject: [PATCH 237/238] fix: [flask] fix escape import --- bin/core/ail_2_ail.py | 2 +- bin/lib/Investigations.py | 2 +- bin/lib/Tracker.py | 2 +- var/www/blueprints/hunters.py | 2 +- var/www/modules/restApi/Flask_restApi.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/core/ail_2_ail.py b/bin/core/ail_2_ail.py index 780698fc..30f6f92e 100755 --- a/bin/core/ail_2_ail.py +++ b/bin/core/ail_2_ail.py @@ -11,7 +11,7 @@ import uuid import subprocess -from flask import escape +from markupsafe import escape sys.path.append(os.environ['AIL_BIN']) ################################## diff --git a/bin/lib/Investigations.py b/bin/lib/Investigations.py index 9c6def0f..855beb49 100755 --- a/bin/lib/Investigations.py +++ b/bin/lib/Investigations.py @@ -16,7 +16,7 @@ import time import uuid from enum import Enum -from flask import escape +from markupsafe import escape sys.path.append(os.environ['AIL_BIN']) ################################## diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 9c4702ae..048d7219 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -16,7 +16,7 @@ from ail_typo_squatting import runAll import math from collections import defaultdict -from flask import escape +from markupsafe import escape from textblob import TextBlob from nltk.tokenize import RegexpTokenizer diff --git a/var/www/blueprints/hunters.py b/var/www/blueprints/hunters.py index 9a2b6c3e..1002f6e0 100644 --- a/var/www/blueprints/hunters.py +++ b/var/www/blueprints/hunters.py @@ -9,7 +9,7 @@ import os import sys import json -from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response, escape, abort +from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort from flask_login import login_required, current_user, login_user, logout_user sys.path.append('modules') diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index 1b41bd4e..cc13d703 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -27,7 +27,7 @@ from packages import Import_helper from importer.FeederImporter import api_add_json_feeder_to_queue -from flask import jsonify, request, Blueprint, redirect, url_for, Response, escape +from flask import jsonify, request, Blueprint, redirect, url_for, Response from functools import wraps From 304afd00aa6ed3fa8c323ef5ee6e06e680b553d3 Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 7 Feb 2024 10:32:18 +0100 Subject: [PATCH 238/238] chg: [exif] add debug --- bin/modules/Exif.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/modules/Exif.py b/bin/modules/Exif.py index 190874d9..352de549 100755 --- a/bin/modules/Exif.py +++ b/bin/modules/Exif.py @@ -42,13 +42,17 @@ class Exif(AbstractModule): img_exif = img.getexif() print(img_exif) if img_exif: + self.logger.critical(f'Exif: {self.get_obj().id}') gps = img_exif.get(34853) print(gps) + self.logger.critical(f'gps: {gps}') for key, val in img_exif.items(): if key in ExifTags.TAGS: print(f'{ExifTags.TAGS[key]}:{val}') + self.logger.critical(f'{ExifTags.TAGS[key]}:{val}') else: print(f'{key}:{val}') + self.logger.critical(f'{key}:{val}') sys.exit(0) # tag = 'infoleak:automatic-detection="cve"'