diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index e3aaf503..f62e6194 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -280,6 +280,8 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "OcrExtractor" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./OcrExtractor.py; read x" sleep 0.1 + screen -S "Script_AIL" -X screen -t "QrCodeReader" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./QrCodeReader.py; read x" + sleep 0.1 ################################## # TRACKERS MODULES # diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index 66a55ad9..48da4a08 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -359,6 +359,7 @@ class Crawler(AbstractModule): # Create Correlations screenshot.add_correlation('item', '', item_id) screenshot.add_correlation('domain', '', self.domain.id) + self.add_message_to_queue(obj=screenshot, queue='Images') # HAR if self.har: if 'har' in entries and entries.get('har'): diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 9a86b01b..328c484f 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -17,15 +17,15 @@ r_object = config_loader.get_db_conn("Kvrocks_Objects") config_loader = None AIL_OBJECTS = sorted({'chat', 'chat-subchannel', 'chat-thread', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', - 'domain', 'etag', 'favicon', 'file-name', 'hhhash', - 'item', 'image', 'message', 'ocr', 'pgp', 'screenshot', 'title', 'user-account', 'username'}) + 'domain', 'etag', 'favicon', 'file-name', 'hhhash','item', 'image', 'message', 'ocr', 'pgp', + 'qrcode', 'screenshot', 'title', 'user-account', 'username'}) AIL_OBJECTS_WITH_SUBTYPES = {'chat', 'chat-subchannel', 'cryptocurrency', 'pgp', 'username', 'user-account'} # TODO by object TYPE ???? AIL_OBJECTS_CORRELATIONS_DEFAULT = sorted({'chat', 'chat-subchannel', 'chat-thread', 'cve', 'cryptocurrency', 'decoded', - 'domain', 'favicon', 'file-name', - 'item', 'image', 'message', 'ocr', 'pgp', 'screenshot', 'title', 'user-account', 'username'}) + 'domain', 'favicon', 'file-name', 'item', 'image', 'message', 'ocr', 'pgp', + 'qrcode', 'screenshot', 'title', 'user-account', 'username'}) def get_ail_uuid(): ail_uuid = r_serv_db.get('ail:uuid') @@ -86,10 +86,10 @@ def get_default_correlation_objects(): return AIL_OBJECTS_CORRELATIONS_DEFAULT def get_obj_queued(): - return ['item', 'image', 'message', 'ocr'] + return ['item', 'image', 'message', 'ocr', 'qrcode'] def get_objects_tracked(): - return ['decoded', 'item', 'pgp', 'message', 'ocr', 'title'] + return ['decoded', 'item', 'pgp', 'message', 'ocr', 'qrcode', 'title'] def get_objects_retro_hunted(): return ['decoded', 'item', 'message'] diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index d8a02f85..662ed3a0 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -43,22 +43,23 @@ config_loader = None CORRELATION_TYPES_BY_OBJ = { "chat": ["chat-subchannel", "chat-thread", "image", "message", "ocr", "user-account"], # message or direct correlation like cve, bitcoin, ... ??? "chat-subchannel": ["chat", "chat-thread", "image", "message", "ocr", "user-account"], - "chat-thread": ["chat", "chat-subchannel", "image", "message", "ocr", "user-account"], # TODO user account + "chat-thread": ["chat", "chat-subchannel", "image", "message", "ocr", "user-account"], "cookie-name": ["domain"], - "cryptocurrency": ["domain", "item", "message", "ocr"], - "cve": ["domain", "item", "message", "ocr"], - "decoded": ["domain", "item", "message", "ocr"], + "cryptocurrency": ["domain", "item", "message", "ocr", "qrcode"], + "cve": ["domain", "item", "message", "ocr", "qrcode"], + "decoded": ["domain", "item", "message", "ocr", "qrcode"], "domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"], "etag": ["domain"], "favicon": ["domain", "item"], # TODO Decoded "file-name": ["chat", "message"], "hhhash": ["domain"], - "image": ["chat", "chat-subchannel", "chat-thread", "message", "ocr", "user-account"], # TODO subchannel + threads ???? + "image": ["chat", "chat-subchannel", "chat-thread", "message", "ocr", "qrcode", "user-account"], # TODO subchannel + threads ???? "item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], # chat ??? "message": ["chat", "chat-subchannel", "chat-thread", "cve", "cryptocurrency", "decoded", "file-name", "image", "ocr", "pgp", "user-account"], "ocr": ["chat", "chat-subchannel", "chat-thread", "cve", "cryptocurrency", "decoded", "image", "message", "pgp", "user-account"], "pgp": ["domain", "item", "message", "ocr"], - "screenshot": ["domain", "item"], + "qrcode": ["chat", "cve", "cryptocurrency", "decoded", "domain", "image", "message", "screenshot"], # "chat-subchannel", "chat-thread" ????? + "screenshot": ["domain", "item", "qrcode"], "title": ["domain", "item"], "user-account": ["chat", "chat-subchannel", "chat-thread", "image", "message", "ocr", "username"], "username": ["domain", "item", "message", "user-account"], @@ -67,7 +68,9 @@ CORRELATION_TYPES_BY_OBJ = { def get_obj_correl_types(obj_type): return CORRELATION_TYPES_BY_OBJ.get(obj_type) -def sanityze_obj_correl_types(obj_type, correl_types): +def sanityze_obj_correl_types(obj_type, correl_types, sanityze=True): + if not sanityze: + return correl_types obj_correl_types = get_obj_correl_types(obj_type) if correl_types: correl_types = set(correl_types).intersection(obj_correl_types) @@ -99,11 +102,11 @@ def get_correlation_by_correl_type(obj_type, subtype, obj_id, correl_type, unpac else: return correl -def get_correlations(obj_type, subtype, obj_id, filter_types=[], unpack=False): +def get_correlations(obj_type, subtype, obj_id, filter_types=[], unpack=False, sanityze=True): if subtype is None: subtype = '' obj_correlations = {} - filter_types = sanityze_obj_correl_types(obj_type, filter_types) + filter_types = sanityze_obj_correl_types(obj_type, filter_types, sanityze=sanityze) for correl_type in filter_types: obj_correlations[correl_type] = get_correlation_by_correl_type(obj_type, subtype, obj_id, correl_type, unpack=unpack) diff --git a/bin/lib/objects/QrCodes.py b/bin/lib/objects/QrCodes.py new file mode 100755 index 00000000..69b8f546 --- /dev/null +++ b/bin/lib/objects/QrCodes.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys + +from hashlib import sha256 +from pymisp import MISPObject + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects +from lib.ConfigLoader import ConfigLoader +from packages import Date +# from lib.data_retention_engine import update_obj_date, get_obj_date_first + +from flask import url_for + +config_loader = ConfigLoader() +r_object = config_loader.get_db_conn("Kvrocks_Objects") +r_cache = config_loader.get_redis_conn("Redis_Cache") +baseurl = config_loader.get_config_str("Notifications", "ail_domain") +IMAGE_FOLDER = config_loader.get_files_directory('images') +config_loader = None + +# SET x1,y1:x2,y2:x3,y3:x4,y4:extracted_text + +class Qrcode(AbstractDaterangeObject): + """ + AIL Message Object. (strings) + """ + + def __init__(self, id): + super(Qrcode, self).__init__('qrcode', id) + + def get_content(self, r_type='str'): + """ + Returns content + """ + global_id = self.get_global_id() + content = r_cache.get(f'content:{global_id}') + if not content: + content = self._get_field('content') + # Set Cache + if content: + global_id = self.get_global_id() + r_cache.set(f'content:{global_id}', content) + r_cache.expire(f'content:{global_id}', 300) + if r_type == 'str': + return content + elif r_type == 'bytes': + if content: + return content.encode() + + def get_date(self): # TODO + return Date.get_today_date_str() + + def get_nb_seen(self): + return self.get_nb_correlation('image') + self.get_nb_correlation('screenshot') + + def get_source(self): # TODO + """ + Returns source/feeder name + """ + return 'qrcode' + + def get_basename(self): # TODO + return 'qrcode' + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', type=self.type, id=self.id) + else: + url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}' + return url + + def get_svg_icon(self): # TODO + return {'style': 'fas', 'icon': '\uf029', 'color': 'yellow', 'radius': 5} + + def get_misp_object(self): # TODO + pass + # obj = MISPObject('instant-message', standalone=True) + # obj_date = self.get_date() + # if obj_date: + # obj.first_seen = obj_date + # else: + # self.logger.warning( + # f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={obj_date}') + # + # # obj_attrs = [obj.add_attribute('first-seen', value=obj_date), + # # obj.add_attribute('raw-data', value=self.id, data=self.get_raw_content()), + # # obj.add_attribute('sensor', value=get_ail_uuid())] + # obj_attrs = [] + # for obj_attr in obj_attrs: + # for tag in self.get_tags(): + # obj_attr.add_tag(tag) + # return obj + + # options: set of optional meta fields + def get_meta(self, options=None): + """ + :type options: set + """ + if options is None: + options = set() + meta = self._get_meta(options=options) + meta['tags'] = self.get_tags() + meta['content'] = self.get_content() + + # optional meta fields + if 'investigations' in options: + meta['investigations'] = self.get_investigations() + if 'link' in options: + meta['link'] = self.get_link(flask_context=True) + if 'icon' in options: + meta['svg_icon'] = self.get_svg_icon() + return meta + + def create(self, content, im_obj, tags=[]): + self._set_field('content', content) + if im_obj.type == 'screenshot': + for date in im_obj.get_dates(): + self._add(date, None) + else: + self._copy_from(im_obj.type, im_obj.get_id()) + + for tag in tags: + self.add_tag(tag) + return self.id + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + r_object.delete(f'qrcode:{self.id}') + + +def create(content, im_obj, tags=[]): + if content: + obj_id = sha256(content.encode()).hexdigest() + obj = Qrcode(obj_id) + if not obj.exists(): + obj.create(content, im_obj, tags=tags) + return obj + +class Qrcodes(AbstractDaterangeObjects): + """ + Qrcodes Objects + """ + def __init__(self): + super().__init__('qrcode', Qrcode) + + def sanitize_id_to_search(self, name_to_search): + return name_to_search # TODO + + +#### API #### +def api_get_qrcode(obj_id): + obj = Qrcode(obj_id) + if not obj.exists(): + return {"status": "error", "reason": "Unknown qrcode"}, 404 + meta = obj.get_meta({'content', 'icon', 'link'}) + return meta, 200 diff --git a/bin/lib/objects/Screenshots.py b/bin/lib/objects/Screenshots.py index 26f8543f..19c13657 100755 --- a/bin/lib/objects/Screenshots.py +++ b/bin/lib/objects/Screenshots.py @@ -47,6 +47,22 @@ class Screenshot(AbstractObject): def exists(self): return os.path.isfile(self.get_filepath()) + def get_last_seen(self): + dates = self.get_dates() + date = 0 + for d in dates: + if int(d) > int(date): + date = d + return date + + def get_dates(self): + dates = [] + for i_id in self.get_correlation('item').get('item'): + if i_id.startswith(':crawled'): + i_id = i_id.split('/', 4) + dates.append(f'{i_id[1]}{i_id[2]}{i_id[3]}') + return dates + def get_link(self, flask_context=False): if flask_context: url = url_for('correlation.show_correlation', type=self.type, id=self.id) @@ -116,6 +132,14 @@ def get_all_screenshots(): screenshots.append(screenshot_id) return screenshots +def get_screenshots_obj_iterator(filters=[]): + screenshot_dir = os.path.join(os.environ['AIL_HOME'], SCREENSHOT_FOLDER) + for root, dirs, files in os.walk(screenshot_dir): + for file in files: + screenshot_path = f'{root}{file}' + screenshot_id = screenshot_path.replace(SCREENSHOT_FOLDER, '').replace('/', '')[:-4] + yield Screenshot(screenshot_id) + # FIXME STR SIZE LIMIT def create_screenshot(content, size_limit=5000000, b64=True, force=False): size = (len(content)*3) / 4 @@ -155,5 +179,6 @@ def search_screenshots_by_name(name_to_search, r_pos=False): # if __name__ == '__main__': -# name_to_search = '29ba' -# print(search_screenshots_by_name(name_to_search)) +# obj_id = '' +# obj = Screenshot(obj_id) +# obj.get_last_seen() diff --git a/bin/lib/objects/abstract_daterange_object.py b/bin/lib/objects/abstract_daterange_object.py index 68d03703..c6f2d1aa 100755 --- a/bin/lib/objects/abstract_daterange_object.py +++ b/bin/lib/objects/abstract_daterange_object.py @@ -132,7 +132,7 @@ class AbstractDaterangeObject(AbstractObject, ABC): last_seen = r_object.hget(f'meta:{obj_type}:{obj_id}', 'last_seen') if first_seen and last_seen: for date in Date.get_daterange(first_seen, last_seen): - nb = r_object.zscore(f'{obj_type}:date:{date}', self.id) + nb = r_object.zscore(f'{obj_type}:date:{date}', obj_id) if nb: r_object.zincrby(f'{self.type}:date:{date}', nb, self.id) update_obj_date(first_seen, self.type) diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index 42c8f0ad..7156464a 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -235,7 +235,7 @@ class AbstractObject(ABC): """ Get object correlation """ - return get_correlations(self.type, self.subtype, self.id, filter_types=[obj_type]) + return get_correlations(self.type, self.subtype, self.id, filter_types=[obj_type], sanityze=False) def get_first_correlation(self, obj_type): correlation = self.get_correlation(obj_type) diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index b1089975..82eb2bb6 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -36,7 +36,8 @@ from lib.objects import Images from lib.objects import Messages from lib.objects import Ocrs from lib.objects import Pgps -from lib.objects.Screenshots import Screenshot +from lib.objects import QrCodes +from lib.objects import Screenshots from lib.objects import Titles from lib.objects import UsersAccount from lib.objects import Usernames @@ -98,8 +99,10 @@ def get_object(obj_type, subtype, obj_id): return Messages.Message(obj_id) elif obj_type == 'ocr': return Ocrs.Ocr(obj_id) + elif obj_type == 'qrcode': + return QrCodes.Qrcode(obj_id) elif obj_type == 'screenshot': - return Screenshot(obj_id) + return Screenshots.Screenshot(obj_id) elif obj_type == 'title': return Titles.Title(obj_id) else: @@ -330,6 +333,8 @@ def obj_iterator(obj_type, filters): return get_all_decodeds_objects(filters=filters) elif obj_type == 'image': return Images.get_all_images_objects(filters=filters) + elif obj_type == 'screenshot': + return Screenshots.get_screenshots_obj_iterator(filters=filters) elif obj_type == 'item': return get_all_items_objects(filters=filters) elif obj_type == 'pgp': @@ -616,4 +621,4 @@ def get_chat_relationships_mentions_cord_graph(obj_type, subtype, obj_id): # print(r) # res = get_obj_correlations_objs('username', 'telegram', 'corona', lvl=100) - # print(res) \ No newline at end of file + # print(res) diff --git a/bin/modules/Categ.py b/bin/modules/Categ.py index b3948982..3cfdadb7 100755 --- a/bin/modules/Categ.py +++ b/bin/modules/Categ.py @@ -89,7 +89,7 @@ class Categ(AbstractModule): # Search for pattern categories in obj content for categ, pattern in self.categ_words: - if obj.type == 'message' or obj.type == 'ocr': + if obj.type == 'message' or obj.type == 'ocr' or obj.type == 'qrcode': self.add_message_to_queue(message='0', queue=categ) else: diff --git a/bin/modules/Global.py b/bin/modules/Global.py index 0dfafde1..f5bd5d30 100755 --- a/bin/modules/Global.py +++ b/bin/modules/Global.py @@ -133,6 +133,7 @@ class Global(AbstractModule): self.add_message_to_queue(obj=self.obj, queue='Item') elif self.obj.type == 'image': self.add_message_to_queue(obj=self.obj, queue='Image', message=message) + self.add_message_to_queue(obj=self.obj, queue='Images', message=message) else: self.logger.critical(f"Empty obj: {self.obj} {message} not processed") diff --git a/bin/modules/Mail.py b/bin/modules/Mail.py index 29477ed2..33932cb3 100755 --- a/bin/modules/Mail.py +++ b/bin/modules/Mail.py @@ -135,10 +135,10 @@ class Mail(AbstractModule): # # TODO: sanitize mails def compute(self, message): score = message - item = self.get_obj() + obj = self.get_obj() item_date = item.get_date() - mails = self.regex_findall(self.email_regex, item.id, item.get_content()) + mails = self.regex_findall(self.email_regex, obj.id, obj.get_content()) mxdomains_email = {} for mail in mails: mxdomain = mail.rsplit('@', 1)[1].lower() diff --git a/bin/modules/QrCodeReader.py b/bin/modules/QrCodeReader.py new file mode 100755 index 00000000..a0058eab --- /dev/null +++ b/bin/modules/QrCodeReader.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +The OcrExtractor Module +====================== + +""" + +################################## +# Import External packages +################################## +import cv2 +import os +import sys + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from modules.abstract_module import AbstractModule +from lib.ConfigLoader import ConfigLoader +from lib.objects import QrCodes + + +class QrCodeReader(AbstractModule): + """ + QrCodeReader for AIL framework + """ + + def __init__(self): + super(QrCodeReader, self).__init__() + + # Waiting time in seconds between to message processed + self.pending_seconds = 1 + + config_loader = ConfigLoader() + self.r_cache = config_loader.get_redis_conn("Redis_Cache") + + # Send module state to logs + self.logger.info(f'Module {self.module_name} initialized') + + def is_cached(self): + return self.r_cache.exists(f'qrcode:no:{self.obj.type}:{self.obj.id}') + + def add_to_cache(self): + self.r_cache.setex(f'qrcode:no:{self.obj.type}:{self.obj.id}', 86400, 0) + + def extract_qrcode(self, path): + detector = cv2.QRCodeDetector() # TODO Move me in init ??? + image = cv2.imread(path) + + # multiple extraction + try: + qr_found, contents, qarray, _ = detector.detectAndDecodeMulti(image) + if qr_found: + return contents + else: + # simple extraction + content, box, _ = detector.detectAndDecode(image) + if content: + return [content] + else: + return [] + except cv2.error as e: + self.logger.error(f'{e}, {self.obj.get_global_id()}') + + def compute(self, message): + obj = self.get_obj() + + if self.is_cached(): + return None + + if obj.type == 'image': + if self.obj.is_gif(): + self.logger.warning(f'Ignoring GIF: {self.obj.id}') + return None + + # image - screenshot + path = self.obj.get_filepath() + contents = self.extract_qrcode(path) + if not contents: + # print('no qr code detected') + self.add_to_cache() + return None + + for content in contents: + print(content) + qr_code = QrCodes.create(content, self.obj) # copy screenshot + image daterange + qr_code.add(qr_code.get_date(), self.obj) + + for obj_type in ['chat', 'domain', 'message']: # TODO ITEM ??? + for c_id in self.obj.get_correlation(obj_type).get(obj_type, []): + o_subtype, o_id = c_id.split(':', 1) + qr_code.add_correlation(obj_type, o_subtype, o_id) + + tag = 'infoleak:automatic-detection="qrcode"' + self.add_message_to_queue(obj=self.obj, message=tag, queue='Tags') + + # TODO only if new ??? + self.add_message_to_queue(obj=qr_code, queue='Item') + + +if __name__ == '__main__': + module = QrCodeReader() + module.run() + # from lib.objects.Images import Image + # module.obj = Image('8a690f4d09509dbfe52a6fb139db500b16b3d5f07e22617944752c4d4885737c') + # module.compute(None) diff --git a/configs/modules.cfg b/configs/modules.cfg index 298edb63..a559f626 100644 --- a/configs/modules.cfg +++ b/configs/modules.cfg @@ -1,7 +1,7 @@ ######## IMPORTERS ######## [Crawler] -publish = Importers,Tags +publish = Importers,Tags,Images [ZMQModuleImporter] publish = Importers @@ -29,7 +29,7 @@ publish = Importers,Tags [Global] subscribe = SaveObj -publish = Item,Image +publish = Item,Image,Images ######## ITEM + MESSAGE ######## @@ -156,7 +156,7 @@ publish = Tags #[Sync_module] #publish = Sync -######## IMAGE ######## +######## IMAGE ######## image [Exif] subscribe = Image @@ -166,6 +166,12 @@ publish = Tags subscribe = Image publish = Item +######## IMAGES ######## images + screenshots + +[QrCodeReader] +subscribe = Images +publish = Item,Tags + ######## CORE ######## [Tags] diff --git a/tools/reprocess_objects.py b/tools/reprocess_objects.py index b41f59e3..f6c33236 100755 --- a/tools/reprocess_objects.py +++ b/tools/reprocess_objects.py @@ -31,10 +31,12 @@ from lib.objects import ail_objects from modules.Languages import Languages from modules.OcrExtractor import OcrExtractor +from modules.QrCodeReader import QrCodeReader MODULES = { 'Languages': Languages, - 'OcrExtractor': OcrExtractor + 'OcrExtractor': OcrExtractor, + 'QrCodeReader': QrCodeReader } @@ -68,7 +70,7 @@ if __name__ == "__main__": obj_type = args.type if not is_object_type(obj_type): raise Exception(f'Invalid Object Type: {obj_type}') - if obj_type not in ['image', 'item', 'message']: + if obj_type not in ['image', 'item', 'message', 'screenshot']: raise Exception(f'Currently not supported Object Type: {obj_type}') modulename = args.module diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 7e6aa64e..c2fe5bdf 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -57,6 +57,7 @@ from blueprints.objects_hhhash import objects_hhhash from blueprints.chats_explorer import chats_explorer from blueprints.objects_image import objects_image from blueprints.objects_ocr import objects_ocr +from blueprints.objects_qrcode import objects_qrcode from blueprints.objects_favicon import objects_favicon from blueprints.api_rest import api_rest @@ -140,6 +141,7 @@ app.register_blueprint(objects_hhhash, url_prefix=baseUrl) app.register_blueprint(chats_explorer, url_prefix=baseUrl) app.register_blueprint(objects_image, url_prefix=baseUrl) app.register_blueprint(objects_ocr, url_prefix=baseUrl) +app.register_blueprint(objects_qrcode, url_prefix=baseUrl) app.register_blueprint(objects_favicon, url_prefix=baseUrl) app.register_blueprint(api_rest, url_prefix=baseUrl) diff --git a/var/www/blueprints/objects_qrcode.py b/var/www/blueprints/objects_qrcode.py new file mode 100644 index 00000000..1061aee9 --- /dev/null +++ b/var/www/blueprints/objects_qrcode.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... +''' + +import json +import os +import sys + +from io import BytesIO + +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file, send_from_directory +from flask_login import login_required + +# Import Role_Manager +from Role_Manager import login_admin, login_read_only, no_cache + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import Language +from lib import Tag +from lib.objects import QrCodes + +from packages import Date + +# ============ BLUEPRINT ============ +objects_qrcode = Blueprint('objects_qrcode', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/qrcode')) + +# ============ VARIABLES ============ +bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] + +def create_json_response(data, status_code): + return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code + +# ============ FUNCTIONS ============ + +@objects_qrcode.route("/objects/qrcodes", methods=['GET']) +@login_required +@login_read_only +def objects_qrcodes(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + show_objects = request.args.get('show_objects') + date = Date.sanitise_date_range(date_from, date_to) + date_from = date['date_from'] + date_to = date['date_to'] + + if show_objects: + dict_objects = QrCodes.Qrcodes().api_get_meta_by_daterange(date_from, date_to) + else: + dict_objects = {} + + return render_template("QrcodeDaterange.html", date_from=date_from, date_to=date_to, + dict_objects=dict_objects, show_objects=show_objects) + + +@objects_qrcode.route("/objects/qrcodes/post", methods=['POST']) +@login_required +@login_read_only +def objects_qrcodes_post(): + date_from = request.form.get('date_from') + date_to = request.form.get('date_to') + show_objects = request.form.get('show_objects') + return redirect(url_for('objects_qrcode.objects_qrcodes', date_from=date_from, date_to=date_to, show_objects=show_objects)) + + +@objects_qrcode.route("/objects/qrcodes/range/json", methods=['GET']) +@login_required +@login_read_only +def objects_qrcodes_range_json(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + date = Date.sanitise_date_range(date_from, date_to) + date_from = date['date_from'] + date_to = date['date_to'] + return jsonify(QrCodes.Qrcodes().api_get_chart_nb_by_daterange(date_from, date_to)) + + +@objects_qrcode.route("/objects/qrcodes", methods=['GET']) +@login_required +@login_read_only +def object_qrcode(): + obj_id = request.args.get('id') + meta = QrCodes.api_get_qrcode(obj_id) + if meta[1] != 200: + return create_json_response(meta[0], meta[1]) + else: + meta = meta[0] + languages = Language.get_translation_languages() + return render_template("ShowQrcode.html", meta=meta, + bootstrap_label=bootstrap_label, + ail_tags=Tag.get_modal_add_tags(meta['id'], meta['type'], meta['subtype']), + translation_languages=languages, translation_target=target) + + +# ============= ROUTES ============== + diff --git a/var/www/blueprints/tags_ui.py b/var/www/blueprints/tags_ui.py index 0c000219..6e5cbea5 100644 --- a/var/www/blueprints/tags_ui.py +++ b/var/www/blueprints/tags_ui.py @@ -299,7 +299,6 @@ def tags_search_messages(): def tags_search_images(): object_type = 'image' dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s"} - dict_tagged['date'] = Date.sanitise_date_range('', '', separator='-') return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged) @tags_ui.route('/tag/search/ocr') @@ -308,7 +307,6 @@ def tags_search_images(): def tags_search_ocrs(): object_type = 'ocr' dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s"} - dict_tagged['date'] = Date.sanitise_date_range('', '', separator='-') return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged) @tags_ui.route('/tag/search/domain') diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 693a0ac0..8012f66c 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -132,6 +132,8 @@ {% include 'chats_explorer/card_image.html' %} {% elif dict_object["object_type"] == "ocr" %} {% include 'objects/ocr/card_ocr.html' %} + {% elif dict_object["object_type"] == "qrcode" %} + {% include 'objects/qrcode/card_qrcode.html' %} {% elif dict_object["object_type"] == "item" %} {% include 'correlation/metadata_card_item.html' %} {% elif dict_object["object_type"] == "favicon" %} @@ -319,6 +321,10 @@ +
+ + +
diff --git a/var/www/templates/hunter/tracker_add.html b/var/www/templates/hunter/tracker_add.html index 6e0d8d58..02279492 100644 --- a/var/www/templates/hunter/tracker_add.html +++ b/var/www/templates/hunter/tracker_add.html @@ -140,9 +140,13 @@
+
+ + +
- +
{#
#} diff --git a/var/www/templates/objects/ocr/QrcodeDaterange.html b/var/www/templates/objects/ocr/OcrDaterange.html similarity index 100% rename from var/www/templates/objects/ocr/QrcodeDaterange.html rename to var/www/templates/objects/ocr/OcrDaterange.html diff --git a/var/www/templates/objects/ocr/ShowQrcode.html b/var/www/templates/objects/ocr/ShowOcr.html similarity index 100% rename from var/www/templates/objects/ocr/ShowQrcode.html rename to var/www/templates/objects/ocr/ShowOcr.html diff --git a/var/www/templates/objects/ocr/card_ocr.html b/var/www/templates/objects/ocr/card_ocr.html new file mode 100644 index 00000000..3e7292a6 --- /dev/null +++ b/var/www/templates/objects/ocr/card_ocr.html @@ -0,0 +1,167 @@ + + + +{% with modal_add_tags=ail_tags %} + {% include 'modals/add_tags.html' %} +{% endwith %} + +{% include 'modals/edit_tag.html' %} + + + +
+
+

{{ meta["id"] }} :

+
    +
  • + + + + + + + + +
    + + + + {{ meta["svg_icon"]["icon"] }} + + + {{ meta['type'] }} + First Seen: {% if meta['first_seen'] %}{{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }}{% endif %}Last Seen: {% if meta['last_seen'] %}{{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }}{% endif %}
    +
    • +
  • +
    +
    +
    + +
    +
    +
    + {% include 'objects/image/block_blur_img_slider.html' %} + + + + {% for c in meta['map'] %} + + {% endfor %} + +
    +
    +
    +
    +
    • +
  • + 
    {{ meta['content'] }}
    + {% if meta['translation'] %} +
    + 
    {{ meta['translation'] }}
    + {% endif %} + + + + {% if not is_correlation %} +
    + + {% set mess_id_escape= meta['id'] | replace("/", "_") %} + +
    +
    +
    + + + Source: + + + + {% if translation_target %} + +     Target:{{translation_target}} + + + {% else %} + + {% endif %} +
    + +
    +
    +
    + {% endif %} + + + + +
    • + +
  • +
    + Tags: + {% for tag in meta['tags'] %} + + {% endfor %} + +
    +
    • +
+ + {% with obj_type='ocr', obj_id=meta['id'], obj_subtype='' %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + + + {% if is_correlation %} + + + + {% else %} + + + + {% endif %} + + +
+
\ No newline at end of file diff --git a/var/www/templates/objects/qrcode/QrcodeDaterange.html b/var/www/templates/objects/qrcode/QrcodeDaterange.html new file mode 100644 index 00000000..fd9d904e --- /dev/null +++ b/var/www/templates/objects/qrcode/QrcodeDaterange.html @@ -0,0 +1,607 @@ + + + + + Qrcodes - AIL + + + + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ +
+
+
+ +{# {% include 'image/block_images_search.html' %}#} + +
+ + +
+ +
+
+
Select a date range :
+
+
+
+ +
+
+
+ +
+
+ + +
+ +
+
+
+ +
+
+
+
+
+
+ + {% if dict_objects %} + {% if date_from|string == date_to|string %} +

{{ date_from }} Qrcodes Name:

+ {% else %} +

{{ date_from }} to {{ date_to }} Qrcodes Name:

+ {% endif %} + + + + + + + + + + + + {% for obj_id in dict_objects %} + + + + + + + + {% endfor %} + +
First SeenLast SeenTotalLast days
+ {{ dict_objects[obj_id]['id'] }} + {% if dict_objects[obj_id]['content'] %} +
{{ dict_objects[obj_id]['content'] }}
+ {% endif %} + 		{{ dict_objects[obj_id]['first_seen'] }}{{ dict_objects[obj_id]['last_seen'] }}{{ dict_objects[obj_id]['nb_seen'] }}
+ + + {% else %} + {% if show_objects %} + {% if date_from|string == date_to|string %} +

{{ date_from }}, No Qrcode

+ {% else %} +

{{ date_from }} to {{ date_to }}, No Qrcode

+ {% endif %} + {% endif %} + {% endif %} +
+ +
+
+ + + + + + + + + + + + + + + + + diff --git a/var/www/templates/objects/qrcode/ShowQrcode.html b/var/www/templates/objects/qrcode/ShowQrcode.html new file mode 100644 index 00000000..da3bfb49 --- /dev/null +++ b/var/www/templates/objects/qrcode/ShowQrcode.html @@ -0,0 +1,68 @@ + + + + + Qrcode - AIL + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ + {% with meta=meta, is_correlation=False %} + {% include 'objects/qrcode/card_qrcode.html' %} + {% endwith %} + +
+ +
+
+ + + + + + diff --git a/var/www/templates/objects/ocr/card_qrcode.html b/var/www/templates/objects/qrcode/card_qrcode.html similarity index 63% rename from var/www/templates/objects/ocr/card_qrcode.html rename to var/www/templates/objects/qrcode/card_qrcode.html index 84c789b2..8e1302b6 100644 --- a/var/www/templates/objects/ocr/card_qrcode.html +++ b/var/www/templates/objects/qrcode/card_qrcode.html @@ -36,38 +36,6 @@ -
  • -
    -
    -
    - -
    -{#
    #} -{#
    #} -{# {% include 'objects/image/block_blur_img_slider.html' %}#} -{##} -{# #} -{# #} -{# {% for c in meta['map'] %}#} -{# #} -{# {% endfor %}#} -{# #} -{#
    #} -{#
    #} -
    -
    -
  • diff --git a/var/www/templates/sidebars/sidebar_objects.html b/var/www/templates/sidebars/sidebar_objects.html index 93c86675..2d9e2fb2 100644 --- a/var/www/templates/sidebars/sidebar_objects.html +++ b/var/www/templates/sidebars/sidebar_objects.html @@ -88,6 +88,12 @@ OCR
    • +
  • + + + Qrcode + +
  • diff --git a/var/www/templates/tags/search_obj_by_tags.html b/var/www/templates/tags/search_obj_by_tags.html index c1700e5d..99634b4b 100644 --- a/var/www/templates/tags/search_obj_by_tags.html +++ b/var/www/templates/tags/search_obj_by_tags.html @@ -228,10 +228,12 @@ {% endwith %} {% else %} -

    - {% if dict_tagged['date']['date_from'] %}{{ dict_tagged['date']['date_from'][0:4] }}-{{ dict_tagged['date']['date_from'][4:6] }}-{{ dict_tagged['date']['date_from'][6:8] }} {% endif %} - - {{ dict_tagged['date']['date_to'][0:4] }}-{{ dict_tagged['date']['date_to'][4:6] }}-{{ dict_tagged['date']['date_to'][6:8] }} No Result -

    + {% if 'date' in dict_tagged and 'current_tags' in dict_tagged %} +

    + {% if dict_tagged['date']['date_from'] %}{{ dict_tagged['date']['date_from'][0:4] }}-{{ dict_tagged['date']['date_from'][4:6] }}-{{ dict_tagged['date']['date_from'][6:8] }} {% endif %} + - {{ dict_tagged['date']['date_to'][0:4] }}-{{ dict_tagged['date']['date_to'][4:6] }}-{{ dict_tagged['date']['date_to'][6:8] }} No Result +

    + {% endif %} {% for tag in dict_tagged['current_tags'] %} {{ tag }}" {% endfor %}