AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-22 22:27:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [tools] add reprocess title + CEDetector

Browse source
This commit is contained in:
terrtia 2024-10-16 10:56:35 +02:00
parent 1112615ec3
commit 3f78457dff
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
3 changed files with 24 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
bin/lib/objects/ail_objects.py
View file

@ -341,6 +341,8 @@ def obj_iterator(obj_type, filters):
return Pgps.get_all_pgps_objects(filters=filters) return Pgps.get_all_pgps_objects(filters=filters)
elif obj_type == 'message': elif obj_type == 'message':
return chats_viewer.get_messages_iterator(filters=filters) return chats_viewer.get_messages_iterator(filters=filters)
elif obj_type == 'title':
return Titles.Titles().get_iterator()
def card_objs_iterators(filters): def card_objs_iterators(filters):

27
bin/modules/CEDetector.py
View file

@ -111,17 +111,28 @@ def test_detection():
if not is_detected: if not is_detected:
not_detected.add(domain) not_detected.add(domain)
print()
print()
print()
print()
for domain in not_detected: for domain in not_detected:
dom = Domain(domain) dom = Domain(domain)
print('-----------', domain) # print('-----------', domain)
for h in dom.get_correlation('title').get('title', []): for h in dom.get_correlation('title').get('title', []):
print(Title(h[1:]).get_content().lower()) c = Title(h[1:]).get_content().lower()
print() if c == '404 not found':
print() lt = []
dom = Domain(domain)
print('-----------', domain)
for hi in dom.get_correlation('title').get('title', []):
print(Title(hi[1:]).get_content().lower())
ci = Title(hi[1:]).get_content().lower()
if ci != '404 not found' and ci not in []:
lt.append(ci)
if lt:
print('-----------', domain)
for ti in lt:
print(ti)
print()
print()
# Tag.delete_object_tag(tag, 'domain', domain)
if __name__ == "__main__": if __name__ == "__main__":

4
tools/reprocess_objects.py
View file

@ -29,11 +29,13 @@ from lib.objects import ail_objects
# from modules.Onion import Onion # from modules.Onion import Onion
# from modules.Telegram import Telegram # from modules.Telegram import Telegram
from modules.CEDetector import CEDetector
from modules.Languages import Languages from modules.Languages import Languages
from modules.OcrExtractor import OcrExtractor from modules.OcrExtractor import OcrExtractor
from modules.QrCodeReader import QrCodeReader from modules.QrCodeReader import QrCodeReader
MODULES = { MODULES = {
'CEDetector': CEDetector,
'Languages': Languages, 'Languages': Languages,
'OcrExtractor': OcrExtractor, 'OcrExtractor': OcrExtractor,
'QrCodeReader': QrCodeReader 'QrCodeReader': QrCodeReader
@ -70,7 +72,7 @@ if __name__ == "__main__":
obj_type = args.type obj_type = args.type
if not is_object_type(obj_type): if not is_object_type(obj_type):
raise Exception(f'Invalid Object Type: {obj_type}') raise Exception(f'Invalid Object Type: {obj_type}')
if obj_type not in ['image', 'item', 'message', 'screenshot']: if obj_type not in ['image', 'item', 'message', 'screenshot', 'title']:
raise Exception(f'Currently not supported Object Type: {obj_type}') raise Exception(f'Currently not supported Object Type: {obj_type}')
modulename = args.module modulename = args.module