AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-10 08:38:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [telegram] migration

Browse source
This commit is contained in:
Terrtia 2022-11-22 10:49:38 +01:00
parent aac024565f
commit 4c2746a2d6
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
3 changed files with 32 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
bin/DB_KVROCKS_MIGRATION.py
View file

@ -30,6 +30,7 @@ from packages import Date
# # # # CONFIGS # # # # # # # # CONFIGS # # # #
config_loader = ConfigLoader() config_loader = ConfigLoader()
r_kvrocks = config_loader.get_db_conn("Kvrocks_DB") r_kvrocks = config_loader.get_db_conn("Kvrocks_DB")
r_obj = config_loader.get_db_conn("Kvrocks_Objects")
r_serv_db = config_loader.get_redis_conn("ARDB_DB") r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker") r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker")
@ -173,6 +174,9 @@ def user_migration():
Users._delete_user_token(user_id) Users._delete_user_token(user_id)
Users._set_user_token(user_id, token) Users._set_user_token(user_id, token)
for invite_row in r_crawler.smembers('telegram:invite_code'):
r_obj.sadd('telegram:invite_code', invite_row)
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# AIL 2 AIL # AIL 2 AIL
def ail_2_ail_migration(): def ail_2_ail_migration():

14
bin/lib/telegram.py
View file

@ -7,12 +7,14 @@ import sys
from urllib.parse import urlparse from urllib.parse import urlparse
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) sys.path.append(os.environ['AIL_BIN'])
import ConfigLoader ##################################
import Username # Import Project packages
##################################
from lib.ConfigLoader import ConfigLoader
config_loader = ConfigLoader.ConfigLoader() config_loader = ConfigLoader()
r_serv_crawler = config_loader.get_redis_conn("ARDB_Onion") r_obj = config_loader.get_db_conn("Kvrocks_Objects")
config_loader = None config_loader = None
REGEX_USERNAME = re.compile(r'[0-9a-zA-z_]+') REGEX_USERNAME = re.compile(r'[0-9a-zA-z_]+')
@ -24,7 +26,7 @@ def save_item_correlation(username, item_id, item_date):
Username.save_item_correlation('telegram', username, item_id, item_date) Username.save_item_correlation('telegram', username, item_id, item_date)
def save_telegram_invite_hash(invite_hash, item_id): def save_telegram_invite_hash(invite_hash, item_id):
r_serv_crawler.sadd('telegram:invite_code', '{};{}'.format(invite_hash, item_id)) r_obj.sadd('telegram:invite_code', f'{invite_hash};{item_id}')
def get_data_from_telegram_url(base_url, url_path): def get_data_from_telegram_url(base_url, url_path):
dict_url = {} dict_url = {}

34
bin/modules/Telegram.py
View file

@ -17,6 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from lib.objects.Items import Item from lib.objects.Items import Item
from lib.objects.Usernames import Username
from lib import regex_helper from lib import regex_helper
from lib import telegram from lib import telegram
@ -40,7 +41,6 @@ class Telegram(AbstractModule):
self.redis_logger.info(f"Module {self.module_name} initialized") self.redis_logger.info(f"Module {self.module_name} initialized")
def compute(self, message, r_result=False): def compute(self, message, r_result=False):
# messsage = item_id
item = Item(message) item = Item(message)
item_content = item.get_content() item_content = item.get_content()
item_date = item.get_date() item_date = item.get_date()
@ -54,24 +54,30 @@ class Telegram(AbstractModule):
print(telegram_link_tuple[2:-2].split("', '", 1)) print(telegram_link_tuple[2:-2].split("', '", 1))
base_url, url_path = telegram_link_tuple[2:-2].split("', '", 1) base_url, url_path = telegram_link_tuple[2:-2].split("', '", 1)
dict_url = telegram.get_data_from_telegram_url(base_url, url_path) dict_url = telegram.get_data_from_telegram_url(base_url, url_path)
if dict_url.get('username'): user_id = dict_url.get('username')
telegram.save_item_correlation(dict_url['username'], item.get_id(), item_date) if user_id:
print(f'username: {dict_url["username"]}') username = Username(user_id, 'telegram')
if dict_url.get('invite_hash'): username.add(item_date, item.id)
telegram.save_telegram_invite_hash(dict_url['invite_hash'], item.get_id()) print(f'username: {user_id}')
print(f'invite code: {dict_url["invite_hash"]}') invite_hash = dict_url.get('invite_hash')
if invite_hash:
telegram.save_telegram_invite_hash(invite_hash, item.id)
print(f'invite code: {invite_hash}')
invite_code_found = True invite_code_found = True
# extract tg links # extract tg links
tg_links = self.regex_findall(self.re_tg_link, item.get_id(), item_content) tg_links = self.regex_findall(self.re_tg_link, item.get_id(), item_content)
for tg_link in tg_links: for tg_link in tg_links:
dict_url = telegram.get_data_from_tg_url(tg_link) dict_url = telegram.get_data_from_tg_url(tg_link)
if dict_url.get('username'): user_id = dict_url.get('username')
telegram.save_item_correlation(dict_url['username'], item.get_id(), item_date) if user_id:
print(f'username: {dict_url["username"]}') username = Username(user_id, 'telegram')
if dict_url.get('invite_hash'): username.add(item_date, item.id)
telegram.save_telegram_invite_hash(dict_url['invite_hash'], item.get_id()) print(f'username: {user_id}')
print(f'invite code: {dict_url["invite_hash"]}') invite_hash = dict_url.get('invite_hash')
if invite_hash:
telegram.save_telegram_invite_hash(invite_hash, item.get_id())
print(f'invite code: {invite_hash}')
invite_code_found = True invite_code_found = True
if dict_url.get('login_code'): if dict_url.get('login_code'):
print(f'login code: {dict_url["login_code"]}') print(f'login code: {dict_url["login_code"]}')
@ -79,7 +85,7 @@ class Telegram(AbstractModule):
# CREATE TAG # CREATE TAG
if invite_code_found: if invite_code_found:
# tags # tags
msg = f'infoleak:automatic-detection="telegram-invite-hash";{item.get_id()}' msg = f'infoleak:automatic-detection="telegram-invite-hash";{item.id}'
self.send_message_to_queue(msg, 'Tags') self.send_message_to_queue(msg, 'Tags')