diff --git a/bin/DB_KVROCKS_MIGRATION.py b/bin/DB_KVROCKS_MIGRATION.py index 1fb54348..b6220036 100755 --- a/bin/DB_KVROCKS_MIGRATION.py +++ b/bin/DB_KVROCKS_MIGRATION.py @@ -30,6 +30,7 @@ from packages import Date # # # # CONFIGS # # # # config_loader = ConfigLoader() r_kvrocks = config_loader.get_db_conn("Kvrocks_DB") +r_obj = config_loader.get_db_conn("Kvrocks_Objects") r_serv_db = config_loader.get_redis_conn("ARDB_DB") r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker") @@ -173,6 +174,9 @@ def user_migration(): Users._delete_user_token(user_id) Users._set_user_token(user_id, token) + for invite_row in r_crawler.smembers('telegram:invite_code'): + r_obj.sadd('telegram:invite_code', invite_row) + # # # # # # # # # # # # # # # # # AIL 2 AIL def ail_2_ail_migration(): diff --git a/bin/lib/telegram.py b/bin/lib/telegram.py index db04699d..aca08d38 100755 --- a/bin/lib/telegram.py +++ b/bin/lib/telegram.py @@ -7,12 +7,14 @@ import sys from urllib.parse import urlparse -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader -import Username +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader -config_loader = ConfigLoader.ConfigLoader() -r_serv_crawler = config_loader.get_redis_conn("ARDB_Onion") +config_loader = ConfigLoader() +r_obj = config_loader.get_db_conn("Kvrocks_Objects") config_loader = None REGEX_USERNAME = re.compile(r'[0-9a-zA-z_]+') @@ -24,7 +26,7 @@ def save_item_correlation(username, item_id, item_date): Username.save_item_correlation('telegram', username, item_id, item_date) def save_telegram_invite_hash(invite_hash, item_id): - r_serv_crawler.sadd('telegram:invite_code', '{};{}'.format(invite_hash, item_id)) + r_obj.sadd('telegram:invite_code', f'{invite_hash};{item_id}') def get_data_from_telegram_url(base_url, url_path): dict_url = {} diff --git a/bin/modules/Telegram.py b/bin/modules/Telegram.py index 4717e3db..cb5538af 100755 --- a/bin/modules/Telegram.py +++ b/bin/modules/Telegram.py @@ -17,6 +17,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from modules.abstract_module import AbstractModule from lib.objects.Items import Item +from lib.objects.Usernames import Username from lib import regex_helper from lib import telegram @@ -40,7 +41,6 @@ class Telegram(AbstractModule): self.redis_logger.info(f"Module {self.module_name} initialized") def compute(self, message, r_result=False): - # messsage = item_id item = Item(message) item_content = item.get_content() item_date = item.get_date() @@ -54,24 +54,30 @@ class Telegram(AbstractModule): print(telegram_link_tuple[2:-2].split("', '", 1)) base_url, url_path = telegram_link_tuple[2:-2].split("', '", 1) dict_url = telegram.get_data_from_telegram_url(base_url, url_path) - if dict_url.get('username'): - telegram.save_item_correlation(dict_url['username'], item.get_id(), item_date) - print(f'username: {dict_url["username"]}') - if dict_url.get('invite_hash'): - telegram.save_telegram_invite_hash(dict_url['invite_hash'], item.get_id()) - print(f'invite code: {dict_url["invite_hash"]}') + user_id = dict_url.get('username') + if user_id: + username = Username(user_id, 'telegram') + username.add(item_date, item.id) + print(f'username: {user_id}') + invite_hash = dict_url.get('invite_hash') + if invite_hash: + telegram.save_telegram_invite_hash(invite_hash, item.id) + print(f'invite code: {invite_hash}') invite_code_found = True # extract tg links tg_links = self.regex_findall(self.re_tg_link, item.get_id(), item_content) for tg_link in tg_links: dict_url = telegram.get_data_from_tg_url(tg_link) - if dict_url.get('username'): - telegram.save_item_correlation(dict_url['username'], item.get_id(), item_date) - print(f'username: {dict_url["username"]}') - if dict_url.get('invite_hash'): - telegram.save_telegram_invite_hash(dict_url['invite_hash'], item.get_id()) - print(f'invite code: {dict_url["invite_hash"]}') + user_id = dict_url.get('username') + if user_id: + username = Username(user_id, 'telegram') + username.add(item_date, item.id) + print(f'username: {user_id}') + invite_hash = dict_url.get('invite_hash') + if invite_hash: + telegram.save_telegram_invite_hash(invite_hash, item.get_id()) + print(f'invite code: {invite_hash}') invite_code_found = True if dict_url.get('login_code'): print(f'login code: {dict_url["login_code"]}') @@ -79,7 +85,7 @@ class Telegram(AbstractModule): # CREATE TAG if invite_code_found: # tags - msg = f'infoleak:automatic-detection="telegram-invite-hash";{item.get_id()}' + msg = f'infoleak:automatic-detection="telegram-invite-hash";{item.id}' self.send_message_to_queue(msg, 'Tags')