mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 00:28:22 +00:00
chg: [telegram] migration
This commit is contained in:
parent
aac024565f
commit
4c2746a2d6
3 changed files with 32 additions and 20 deletions
|
@ -30,6 +30,7 @@ from packages import Date
|
||||||
# # # # CONFIGS # # # #
|
# # # # CONFIGS # # # #
|
||||||
config_loader = ConfigLoader()
|
config_loader = ConfigLoader()
|
||||||
r_kvrocks = config_loader.get_db_conn("Kvrocks_DB")
|
r_kvrocks = config_loader.get_db_conn("Kvrocks_DB")
|
||||||
|
r_obj = config_loader.get_db_conn("Kvrocks_Objects")
|
||||||
|
|
||||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||||
r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker")
|
r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker")
|
||||||
|
@ -173,6 +174,9 @@ def user_migration():
|
||||||
Users._delete_user_token(user_id)
|
Users._delete_user_token(user_id)
|
||||||
Users._set_user_token(user_id, token)
|
Users._set_user_token(user_id, token)
|
||||||
|
|
||||||
|
for invite_row in r_crawler.smembers('telegram:invite_code'):
|
||||||
|
r_obj.sadd('telegram:invite_code', invite_row)
|
||||||
|
|
||||||
# # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # #
|
||||||
# AIL 2 AIL
|
# AIL 2 AIL
|
||||||
def ail_2_ail_migration():
|
def ail_2_ail_migration():
|
||||||
|
|
|
@ -7,12 +7,14 @@ import sys
|
||||||
|
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
sys.path.append(os.environ['AIL_BIN'])
|
||||||
import ConfigLoader
|
##################################
|
||||||
import Username
|
# Import Project packages
|
||||||
|
##################################
|
||||||
|
from lib.ConfigLoader import ConfigLoader
|
||||||
|
|
||||||
config_loader = ConfigLoader.ConfigLoader()
|
config_loader = ConfigLoader()
|
||||||
r_serv_crawler = config_loader.get_redis_conn("ARDB_Onion")
|
r_obj = config_loader.get_db_conn("Kvrocks_Objects")
|
||||||
config_loader = None
|
config_loader = None
|
||||||
|
|
||||||
REGEX_USERNAME = re.compile(r'[0-9a-zA-z_]+')
|
REGEX_USERNAME = re.compile(r'[0-9a-zA-z_]+')
|
||||||
|
@ -24,7 +26,7 @@ def save_item_correlation(username, item_id, item_date):
|
||||||
Username.save_item_correlation('telegram', username, item_id, item_date)
|
Username.save_item_correlation('telegram', username, item_id, item_date)
|
||||||
|
|
||||||
def save_telegram_invite_hash(invite_hash, item_id):
|
def save_telegram_invite_hash(invite_hash, item_id):
|
||||||
r_serv_crawler.sadd('telegram:invite_code', '{};{}'.format(invite_hash, item_id))
|
r_obj.sadd('telegram:invite_code', f'{invite_hash};{item_id}')
|
||||||
|
|
||||||
def get_data_from_telegram_url(base_url, url_path):
|
def get_data_from_telegram_url(base_url, url_path):
|
||||||
dict_url = {}
|
dict_url = {}
|
||||||
|
|
|
@ -17,6 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
||||||
##################################
|
##################################
|
||||||
from modules.abstract_module import AbstractModule
|
from modules.abstract_module import AbstractModule
|
||||||
from lib.objects.Items import Item
|
from lib.objects.Items import Item
|
||||||
|
from lib.objects.Usernames import Username
|
||||||
from lib import regex_helper
|
from lib import regex_helper
|
||||||
from lib import telegram
|
from lib import telegram
|
||||||
|
|
||||||
|
@ -40,7 +41,6 @@ class Telegram(AbstractModule):
|
||||||
self.redis_logger.info(f"Module {self.module_name} initialized")
|
self.redis_logger.info(f"Module {self.module_name} initialized")
|
||||||
|
|
||||||
def compute(self, message, r_result=False):
|
def compute(self, message, r_result=False):
|
||||||
# messsage = item_id
|
|
||||||
item = Item(message)
|
item = Item(message)
|
||||||
item_content = item.get_content()
|
item_content = item.get_content()
|
||||||
item_date = item.get_date()
|
item_date = item.get_date()
|
||||||
|
@ -54,24 +54,30 @@ class Telegram(AbstractModule):
|
||||||
print(telegram_link_tuple[2:-2].split("', '", 1))
|
print(telegram_link_tuple[2:-2].split("', '", 1))
|
||||||
base_url, url_path = telegram_link_tuple[2:-2].split("', '", 1)
|
base_url, url_path = telegram_link_tuple[2:-2].split("', '", 1)
|
||||||
dict_url = telegram.get_data_from_telegram_url(base_url, url_path)
|
dict_url = telegram.get_data_from_telegram_url(base_url, url_path)
|
||||||
if dict_url.get('username'):
|
user_id = dict_url.get('username')
|
||||||
telegram.save_item_correlation(dict_url['username'], item.get_id(), item_date)
|
if user_id:
|
||||||
print(f'username: {dict_url["username"]}')
|
username = Username(user_id, 'telegram')
|
||||||
if dict_url.get('invite_hash'):
|
username.add(item_date, item.id)
|
||||||
telegram.save_telegram_invite_hash(dict_url['invite_hash'], item.get_id())
|
print(f'username: {user_id}')
|
||||||
print(f'invite code: {dict_url["invite_hash"]}')
|
invite_hash = dict_url.get('invite_hash')
|
||||||
|
if invite_hash:
|
||||||
|
telegram.save_telegram_invite_hash(invite_hash, item.id)
|
||||||
|
print(f'invite code: {invite_hash}')
|
||||||
invite_code_found = True
|
invite_code_found = True
|
||||||
|
|
||||||
# extract tg links
|
# extract tg links
|
||||||
tg_links = self.regex_findall(self.re_tg_link, item.get_id(), item_content)
|
tg_links = self.regex_findall(self.re_tg_link, item.get_id(), item_content)
|
||||||
for tg_link in tg_links:
|
for tg_link in tg_links:
|
||||||
dict_url = telegram.get_data_from_tg_url(tg_link)
|
dict_url = telegram.get_data_from_tg_url(tg_link)
|
||||||
if dict_url.get('username'):
|
user_id = dict_url.get('username')
|
||||||
telegram.save_item_correlation(dict_url['username'], item.get_id(), item_date)
|
if user_id:
|
||||||
print(f'username: {dict_url["username"]}')
|
username = Username(user_id, 'telegram')
|
||||||
if dict_url.get('invite_hash'):
|
username.add(item_date, item.id)
|
||||||
telegram.save_telegram_invite_hash(dict_url['invite_hash'], item.get_id())
|
print(f'username: {user_id}')
|
||||||
print(f'invite code: {dict_url["invite_hash"]}')
|
invite_hash = dict_url.get('invite_hash')
|
||||||
|
if invite_hash:
|
||||||
|
telegram.save_telegram_invite_hash(invite_hash, item.get_id())
|
||||||
|
print(f'invite code: {invite_hash}')
|
||||||
invite_code_found = True
|
invite_code_found = True
|
||||||
if dict_url.get('login_code'):
|
if dict_url.get('login_code'):
|
||||||
print(f'login code: {dict_url["login_code"]}')
|
print(f'login code: {dict_url["login_code"]}')
|
||||||
|
@ -79,7 +85,7 @@ class Telegram(AbstractModule):
|
||||||
# CREATE TAG
|
# CREATE TAG
|
||||||
if invite_code_found:
|
if invite_code_found:
|
||||||
# tags
|
# tags
|
||||||
msg = f'infoleak:automatic-detection="telegram-invite-hash";{item.get_id()}'
|
msg = f'infoleak:automatic-detection="telegram-invite-hash";{item.id}'
|
||||||
self.send_message_to_queue(msg, 'Tags')
|
self.send_message_to_queue(msg, 'Tags')
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue