chg: [v4.0 AIL SYNC / AIL 2 AIL] SYNC Manager + fixs + views

This commit is contained in:
Terrtia 2021-11-22 23:45:41 +01:00
parent 966f61bb94
commit 997a2c602a
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
21 changed files with 2006 additions and 73 deletions

View file

@ -51,9 +51,11 @@ class Sync_module(AbstractModule):
print(message) print(message)
### REFRESH DICT ### REFRESH DICT
if self.last_refresh < ail_2_ail.get_last_updated_ail_instance(): if self.last_refresh < ail_2_ail.get_last_updated_sync_config():
self.dict_sync_queues = ail_2_ail.get_all_sync_queue_dict()
self.last_refresh = time.time() self.last_refresh = time.time()
self.dict_sync_queues = ail_2_ail.get_all_sync_queue_dict()
print('sync queues refreshed')
print(self.dict_sync_queues)
# Extract object from message # Extract object from message
# # TODO: USE JSON DICT ???? # # TODO: USE JSON DICT ????

View file

@ -4,10 +4,13 @@
import os import os
import json import json
import secrets import secrets
import re
import sys import sys
import time import time
import uuid import uuid
from flask import escape
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader import ConfigLoader
@ -16,6 +19,7 @@ import screen
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
from Item import Item from Item import Item
import Tag
config_loader = ConfigLoader.ConfigLoader() config_loader = ConfigLoader.ConfigLoader()
r_cache = config_loader.get_redis_conn("Redis_Cache") r_cache = config_loader.get_redis_conn("Redis_Cache")
@ -23,6 +27,20 @@ r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_serv_sync = config_loader.get_redis_conn("ARDB_DB") r_serv_sync = config_loader.get_redis_conn("ARDB_DB")
config_loader = None config_loader = None
def is_valid_uuid_v4(UUID):
if not UUID:
return False
UUID = UUID.replace('-', '')
try:
uuid_test = uuid.UUID(hex=UUID, version=4)
return uuid_test.hex == UUID
except:
return False
def sanityze_uuid(UUID):
sanityzed_uuid = uuid.UUID(hex=UUID, version=4)
return str(sanityzed_uuid)
def generate_uuid(): def generate_uuid():
return str(uuid.uuid4()).replace('-', '') return str(uuid.uuid4()).replace('-', '')
@ -32,7 +50,31 @@ def generate_sync_api_key():
def get_ail_uuid(): def get_ail_uuid():
return r_serv_db.get('ail:uuid') return r_serv_db.get('ail:uuid')
# # TODO: # TODO: # TODO: # TODO: # TODO: ADD SYNC MODE == PUSH def is_valid_websocket_url(websocket_url):
regex_websocket_url = r'^(wss:\/\/)([0-9]{1,3}(?:\.[0-9]{1,3}){3}|(?=[^\/]{1,254}(?![^\/]))(?:(?=[a-zA-Z0-9-]{1,63}\.?)(?:xn--+)?[a-zA-Z0-9]+(?:-[a-zA-Z0-9]+)*\.?)+[a-zA-Z]{2,63}):([0-9]{1,5})$'
if re.match(regex_websocket_url, websocket_url):
return True
return False
def is_valid_websocket_key(ail_key):
regex_key = r'^[A-Za-z0-9-_]{56}$'
if re.match(regex_key, ail_key):
return True
return False
#### HANDLE CONFIG UPDATE ####
def get_last_updated_sync_config():
epoch = r_serv_sync.get(f'ail:instance:queue:last_updated_sync_config')
if not epoch:
epoch = 0
return float(epoch)
def set_last_updated_sync_config():
epoch = int(time.time())
r_serv_sync.set(f'ail:instance:queue:last_updated_sync_config', epoch)
return epoch
# # TODO: get connection status # # TODO: get connection status
# # TODO: get connection METADATA # # TODO: get connection METADATA
############################# #############################
@ -49,12 +91,21 @@ def get_all_sync_clients(r_set=False):
def get_sync_client_ail_uuid(client_id): def get_sync_client_ail_uuid(client_id):
return r_cache.hget(f'ail_2_ail:sync_client:{client_id}', 'ail_uuid') return r_cache.hget(f'ail_2_ail:sync_client:{client_id}', 'ail_uuid')
def get_sync_client_queue_uuid(client_id): # current: only one push registred
return r_cache.hget(f'ail_2_ail:sync_client:{client_id}', 'queue_uuid') def get_client_id_by_ail_uuid(ail_uuid):
res = r_cache.smembers(f'ail_2_ail:ail_uuid:{ail_uuid}')
if res:
return int(res.pop())
def get_all_running_sync_servers():
running_ail_servers= []
for client_id in get_all_sync_clients():
ail_uuid = get_sync_client_ail_uuid(client_id)
running_ail_servers.append(ail_uuid)
return running_ail_servers
def delete_sync_client_cache(client_id): def delete_sync_client_cache(client_id):
ail_uuid = get_sync_client_ail_uuid(client_id) ail_uuid = get_sync_client_ail_uuid(client_id)
queue_uuid = get_sync_client_queue_uuid(client_id)
# map ail_uuid/queue_uuid # map ail_uuid/queue_uuid
r_cache.srem(f'ail_2_ail:ail_uuid:{ail_uuid}', client_id) r_cache.srem(f'ail_2_ail:ail_uuid:{ail_uuid}', client_id)
r_cache.srem(f'ail_2_ail:queue_uuid:{queue_uuid}', client_id) r_cache.srem(f'ail_2_ail:queue_uuid:{queue_uuid}', client_id)
@ -71,9 +122,31 @@ def delete_all_sync_clients_cache():
# -kill # -kill
# -relaunch # -relaunch
## TODO: check command ## TODO: check command
def send_command_to_manager(command, client_id=-1): def send_command_to_manager(command, client_id=-1, ail_uuid=None):
dict_action = {'command': command, 'client_id': client_id} dict_action = {'command': command, 'client_id': client_id}
r_cache.sadd('ail_2_ail:client_manager:command', dict_action) if ail_uuid:
dict_action['ail_uuid'] = ail_uuid
str_command = json.dumps(dict_action)
r_cache.sadd('ail_2_ail:client_manager:command', str_command)
def refresh_ail_instance_connection(ail_uuid):
client_id = get_client_id_by_ail_uuid(ail_uuid)
launch_required = is_ail_instance_push_enabled(ail_uuid)
print(client_id)
print(launch_required)
# relaunch
if client_id and launch_required:
send_command_to_manager('relaunch', client_id=client_id)
# kill
elif client_id:
send_command_to_manager('kill', client_id=client_id)
# launch
elif launch_required:
send_command_to_manager('launch', ail_uuid=ail_uuid)
class AIL2AILClientManager(object): class AIL2AILClientManager(object):
"""AIL2AILClientManager.""" """AIL2AILClientManager."""
@ -93,19 +166,23 @@ class AIL2AILClientManager(object):
# return new client id # return new client id
def get_new_sync_client_id(self): def get_new_sync_client_id(self):
for new_id in range(100000): for new_id in range(1, 100000):
new_id = str(new_id)
if new_id not in self.clients: if new_id not in self.clients:
return str(new_id) return str(new_id)
def get_sync_client_ail_uuid(self, client_id): def get_sync_client_ail_uuid(self, client_id):
return self.clients[client_id]['ail_uuid'] return self.clients[client_id]['ail_uuid']
def get_sync_client_queue_uuid(self, client_id): # def get_sync_client_queue_uuid(self, client_id):
return self.clients[client_id]['queue_uuid'] # return self.clients[client_id]['queue_uuid']
# # TODO: check PUSH ACL
def get_all_sync_clients_to_launch(self): def get_all_sync_clients_to_launch(self):
return get_all_ail_instance() ail_instances_to_launch = []
for ail_uuid in get_all_ail_instance():
if is_ail_instance_push_enabled(ail_uuid):
ail_instances_to_launch.append(ail_uuid)
return ail_instances_to_launch
def relaunch_all_sync_clients(self): def relaunch_all_sync_clients(self):
delete_all_sync_clients_cache() delete_all_sync_clients_cache()
@ -127,6 +204,8 @@ class AIL2AILClientManager(object):
r_cache.hset(f'ail_2_ail:sync_client:{client_id}', 'ail_uuid', ail_uuid) r_cache.hset(f'ail_2_ail:sync_client:{client_id}', 'ail_uuid', ail_uuid)
r_cache.hset(f'ail_2_ail:sync_client:{client_id}', 'launch_time', int(time.time())) r_cache.hset(f'ail_2_ail:sync_client:{client_id}', 'launch_time', int(time.time()))
r_cache.sadd('ail_2_ail:all_sync_clients', client_id)
# create map ail_uuid/queue_uuid # create map ail_uuid/queue_uuid
r_cache.sadd(f'ail_2_ail:ail_uuid:{ail_uuid}', client_id) r_cache.sadd(f'ail_2_ail:ail_uuid:{ail_uuid}', client_id)
@ -135,7 +214,7 @@ class AIL2AILClientManager(object):
# # TODO: FORCE KILL ???????????? # # TODO: FORCE KILL ????????????
# # TODO: check if exists # # TODO: check if exists
def kill_sync_client(self, client_id): def kill_sync_client(self, client_id):
if not kill_screen_window('AIL_2_AIL', client_id): if not screen.kill_screen_window('AIL_2_AIL',client_id):
# # TODO: log kill error # # TODO: log kill error
pass pass
@ -147,17 +226,19 @@ class AIL2AILClientManager(object):
def get_manager_command(self): def get_manager_command(self):
res = r_cache.spop('ail_2_ail:client_manager:command') res = r_cache.spop('ail_2_ail:client_manager:command')
if res: if res:
return json.dumps(res) print(res)
print(type(res))
return json.loads(res)
else: else:
return None return None
def execute_manager_command(self, command_dict): def execute_manager_command(self, command_dict):
print(command_dict)
command = command_dict.get('command') command = command_dict.get('command')
if command == 'launch': if command == 'launch':
ail_uuid = int(command_dict.get('ail_uuid')) ail_uuid = command_dict.get('ail_uuid')
queue_uuid = int(command_dict.get('queue_uuid')) self.launch_sync_client(ail_uuid)
self.launch_sync_client(ail_uuid, queue_uuid) elif command == 'relaunch_all':
elif command == 'relaunch':
self.relaunch_all_sync_clients() self.relaunch_all_sync_clients()
else: else:
# only one sync client # only one sync client
@ -165,13 +246,13 @@ class AIL2AILClientManager(object):
if client_id < 1: if client_id < 1:
print('Invalid client id') print('Invalid client id')
return None return None
client_id = str(client_id)
if command == 'kill': if command == 'kill':
self.kill_sync_client(client_id) self.kill_sync_client(client_id)
elif command == 'relaunch': elif command == 'relaunch':
ail_uuid = self.get_sync_client_ail_uuid(client_id) ail_uuid = self.get_sync_client_ail_uuid(client_id)
queue_uuid = self.get_sync_client_queue_uuid(client_id)
self.kill_sync_client(client_id) self.kill_sync_client(client_id)
self.launch_sync_client(ail_uuid, queue_uuid) self.launch_sync_client(ail_uuid)
######################################## ########################################
######################################## ########################################
@ -181,7 +262,6 @@ class AIL2AILClientManager(object):
def get_sync_client_status(client_id): def get_sync_client_status(client_id):
dict_client = {'id': client_id} dict_client = {'id': client_id}
dict_client['ail_uuid'] = get_sync_client_ail_uuid(client_id) dict_client['ail_uuid'] = get_sync_client_ail_uuid(client_id)
dict_client['queue_uuid'] = get_sync_client_queue_uuid(client_id)
return dict_client return dict_client
def get_all_sync_client_status(): def get_all_sync_client_status():
@ -232,12 +312,18 @@ def get_ail_instance_all_sync_queue(ail_uuid):
def is_ail_instance_queue(ail_uuid, queue_uuid): def is_ail_instance_queue(ail_uuid, queue_uuid):
return r_serv_sync.sismember(f'ail:instance:sync_queue:{ail_uuid}', queue_uuid) return r_serv_sync.sismember(f'ail:instance:sync_queue:{ail_uuid}', queue_uuid)
def exists_ail_instance(ail_uuid):
return r_serv_sync.exists(f'ail:instance:{ail_uuid}')
def get_ail_instance_url(ail_uuid): def get_ail_instance_url(ail_uuid):
return r_serv_sync.hget(f'ail:instance:{ail_uuid}', 'url') return r_serv_sync.hget(f'ail:instance:{ail_uuid}', 'url')
def get_ail_instance_description(ail_uuid): def get_ail_instance_description(ail_uuid):
return r_serv_sync.hget(f'ail:instance:{ail_uuid}', 'description') return r_serv_sync.hget(f'ail:instance:{ail_uuid}', 'description')
def exists_ail_instance(ail_uuid):
return r_serv_sync.sismember('ail:instance:all', ail_uuid)
def is_ail_instance_push_enabled(ail_uuid): def is_ail_instance_push_enabled(ail_uuid):
res = r_serv_sync.hget(f'ail:instance:{ail_uuid}', 'push') res = r_serv_sync.hget(f'ail:instance:{ail_uuid}', 'push')
return res == 'True' return res == 'True'
@ -268,27 +354,44 @@ def change_pull_push_state(ail_uuid, pull=False, push=False):
push = False push = False
r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'push', push) r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'push', push)
r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'pull', pull) r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'pull', pull)
set_last_updated_sync_config()
refresh_ail_instance_connection(ail_uuid)
# # TODO: HIDE ADD GLOBAL FILTER (ON BOTH SIDE) # # TODO: HIDE ADD GLOBAL FILTER (ON BOTH SIDE)
# # TODO: push/pull def get_ail_instance_metadata(ail_uuid, sync_queues=False):
def get_ail_instance_metadata(ail_uuid):
dict_meta = {} dict_meta = {}
dict_meta['uuid'] = ail_uuid dict_meta['uuid'] = ail_uuid
dict_meta['url'] = get_ail_instance_url(ail_uuid) dict_meta['url'] = get_ail_instance_url(ail_uuid)
dict_meta['description'] = get_ail_instance_description(ail_uuid) dict_meta['description'] = get_ail_instance_description(ail_uuid)
dict_meta['pull'] = is_ail_instance_pull_enabled(ail_uuid)
dict_meta['push'] = is_ail_instance_pull_enabled(ail_uuid)
# # TODO: HIDE # # TODO: HIDE
dict_meta['api_key'] = get_ail_instance_key(ail_uuid) dict_meta['api_key'] = get_ail_instance_key(ail_uuid)
if sync_queues:
dict_meta['sync_queues'] = get_ail_instance_all_sync_queue(ail_uuid)
# # TODO: # # TODO:
# - set UUID sync_queue # - set UUID sync_queue
return dict_meta return dict_meta
def get_all_ail_instances_metadata():
l_servers = []
for ail_uuid in get_all_ail_instance():
l_servers.append(get_ail_instance_metadata(ail_uuid, sync_queues=True))
return l_servers
def get_ail_instances_metadata(l_ail_servers):
l_servers = []
for ail_uuid in l_ail_servers:
l_servers.append(get_ail_instance_metadata(ail_uuid, sync_queues=True))
return l_servers
# # TODO: VALIDATE URL # # TODO: VALIDATE URL
# API KEY # API KEY
def create_ail_instance(ail_uuid, url, api_key=None, description=None): def create_ail_instance(ail_uuid, url, api_key=None, description=None, pull=True, push=True):
r_serv_sync.sadd('ail:instance:all', ail_uuid) r_serv_sync.sadd('ail:instance:all', ail_uuid)
r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'url', url) r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'url', url)
## API KEY ## ## API KEY ##
@ -300,6 +403,9 @@ def create_ail_instance(ail_uuid, url, api_key=None, description=None):
#- API KEY -# #- API KEY -#
if description: if description:
r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'description', description) r_serv_sync.hset(f'ail:instance:{ail_uuid}', 'description', description)
change_pull_push_state(ail_uuid, pull=pull, push=push)
set_last_updated_sync_config()
refresh_ail_instance_connection(ail_uuid)
return ail_uuid return ail_uuid
def delete_ail_instance(ail_uuid): def delete_ail_instance(ail_uuid):
@ -311,12 +417,54 @@ def delete_ail_instance(ail_uuid):
r_serv_sync.srem('ail:instance:key:all', ail_uuid) r_serv_sync.srem('ail:instance:key:all', ail_uuid)
r_serv_sync.delete(f'ail:instance:key:{key}', ail_uuid) r_serv_sync.delete(f'ail:instance:key:{key}', ail_uuid)
r_serv_sync.srem('ail:instance:all', ail_uuid) r_serv_sync.srem('ail:instance:all', ail_uuid)
set_last_updated_sync_config()
refresh_ail_instance_connection(ail_uuid)
return ail_uuid
def get_last_updated_ail_instance(): ## API ##
epoch = r_serv_sync.get(f'ail:instance:queue:last_updated')
if not epoch: def api_create_ail_instance(json_dict):
epoch = 0 ail_uuid = json_dict.get('uuid').replace(' ', '')
return float(epoch) if not is_valid_uuid_v4(ail_uuid):
return {"status": "error", "reason": "Invalid ail uuid"}, 400
ail_uuid = sanityze_uuid(ail_uuid)
if exists_ail_instance(ail_uuid):
return {"status": "error", "reason": "AIL uuid already exists"}, 400
if json_dict.get('pull'):
pull = True
else:
pull = False
if json_dict.get('push'):
push = True
else:
push = False
description = json_dict.get('description')
ail_url = json_dict.get('url').replace(' ', '')
if not is_valid_websocket_url(ail_url):
return {"status": "error", "reason": "Invalid websocket url"}, 400
ail_key = json_dict.get('key')
if ail_key:
ail_key = ail_key.replace(' ', '')
if not is_valid_websocket_key(ail_key):
return {"status": "error", "reason": "Invalid websocket key"}, 400
res = create_ail_instance(ail_uuid, ail_url, api_key=ail_key, description=description,
pull=pull, push=push)
return res, 200
def api_delete_ail_instance(json_dict):
ail_uuid = json_dict.get('uuid', '').replace(' ', '')
if not is_valid_uuid_v4(ail_uuid):
return {"status": "error", "reason": "Invalid AIL uuid"}, 400
ail_uuid = sanityze_uuid(ail_uuid)
if not exists_ail_instance(ail_uuid):
return {"status": "error", "reason": "AIL server not found"}, 404
res = delete_ail_instance(ail_uuid)
return res, 200
#################### ####################
# # # #
@ -334,32 +482,57 @@ def get_all_sync_queue():
def get_sync_queue_all_ail_instance(queue_uuid): def get_sync_queue_all_ail_instance(queue_uuid):
return r_serv_sync.smembers(f'ail2ail:sync_queue:ail_instance:{queue_uuid}') return r_serv_sync.smembers(f'ail2ail:sync_queue:ail_instance:{queue_uuid}')
def exists_sync_queue(queue_uuid):
return r_serv_sync.exists(f'ail2ail:sync_queue:{queue_uuid}')
# # TODO: check if push or pull enabled ? # # TODO: check if push or pull enabled ?
def is_queue_used_by_ail_instace(queue_uuid): def is_queue_used_by_ail_instance(queue_uuid):
return r_serv_sync.exists(f'ail2ail:sync_queue:ail_instance:{queue_uuid}') return r_serv_sync.exists(f'ail2ail:sync_queue:ail_instance:{queue_uuid}')
# # TODO: add others filter # # TODO: add others filter
def get_sync_queue_filter(queue_uuid): def get_sync_queue_filter(queue_uuid):
return r_serv_sync.smembers(f'ail2ail:sync_queue:filter:tags:{queue_uuid}') return r_serv_sync.smembers(f'ail2ail:sync_queue:filter:tags:{queue_uuid}')
def get_sync_queue_name(queue_uuid):
return r_serv_sync.hget(f'ail2ail:sync_queue:{queue_uuid}', 'name')
def get_sync_queue_description(queue_uuid):
return r_serv_sync.hget(f'ail2ail:sync_queue:{queue_uuid}', 'description')
def get_sync_queue_max_size(queue_uuid):
return r_serv_sync.hget(f'ail2ail:sync_queue:{queue_uuid}', 'max_size')
# # TODO: ADD FILTER # # TODO: ADD FILTER
def get_sync_queue_metadata(queue_uuid): def get_sync_queue_metadata(queue_uuid):
dict_meta = {} dict_meta = {}
dict_meta['uuid'] = queue_uuid dict_meta['uuid'] = queue_uuid
dict_meta['name'] = r_serv_sync.hget(f'ail2ail:sync_queue:{queue_uuid}', 'name') dict_meta['name'] = get_sync_queue_name(queue_uuid)
dict_meta['description'] = r_serv_sync.hget(f'ail2ail:sync_queue:{queue_uuid}', 'description') dict_meta['description'] = get_sync_queue_description(queue_uuid)
dict_meta['max_size'] = r_serv_sync.hget(f'ail2ail:sync_queue:{queue_uuid}', 'max_size') dict_meta['max_size'] = get_sync_queue_max_size(queue_uuid)
dict_meta['tags'] = get_sync_queue_filter(queue_uuid)
# # TODO: TO ADD: # # TODO: TO ADD:
# - set uuid instance # - get uuid instance
return dict_meta return dict_meta
def get_all_queues_metadata():
l_queues = []
for queue_uuid in get_all_sync_queue():
l_queues.append(get_sync_queue_metadata(queue_uuid))
return l_queues
def get_queues_metadata(l_queues_uuid):
l_queues = []
for queue_uuid in l_queues_uuid:
l_queues.append(get_sync_queue_metadata(queue_uuid))
return l_queues
##################################################### #####################################################
def get_all_sync_queue_dict(): def get_all_sync_queue_dict():
dict_sync_queues = {} dict_sync_queues = {}
for queue_uuid in get_all_sync_queue(): for queue_uuid in get_all_sync_queue():
if is_queue_used_by_ail_instace(queue_uuid): if is_queue_used_by_ail_instance(queue_uuid):
dict_queue = {} dict_queue = {}
dict_queue['filter'] = get_sync_queue_filter(queue_uuid) dict_queue['filter'] = get_sync_queue_filter(queue_uuid)
@ -374,14 +547,22 @@ def get_all_sync_queue_dict():
dict_sync_queues[queue_uuid] = dict_queue dict_sync_queues[queue_uuid] = dict_queue
return dict_sync_queues return dict_sync_queues
def is_queue_registred_by_ail_instance(queue_uuid, ail_uuid):
return r_serv_sync.sismember(f'ail:instance:sync_queue:{ail_uuid}', queue_uuid)
def register_ail_to_sync_queue(ail_uuid, queue_uuid): def register_ail_to_sync_queue(ail_uuid, queue_uuid):
r_serv_sync.sadd(f'ail2ail:sync_queue:ail_instance:{queue_uuid}', ail_uuid) r_serv_sync.sadd(f'ail2ail:sync_queue:ail_instance:{queue_uuid}', ail_uuid)
r_serv_sync.sadd(f'ail:instance:sync_queue:{ail_uuid}', queue_uuid) r_serv_sync.sadd(f'ail:instance:sync_queue:{ail_uuid}', queue_uuid)
set_last_updated_sync_config()
# # # FIXME: TODO: delete sync queue ????????????????????????????????????????????????????
def unregister_ail_to_sync_queue(ail_uuid, queue_uuid): def unregister_ail_to_sync_queue(ail_uuid, queue_uuid):
r_serv_sync.srem(f'ail2ail:sync_queue:ail_instance:{queue_uuid}', ail_uuid) r_serv_sync.srem(f'ail2ail:sync_queue:ail_instance:{queue_uuid}', ail_uuid)
r_serv_sync.srem(f'ail:instance:sync_queue:{ail_uuid}', queue_uuid) r_serv_sync.srem(f'ail:instance:sync_queue:{ail_uuid}', queue_uuid)
set_last_updated_sync_config()
def get_all_unregistred_queue_by_ail_instance(ail_uuid):
return r_serv_sync.sdiff('ail2ail:sync_queue:all', f'ail:instance:sync_queue:{ail_uuid}')
# # TODO: optionnal name ??? # # TODO: optionnal name ???
# # TODO: SANITYZE TAGS # # TODO: SANITYZE TAGS
@ -397,15 +578,96 @@ def create_sync_queue(name, tags=[], description=None, max_size=100):
for tag in tags: for tag in tags:
r_serv_sync.sadd(f'ail2ail:sync_queue:filter:tags:{queue_uuid}', tag) r_serv_sync.sadd(f'ail2ail:sync_queue:filter:tags:{queue_uuid}', tag)
set_last_updated_sync_config()
return queue_uuid return queue_uuid
def delete_sync_queue(queue_uuid): def delete_sync_queue(queue_uuid):
for ail_uuid in get_sync_queue_all_ail_instance(queue_uuid): for ail_uuid in get_sync_queue_all_ail_instance(queue_uuid):
unregister_ail_to_sync_queue(ail_uuid, queue_uuid) unregister_ail_to_sync_queue(ail_uuid, queue_uuid)
r_serv_sync.delete(f'ail2ail:sync_queue:{queue_uuid}') r_serv_sync.delete(f'ail2ail:sync_queue:{queue_uuid}')
r_serv_sync.delete(f'ail2ail:sync_queue:filter:tags:{queue_uuid}')
r_serv_sync.srem('ail2ail:sync_queue:all', queue_uuid) r_serv_sync.srem('ail2ail:sync_queue:all', queue_uuid)
set_last_updated_sync_config()
return queue_uuid return queue_uuid
## API ##
# # TODO: sanityze queue_name
def api_create_sync_queue(json_dict):
description = json_dict.get('description')
description = escape(description)
queue_name = json_dict.get('name')
if queue_name: #################################################
queue_name = escape(queue_name)
tags = json_dict.get('tags')
if not tags:
{"status": "error", "reason": "no tags provided"}, 400
if not Tag.are_enabled_tags(tags):
{"status": "error", "reason": "Invalid/Disabled tags"}, 400
max_size = json_dict.get('max_size')
if not max_size:
max_size = 100
try:
max_size = int(max_size)
except ValueError:
{"status": "error", "reason": "Invalid queue size value"}, 400
if not max_size > 0:
return {"status": "error", "reason": "Invalid queue size value"}, 400
queue_uuid = create_sync_queue(queue_name, tags=tags, description=description,
max_size=max_size)
return queue_uuid, 200
def api_delete_sync_queue(json_dict):
queue_uuid = json_dict.get('uuid', '').replace(' ', '').replace('-', '')
if not is_valid_uuid_v4(queue_uuid):
return {"status": "error", "reason": "Invalid Queue uuid"}, 400
if not exists_sync_queue(queue_uuid):
return {"status": "error", "reason": "Queue Sync not found"}, 404
res = delete_sync_queue(queue_uuid)
return res, 200
def api_register_ail_to_sync_queue(json_dict):
ail_uuid = json_dict.get('ail_uuid', '').replace(' ', '')
if not is_valid_uuid_v4(ail_uuid):
return {"status": "error", "reason": "Invalid AIL uuid"}, 400
ail_uuid = sanityze_uuid(ail_uuid)
queue_uuid = json_dict.get('queue_uuid', '').replace(' ', '').replace('-', '')
if not is_valid_uuid_v4(queue_uuid):
return {"status": "error", "reason": "Invalid Queue uuid"}, 400
if not exists_ail_instance(ail_uuid):
return {"status": "error", "reason": "AIL server not found"}, 404
if not exists_sync_queue(queue_uuid):
return {"status": "error", "reason": "Queue Sync not found"}, 404
if is_queue_registred_by_ail_instance(queue_uuid, ail_uuid):
return {"status": "error", "reason": "Queue already registred"}, 400
res = register_ail_to_sync_queue(ail_uuid, queue_uuid)
return res, 200
def api_unregister_ail_to_sync_queue(json_dict):
ail_uuid = json_dict.get('ail_uuid', '').replace(' ', '')
if not is_valid_uuid_v4(ail_uuid):
return {"status": "error", "reason": "Invalid ail uuid"}, 400
ail_uuid = sanityze_uuid(ail_uuid)
queue_uuid = json_dict.get('queue_uuid', '').replace(' ', '').replace('-', '')
if not is_valid_uuid_v4(queue_uuid):
return {"status": "error", "reason": "Invalid ail uuid"}, 400
if not exists_ail_instance(ail_uuid):
return {"status": "error", "reason": "AIL server not found"}, 404
if not exists_sync_queue(queue_uuid):
return {"status": "error", "reason": "Queue Sync not found"}, 404
if not is_queue_registred_by_ail_instance(queue_uuid, ail_uuid):
return {"status": "error", "reason": "Queue not registred"}, 400
res = unregister_ail_to_sync_queue(ail_uuid, queue_uuid)
return res, 200
############################# #############################
# # # #
#### SYNC REDIS QUEUE ####### #### SYNC REDIS QUEUE #######
@ -487,6 +749,13 @@ if __name__ == '__main__':
#res = delete_sync_queue(queue_uuid) #res = delete_sync_queue(queue_uuid)
#res = register_ail_to_sync_queue(ail_uuid, queue_uuid) #res = register_ail_to_sync_queue(ail_uuid, queue_uuid)
res = change_pull_push_state(ail_uuid, push=True, pull=True) #res = change_pull_push_state(ail_uuid, push=True, pull=True)
# print(get_ail_instance_all_sync_queue(ail_uuid))
# print(get_all_sync_queue())
# res = get_all_unregistred_queue_by_ail_instance(ail_uuid)
ail_uuid = 'd82d3e61-2438-4ede-93bf-37b6fd9d7510'
res = get_client_id_by_ail_uuid(ail_uuid)
print(res) print(res)

View file

@ -15,23 +15,14 @@ sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
# Import Project packages # Import Project packages
################################## ##################################
from pubsublogger import publisher
from core import ail_2_ail from core import ail_2_ail
# # TODO: refactor logging
obj_test = { #### LOGS ####
"format": "ail", redis_logger = publisher
"version": 1, redis_logger.port = 6380
"type": "item", redis_logger.channel = 'AIL_SYNC'
"meta": {
"ail:uuid": "03c51929-eeab-4d47-9dc0-c667f94c7d2c",
"ail:uuid_org": "28bc3db3-16da-461c-b20b-b944f4058708",
},
"payload": {
"raw" : "MjhiYzNkYjMtMTZkYS00NjFjLWIyMGItYjk0NGY0MDU4NzA4Cg==",
"compress": "gzip",
"encoding": "base64"
}
}
############################# #############################
@ -92,7 +83,7 @@ async def pull(websocket, ail_uuid):
if Obj: if Obj:
obj_ail_stream = ail_2_ail.create_ail_stream(Obj) obj_ail_stream = ail_2_ail.create_ail_stream(Obj)
Obj = json.dumps(obj_ail_stream) Obj = json.dumps(obj_ail_stream)
print(Obj) #print(Obj)
# send objects # send objects
await websocket.send(Obj) await websocket.send(Obj)
@ -107,27 +98,29 @@ async def pull(websocket, ail_uuid):
# PUSH: receive data from client # PUSH: receive data from client
# # TODO: optional queue_uuid # # TODO: optional queue_uuid
async def push(websocket, ail_uuid): async def push(websocket, ail_uuid):
print(ail_uuid) #print(ail_uuid)
while True: while True:
ail_stream = await websocket.recv() ail_stream = await websocket.recv()
# # TODO: CHECK ail_stream # # TODO: CHECK ail_stream
ail_stream = json.loads(ail_stream) ail_stream = json.loads(ail_stream)
print(ail_stream) #print(ail_stream)
ail_2_ail.add_ail_stream_to_sync_importer(ail_stream) ail_2_ail.add_ail_stream_to_sync_importer(ail_stream)
async def ail_to_ail_serv(websocket, path): async def ail_to_ail_serv(websocket, path):
# # TODO: save in class
ail_uuid = websocket.ail_uuid
remote_address = websocket.remote_address
path = unpack_path(path)
sync_mode = path['sync_mode']
# # TODO: check if it works # # TODO: check if it works
# # DEBUG: # # DEBUG:
print(websocket.ail_key) print(websocket.ail_key)
print(websocket.ail_uuid) print(websocket.ail_uuid)
print(websocket.remote_address) print(websocket.remote_address)
path = unpack_path(path)
sync_mode = path['sync_mode']
print(f'sync mode: {sync_mode}') print(f'sync mode: {sync_mode}')
await register(websocket) await register(websocket)
@ -135,7 +128,8 @@ async def ail_to_ail_serv(websocket, path):
if sync_mode == 'pull': if sync_mode == 'pull':
await pull(websocket, websocket.ail_uuid) await pull(websocket, websocket.ail_uuid)
await websocket.close() await websocket.close()
print('closed') redis_logger.info(f'Connection closed: {ail_uuid} {remote_address}')
print(f'Connection closed: {ail_uuid} {remote_address}')
elif sync_mode == 'push': elif sync_mode == 'push':
await push(websocket, websocket.ail_uuid) await push(websocket, websocket.ail_uuid)
@ -163,29 +157,34 @@ class AIL_2_AIL_Protocol(websockets.WebSocketServerProtocol):
api_key = request_headers.get('Authorization', '') api_key = request_headers.get('Authorization', '')
print(api_key) print(api_key)
if api_key is None: if api_key is None:
print('Missing token') redis_logger.warning(f'Missing token: {self.remote_address}')
print(f'Missing token: {self.remote_address}')
return http.HTTPStatus.UNAUTHORIZED, [], b"Missing token\n" return http.HTTPStatus.UNAUTHORIZED, [], b"Missing token\n"
if not ail_2_ail.is_allowed_ail_instance_key(api_key): if not ail_2_ail.is_allowed_ail_instance_key(api_key):
print('Invalid token') redis_logger.warning(f'Invalid token: {self.remote_address}')
print(f'Invalid token: {self.remote_address}')
return http.HTTPStatus.UNAUTHORIZED, [], b"Invalid token\n" return http.HTTPStatus.UNAUTHORIZED, [], b"Invalid token\n"
# PATH # PATH
try: try:
dict_path = unpack_path(path) dict_path = unpack_path(path)
except Exception as e: except Exception as e:
print('Invalid path') redis_logger.warning(f'Invalid path: {self.remote_address}')
print(f'Invalid path: {self.remote_address}')
return http.HTTPStatus.BAD_REQUEST, [], b"Invalid path\n" return http.HTTPStatus.BAD_REQUEST, [], b"Invalid path\n"
ail_uuid = ail_2_ail.get_ail_instance_by_key(api_key) ail_uuid = ail_2_ail.get_ail_instance_by_key(api_key)
if ail_uuid != dict_path['ail_uuid']: if ail_uuid != dict_path['ail_uuid']:
print('Invalid token') redis_logger.warning(f'Invalid token: {self.remote_address} {ail_uuid}')
print(f'Invalid token: {self.remote_address} {ail_uuid}')
return http.HTTPStatus.UNAUTHORIZED, [], b"Invalid token\n" return http.HTTPStatus.UNAUTHORIZED, [], b"Invalid token\n"
if not api_key != ail_2_ail.get_ail_instance_key(api_key): if not api_key != ail_2_ail.get_ail_instance_key(api_key):
print('Invalid token') redis_logger.warning(f'Invalid token: {self.remote_address} {ail_uuid}')
print(f'Invalid token: {self.remote_address} {ail_uuid}')
return http.HTTPStatus.UNAUTHORIZED, [], b"Invalid token\n" return http.HTTPStatus.UNAUTHORIZED, [], b"Invalid token\n"
self.ail_key = api_key self.ail_key = api_key
@ -210,7 +209,9 @@ class AIL_2_AIL_Protocol(websockets.WebSocketServerProtocol):
# SYNC MODE # SYNC MODE
if not ail_2_ail.is_ail_instance_sync_enabled(self.ail_uuid, sync_mode=dict_path['sync_mode']): if not ail_2_ail.is_ail_instance_sync_enabled(self.ail_uuid, sync_mode=dict_path['sync_mode']):
print('SYNC mode disabled') sync_mode = dict_path['sync_mode']
redis_logger.warning(f'SYNC mode disabled: {self.remote_address} {ail_uuid} {sync_mode}')
print(f'SYNC mode disabled: {self.remote_address} {ail_uuid} {sync_mode}')
return http.HTTPStatus.FORBIDDEN, [], b"SYNC mode disabled\n" return http.HTTPStatus.FORBIDDEN, [], b"SYNC mode disabled\n"
# # TODO: CHECK API # # TODO: CHECK API
@ -218,17 +219,16 @@ class AIL_2_AIL_Protocol(websockets.WebSocketServerProtocol):
pass pass
else: else:
print('Invalid path') print(f'Invalid path: {self.remote_address}')
redis_logger.info(f'Invalid path: {self.remote_address}')
return http.HTTPStatus.BAD_REQUEST, [], b"Invalid path\n" return http.HTTPStatus.BAD_REQUEST, [], b"Invalid path\n"
########################################### ###########################################
# # TODO: logging
# # TODO: clean shutdown / kill all connections # # TODO: clean shutdown / kill all connections
# # TODO: API # # TODO: API
# # TODO: Filter object # # TODO: Filter object
# # TODO: process_request check
# # TODO: IP/uuid to block # # TODO: IP/uuid to block
if __name__ == '__main__': if __name__ == '__main__':
@ -237,6 +237,7 @@ if __name__ == '__main__':
port = 4443 port = 4443
print('Launching Server...') print('Launching Server...')
redis_logger.info('Launching Server...')
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
cert_dir = os.environ['AIL_FLASK'] cert_dir = os.environ['AIL_FLASK']
@ -245,6 +246,7 @@ if __name__ == '__main__':
start_server = websockets.serve(ail_to_ail_serv, "localhost", 4443, ssl=ssl_context, create_protocol=AIL_2_AIL_Protocol) start_server = websockets.serve(ail_to_ail_serv, "localhost", 4443, ssl=ssl_context, create_protocol=AIL_2_AIL_Protocol)
print(f'Server Launched: wss://{host}:{port}') print(f'Server Launched: wss://{host}:{port}')
redis_logger.info(f'Server Launched: wss://{host}:{port}')
asyncio.get_event_loop().run_until_complete(start_server) asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever() asyncio.get_event_loop().run_forever()

View file

@ -63,11 +63,17 @@ def get_galaxy_from_tag(tag):
except IndexError: except IndexError:
return None return None
def get_active_taxonomies(): def get_active_taxonomies(r_set=False):
return r_serv_tags.smembers('active_taxonomies') res = r_serv_tags.smembers('active_taxonomies')
if r_set:
return set(res)
return res
def get_active_galaxies(): def get_active_galaxies(r_set=False):
return r_serv_tags.smembers('active_galaxies') res = r_serv_tags.smembers('active_galaxies')
if r_set:
return set(res)
return res
def get_all_taxonomies_tags(): # # TODO: add + REMOVE + Update def get_all_taxonomies_tags(): # # TODO: add + REMOVE + Update
return r_serv_tags.smembers('active_taxonomies_tags') return r_serv_tags.smembers('active_taxonomies_tags')
@ -75,6 +81,44 @@ def get_all_taxonomies_tags(): # # TODO: add + REMOVE + Update
def get_all_galaxies_tags(): # # TODO: add + REMOVE + Update def get_all_galaxies_tags(): # # TODO: add + REMOVE + Update
return r_serv_tags.smembers('active_galaxies_tags') return r_serv_tags.smembers('active_galaxies_tags')
def get_taxonomies_enabled_tags(r_list=False):
l_tag_keys = []
for taxonomie in get_active_taxonomies():
l_tag_keys.append(f'active_tag_{taxonomie}')
if len(l_tag_keys) > 1:
res = r_serv_tags.sunion(l_tag_keys[0], *l_tag_keys[1:])
elif l_tag_keys:
res = r_serv_tags.smembers(l_tag_keys[0])
if r_list:
return list(res)
else:
return res
def get_galaxies_enabled_tags():
l_tag_keys = []
for galaxy in get_active_galaxies():
l_tag_keys.append(f'active_tag_galaxies_{galaxy}')
if len(l_tag_keys) > 1:
return r_serv_tags.sunion(l_tag_keys[0], *l_tag_keys[1:])
elif l_tag_keys:
return r_serv_tags.smembers(l_tag_keys[0])
else:
return []
def get_taxonomie_enabled_tags(taxonomie, r_list=False):
res = r_serv_tags.smembers(f'active_tag_{taxonomie}')
if r_list:
return list(res)
else:
return res
def get_galaxy_enabled_tags(galaxy, r_list=False):
res = r_serv_tags.smembers(f'active_tag_galaxies_{galaxy}')
if r_list:
return list(res)
else:
return res
def is_taxonomie_tag_enabled(taxonomie, tag): def is_taxonomie_tag_enabled(taxonomie, tag):
if tag in r_serv_tags.smembers('active_tag_' + taxonomie): if tag in r_serv_tags.smembers('active_tag_' + taxonomie):
return True return True
@ -136,6 +180,67 @@ def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy):
return False return False
return True return True
def is_taxonomie_tag(tag, namespace=None):
if not namespace:
namespace = tag.split(':')[0]
if namespace != 'misp-galaxy':
return True
else:
return False
def is_galaxy_tag(tag, namespace=None):
if not namespace:
namespace = tag.split(':')[0]
if namespace == 'misp-galaxy':
return True
else:
return False
# # TODO:
# def is_valid_tag(tag):
# pass
def is_enabled_tag(tag, enabled_namespace=None):
if is_taxonomie_tag(tag):
return is_enabled_taxonomie_tag(tag, enabled_taxonomies=enabled_namespace)
else:
return is_enabled_galaxy_tag(tag, enabled_galaxies=enabled_namespace)
def are_enabled_tags(tags):
enabled_taxonomies = get_active_taxonomies(r_set=True)
enabled_galaxies = get_active_galaxies(r_set=True)
for tag in tags:
if is_taxonomie_tag(tag):
res = is_enabled_taxonomie_tag(tag, enabled_taxonomies=enabled_taxonomies)
else:
res = is_enabled_galaxy_tag(tag, enabled_galaxies=enabled_galaxies)
if not res:
return False
return True
def is_enabled_taxonomie_tag(tag, enabled_taxonomies=None):
if not enabled_taxonomies:
enabled_taxonomies = get_active_taxonomies()
taxonomie = get_taxonomie_from_tag(tag)
if taxonomie is None:
return False
if taxonomie not in enabled_taxonomies:
return False
if not is_taxonomie_tag_enabled(taxonomie, tag):
return False
def is_enabled_galaxy_tag(tag, enabled_galaxies=None):
if not enabled_galaxies:
enabled_galaxies = get_active_galaxies()
galaxy = get_galaxy_from_tag(tag)
if galaxy is None:
return False
if galaxy not in enabled_galaxies:
return False
if not is_galaxy_tag_enabled(galaxy, tag):
return False
return True
#### #### #### ####
def is_tag_in_all_tag(tag): def is_tag_in_all_tag(tag):
@ -144,6 +249,31 @@ def is_tag_in_all_tag(tag):
else: else:
return False return False
def get_tag_synonyms(tag):
return r_serv_tags.smembers(f'synonym_tag_{tag}')
def get_tag_dislay_name(tag):
tag_synonyms = get_tag_synonyms(tag)
if not tag_synonyms:
return tag
else:
return tag + ', '.join(tag_synonyms)
def get_tags_selector_dict(tags):
list_tags = []
for tag in tags:
list_tags.append(get_tag_selector_dict(tag))
return list_tags
def get_tag_selector_dict(tag):
return {'name':get_tag_dislay_name(tag),'id':tag}
def get_tags_selector_data():
dict_selector = {}
dict_selector['active_taxonomies'] = get_active_taxonomies()
dict_selector['active_galaxies'] = get_active_galaxies()
return dict_selector
def get_min_tag(tag): def get_min_tag(tag):
tag = tag.split('=') tag = tag.split('=')
if len(tag) > 1: if len(tag) > 1:

View file

@ -9,6 +9,9 @@ redis==2.10.6
python-magic>0.4.15 python-magic>0.4.15
yara-python>4.0.2 yara-python>4.0.2
# AIL Sync
websockets>9.0
# Hashlib # Hashlib
crcmod crcmod
mmh3>2.5 mmh3>2.5

26
update/v4.0/Update.py Executable file
View file

@ -0,0 +1,26 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
from ail_updater import AIL_Updater
class Updater(AIL_Updater):
"""default Updater."""
def __init__(self, version):
super(Updater, self).__init__(version)
if __name__ == '__main__':
updater = Updater('v4.0')
updater.run_update()

29
update/v4.0/Update.sh Executable file
View file

@ -0,0 +1,29 @@
#!/bin/bash
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
export PATH=$AIL_HOME:$PATH
export PATH=$AIL_REDIS:$PATH
export PATH=$AIL_ARDB:$PATH
export PATH=$AIL_BIN:$PATH
export PATH=$AIL_FLASK:$PATH
GREEN="\\033[1;32m"
DEFAULT="\\033[0;39m"
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -ks
wait
# SUBMODULES #
git submodule update
echo ""
echo -e $GREEN"Installing nose ..."$DEFAULT
pip3 install -U websockets
exit 0

View file

@ -45,6 +45,7 @@ from blueprints.import_export import import_export
from blueprints.objects_item import objects_item from blueprints.objects_item import objects_item
from blueprints.hunters import hunters from blueprints.hunters import hunters
from blueprints.old_endpoints import old_endpoints from blueprints.old_endpoints import old_endpoints
from blueprints.ail_2_ail_sync import ail_2_ail_sync
Flask_dir = os.environ['AIL_FLASK'] Flask_dir = os.environ['AIL_FLASK']
@ -103,6 +104,7 @@ app.register_blueprint(import_export, url_prefix=baseUrl)
app.register_blueprint(objects_item, url_prefix=baseUrl) app.register_blueprint(objects_item, url_prefix=baseUrl)
app.register_blueprint(hunters, url_prefix=baseUrl) app.register_blueprint(hunters, url_prefix=baseUrl)
app.register_blueprint(old_endpoints, url_prefix=baseUrl) app.register_blueprint(old_endpoints, url_prefix=baseUrl)
app.register_blueprint(ail_2_ail_sync, url_prefix=baseUrl)
# ========= =========# # ========= =========#
# ========= Cookie name ======== # ========= Cookie name ========

View file

@ -0,0 +1,247 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
'''
import os
import sys
import json
import random
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, make_response
from flask_login import login_required, current_user, login_user, logout_user
sys.path.append('modules')
import Flask_config
# Import Role_Manager
from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
from Role_Manager import login_admin, login_analyst, login_read_only
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import Tag
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import item_basic
import Tracker
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'core'))
import ail_2_ail
bootstrap_label = Flask_config.bootstrap_label
# ============ BLUEPRINT ============
ail_2_ail_sync = Blueprint('ail_2_ail_sync', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/ail_2_ail'))
# ============ VARIABLES ============
# ============ FUNCTIONS ============
def api_validator(api_response):
if api_response:
return Response(json.dumps(api_response[0], indent=2, sort_keys=True), mimetype='application/json'), api_response[1]
def create_json_response(data, status_code):
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
# ============= ROUTES ==============
@ail_2_ail_sync.route('/settings/ail_2_ail', methods=['GET'])
@login_required
@login_admin
def ail_2_ail_dashboard():
l_servers = ail_2_ail.get_all_running_sync_servers()
l_servers = ail_2_ail.get_ail_instances_metadata(l_servers)
return render_template("ail_2_ail_dashboard.html", l_servers=l_servers)
######################
# #
#### AIL INSTANCE ####
# # TODO: add more metadata => queues + connections
@ail_2_ail_sync.route('/settings/ail_2_ail/servers', methods=['GET'])
@login_required
@login_admin
def ail_servers():
l_servers = ail_2_ail.get_all_ail_instances_metadata()
return render_template("ail_servers.html", l_servers=l_servers)
@ail_2_ail_sync.route('/settings/ail_2_ail/server/view', methods=['GET'])
@login_required
@login_admin
def ail_server_view():
ail_uuid = request.args.get('uuid')
server_metadata = ail_2_ail.get_ail_instance_metadata(ail_uuid,sync_queues=True)
server_metadata['sync_queues'] = ail_2_ail.get_queues_metadata(server_metadata['sync_queues'])
return render_template("view_ail_server.html", server_metadata=server_metadata,
bootstrap_label=bootstrap_label)
@ail_2_ail_sync.route('/settings/ail_2_ail/server/add', methods=['GET', 'POST'])
@login_required
@login_admin
def ail_server_add():
if request.method == 'POST':
register_key = request.form.get("register_key")
ail_uuid = request.form.get("ail_uuid")
url = request.form.get("ail_url")
description = request.form.get("ail_description")
pull = request.form.get("ail_pull")
push = request.form.get("ail_push")
input_dict = {"uuid": ail_uuid, "url": url,
"description": description,
"pull": pull, "push": push}
if register_key:
input_dict['key'] = request.form.get("ail_key")
print(input_dict)
res = ail_2_ail.api_create_ail_instance(input_dict)
if res[1] != 200:
return create_json_response(res[0], res[1])
return redirect(url_for('ail_2_ail_sync.ail_server_view', uuid=res))
else:
return render_template("add_ail_server.html")
@ail_2_ail_sync.route('/settings/ail_2_ail/server/edit', methods=['GET', 'POST'])
@login_required
@login_admin
def ail_server_edit():
ail_uuid = request.args.get('ail_uuid')
@ail_2_ail_sync.route('/settings/ail_2_ail/server/delete', methods=['GET'])
@login_required
@login_admin
def ail_server_delete():
ail_uuid = request.args.get('uuid')
input_dict = {"uuid": ail_uuid}
res = ail_2_ail.api_delete_ail_instance(input_dict)
if res[1] != 200:
return create_json_response(res[0], res[1])
return redirect(url_for('ail_2_ail_sync.ail_servers'))
@ail_2_ail_sync.route('/settings/ail_2_ail/server/sync_queues', methods=['GET'])
@login_required
@login_admin
def ail_server_sync_queues():
ail_uuid = request.args.get('uuid')
sync_queues = ail_2_ail.get_all_unregistred_queue_by_ail_instance(ail_uuid)
sync_queues = ail_2_ail.get_queues_metadata(sync_queues)
return render_template("register_queue.html", bootstrap_label=bootstrap_label,
ail_uuid=ail_uuid, sync_queues=sync_queues)
@ail_2_ail_sync.route('/settings/ail_2_ail/server/sync_queues/register', methods=['GET'])
@login_required
@login_admin
def ail_server_sync_queues_register():
ail_uuid = request.args.get('ail_uuid')
queue_uuid = request.args.get('queue_uuid')
input_dict = {"ail_uuid": ail_uuid, "queue_uuid": queue_uuid}
res = ail_2_ail.api_register_ail_to_sync_queue(input_dict)
if res[1] != 200:
return create_json_response(res[0], res[1])
return redirect(url_for('ail_2_ail_sync.ail_server_view', uuid=ail_uuid))
@ail_2_ail_sync.route('/settings/ail_2_ail/server/sync_queues/unregister', methods=['GET'])
@login_required
@login_admin
def ail_server_sync_queues_unregister():
ail_uuid = request.args.get('ail_uuid')
queue_uuid = request.args.get('queue_uuid')
input_dict = {"ail_uuid": ail_uuid, "queue_uuid": queue_uuid}
res = ail_2_ail.api_unregister_ail_to_sync_queue(input_dict)
if res[1] != 200:
return create_json_response(res[0], res[1])
return redirect(url_for('ail_2_ail_sync.ail_server_view', uuid=ail_uuid))
####################
# #
#### SYNC QUEUE ####
@ail_2_ail_sync.route('/settings/ail_2_ail/sync_queues', methods=['GET'])
# @login_required
# @login_admin
def sync_queues():
ail_uuid = request.args.get('ail_uuid')
l_queues = ail_2_ail.get_all_queues_metadata()
return render_template("sync_queues.html", bootstrap_label=bootstrap_label,
ail_uuid=ail_uuid, l_queues=l_queues)
@ail_2_ail_sync.route('/settings/ail_2_ail/sync_queue/view', methods=['GET'])
# @login_required
# @login_admin
def sync_queue_view():
queue_uuid = request.args.get('uuid')
queue_metadata = ail_2_ail.get_sync_queue_metadata(queue_uuid)
ail_servers = ail_2_ail.get_sync_queue_all_ail_instance(queue_uuid)
queue_metadata['ail_servers'] = ail_2_ail.get_ail_instances_metadata(ail_servers)
return render_template("view_sync_queue.html", queue_metadata=queue_metadata,
bootstrap_label=bootstrap_label)
@ail_2_ail_sync.route('/settings/ail_2_ail/sync_queue/add', methods=['GET', 'POST'])
@login_required
@login_admin
def sync_queue_add():
if request.method == 'POST':
queue_name = request.form.get("queue_name")
description = request.form.get("queue_description")
max_size = request.form.get("queue_max_size")
taxonomies_tags = request.form.get('taxonomies_tags')
if taxonomies_tags:
try:
taxonomies_tags = json.loads(taxonomies_tags)
except Exception:
taxonomies_tags = []
else:
taxonomies_tags = []
galaxies_tags = request.form.get('galaxies_tags')
if galaxies_tags:
try:
galaxies_tags = json.loads(galaxies_tags)
except Exception:
galaxies_tags = []
tags = taxonomies_tags + galaxies_tags
input_dict = {"name": queue_name, "tags": tags,
"description": description,
"max_size": max_size}
res = ail_2_ail.api_create_sync_queue(input_dict)
if res[1] != 200:
return create_json_response(res[0], res[1])
return redirect(url_for('ail_2_ail_sync.sync_queue_view', uuid=res))
else:
return render_template("add_sync_queue.html", tags_selector_data=Tag.get_tags_selector_data())
@ail_2_ail_sync.route('/settings/ail_2_ail/sync_queue/edit', methods=['GET', 'POST'])
# @login_required
# @login_admin
def sync_queue_edit():
return ''
@ail_2_ail_sync.route('/settings/ail_2_ail/sync_queue/delete', methods=['GET'])
# @login_required
# @login_admin
def sync_queue_delete():
queue_uuid = request.args.get('uuid')
input_dict = {"uuid": queue_uuid}
res = ail_2_ail.api_delete_sync_queue(input_dict)
if res[1] != 200:
return create_json_response(res[0], res[1])
return redirect(url_for('ail_2_ail_sync.sync_queues'))
#### JSON ####
## - - ##

View file

@ -94,6 +94,34 @@ def get_all_obj_tags():
return jsonify(res) return jsonify(res)
return jsonify(Tag.get_all_obj_tags(object_type)) return jsonify(Tag.get_all_obj_tags(object_type))
@tags_ui.route('/tag/taxonomies/tags/enabled/json')
@login_required
@login_read_only
def tag_taxonomies_tags_enabled_json():
return jsonify(Tag.get_taxonomies_enabled_tags(r_list=True))
@tags_ui.route('/tag/galaxies/tags/enabled/json')
@login_required
@login_read_only
def tag_galaxies_tags_enabled_json():
tags = Tag.get_galaxies_enabled_tags()
return jsonify(Tag.get_tags_selector_dict(tags))
@tags_ui.route('/tag/taxonomie/tags/enabled/json')
@login_required
@login_read_only
def tag_taxonomie_tags_enabled_json():
taxonomie = request.args.get('taxonomie')
return jsonify(Tag.get_taxonomie_enabled_tags(taxonomie, r_list=True))
@tags_ui.route('/tag/galaxy/tags/enabled/json')
@login_required
@login_read_only
def tag_galaxy_tags_enabled_json():
galaxy = request.args.get('galaxy')
tags = Tag.get_galaxy_enabled_tags(galaxy, r_list=True)
return jsonify(Tag.get_tags_selector_dict(tags))
@tags_ui.route('/tag/search/item') @tags_ui.route('/tag/search/item')
@login_required @login_required
@login_read_only @login_read_only

View file

@ -59,6 +59,13 @@ for name, tags in clusters.items(): #galaxie name + tags
def one(): def one():
return 1 return 1
# TODO:
# TODO:
# TODO:
# TODO:
# TODO:
# TODO:
# # TODO: replace me with get_tag_selector_dict()
def get_tags_with_synonyms(tag): def get_tags_with_synonyms(tag):
str_synonyms = ' - synonyms: ' str_synonyms = ' - synonyms: '
synonyms = r_serv_tags.smembers('synonym_tag_' + tag) synonyms = r_serv_tags.smembers('synonym_tag_' + tag)

View file

@ -0,0 +1,170 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL-Framework</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'settings/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-header bg-dark text-white">
<h5 class="card-title">Create AIL Server</h5>
</div>
<div class="card-body">
<form action="{{ url_for('ail_2_ail_sync.ail_server_add') }}" method='post'>
<div class="row">
<div class="col-12 col-xl-9">
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-dark text-white"><i class="fas fa-quote-right"></i></div>
</div>
<input id="ail_uuid" name="ail_uuid" class="form-control" placeholder="AIL Server UUID" type="text" required>
</div>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-secondary text-white"><i class="fas fa-server"></i></div>
</div>
<input id="ail_url" class="form-control" type="text" name="ail_url" placeholder="AIL Server URL">
</div>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-info text-white"><i class="fas fa-pencil-alt"></i></div>
</div>
<textarea id="ail_description" name="ail_description" class="form-control" placeholder="AIL Server Description (optional)" rows="3"></textarea>
</div>
<h5 class="mt-2 text-secondary">
SYNC Modes:
</h5>
<div class="row">
<div class="col-12 col-xl-6">
<div class="custom-control custom-checkbox">
<input class="custom-control-input" type="checkbox" value="True" id="pull" name="ail_pull" checked>
<label class="custom-control-label" for="ail_pull"><h5>PULL</h5></label>
</div>
</div>
<div class="col-12 col-xl-6">
<div class="custom-control custom-checkbox">
<input class="custom-control-input" type="checkbox" value="True" id="ail_push" name="ail_push" checked>
<label class="custom-control-label" for="ail_push"><h5 class="">PUSH</h5></label>
</div>
</div>
</div>
<h5 class="mt-2 text-secondary">
Server Key <i class="fas fa-key"></i>:
</h5>
<div class="d-flex mt-2">
&nbsp;Generate Server Key&nbsp;&nbsp;
<div class="custom-control custom-switch">
<input class="custom-control-input" type="checkbox" name="register_key" value="True" id="register_key">
<label class="custom-control-label" for="register_key">
&nbsp;Register Server Key
</label>
</div>
</div>
<div class="mt-3" id="div_generate_key">
<b>A new key will be generated for this AIL Server</b>
</div>
<div id="div_register_key">
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-danger text-white"><i class="fas fa-key"></i></div>
</div>
<input id="ail_key" class="form-control" type="text" minlength="56" maxlength="56" name="ail_key" placeholder="AIL key">
</div>
</div>
</div>
<div class="col-12 col-xl-3">
SYNC: create a new AIL Server
</div>
</div>
<br>
<button class="btn btn-primary mt-2">
<i class="fas fa-plus"></i> Create AIL Server
</button>
</form>
</div>
</div>
</div>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$('#nav_sync').removeClass("text-muted");
$("#nav_ail_servers").addClass("active");
$("#div_register_key").hide();
$('#register_key').on("change", function () {
register_key_input_controler();
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
function register_key_input_controler() {
if($('#register_key').is(':checked')){
$("#div_generate_key").hide();
$("#div_register_key").show();
}else{
$("#div_generate_key").show();
$("#div_register_key").hide();
}
}
</script>

View file

@ -0,0 +1,139 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL-Framework</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'settings/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-header bg-dark text-white">
<h5 class="card-title">Create SYNC Queue</h5>
</div>
<div class="card-body">
<form action="{{ url_for('ail_2_ail_sync.sync_queue_add') }}" method='post' onsubmit="SubmitCreateQueue();">
<div class="row">
<div class="col-12 col-xl-9">
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-dark text-white"><i class="fas fa-quote-right"></i></div>
</div>
<input id="queue_name" name="queue_name" class="form-control" placeholder="Queue Name" type="text" required>
</div>
<div class="input-group form-group mb-2">
<div class="input-group-prepend">
<span class="input-group-text bg-success"><i class="fas fa-water"></i></span>
</div>
<input class="form-control" type="number" id="queue_max_size" name="queue_max_size" min="1" value="100" required>
<div class="input-group-append">
<span class="input-group-text">Queue Max Size</span>
</div>
</div>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-info text-white"><i class="fas fa-pencil-alt"></i></div>
</div>
<textarea id="queue_description" name="queue_description" class="form-control" placeholder="Sync Queue Description (optional)" rows="3"></textarea>
</div>
<div class="card my-4">
<div class="card-header bg-secondary text-white">
<b>Tags Filter</b>
</div>
<div class="card-body">
{% include 'tags/block_tags_selector.html' %}
</div>
</div>
</div>
<div class="col-12 col-xl-3">
SYNC: create a new Sync Queue
</div>
</div>
<br>
<button class="btn btn-primary mt-2">
<i class="fas fa-plus"></i> Create Sync Queue
</button>
</form>
</div>
</div>
</div>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$('#nav_sync').removeClass("text-muted");
$("#nav_ail_servers").addClass("active");
$("#div_register_key").hide();
$('#register_key').on("change", function () {
register_key_input_controler();
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
function SubmitCreateQueue() {
var tags = ltags.getValue();
var tagsgalaxy = ltagsgalaxies.getValue();
console.log(tags);
console.log(tagsgalaxy);
$('#ltags').val(tags);
$('#ltagsgalaxies').val(tagsgalaxy);
return true;
}
</script>

View file

@ -0,0 +1,100 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL-SYNC</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'settings/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<h1>Connected Servers:</h3>
<table id="table_servers" class="table table-striped border-primary">
<thead class="bg-dark text-white">
<tr>
<th>uuid</th>
<th>url</th>
<th>description</th>
<th>sync queues</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for dict_server in l_servers %}
<tr class="border-color: blue;">
<td>
<a href="{{ url_for('ail_2_ail_sync.ail_server_view') }}?uuid={{ dict_server['uuid'] }}">
{{ dict_server['uuid']}}
</a>
</td>
<td>{{ dict_server['url']}}</td>
<td>{{ dict_server['description']}}</td>
<td class="text-center">
{% for queue_uuid in dict_server['sync_queues'] %}
<a href="{{ url_for('ail_2_ail_sync.sync_queue_view') }}?uuid={{queue_uuid}}">
{{queue_uuid}}</br>
</a>
{% endfor %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$('#nav_sync').removeClass("text-muted");
$("#nav_ail_servers").addClass("active");
$('#table_servers').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 1, "desc" ]]
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>

View file

@ -0,0 +1,106 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL-SYNC</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'settings/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="my-4">
<a href="{{ url_for('ail_2_ail_sync.ail_server_add') }}">
<button type="button" class="btn btn-primary">
<i class="fas fa-plus"></i> Add AIL Server
</button>
</a>
</div>
<table id="table_servers" class="table table-striped border-primary">
<thead class="bg-dark text-white">
<tr>
<th>uuid</th>
<th>url</th>
<th>description</th>
<th>sync queues</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for dict_server in l_servers %}
<tr class="border-color: blue;">
<td>
<a href="{{ url_for('ail_2_ail_sync.ail_server_view') }}?uuid={{ dict_server['uuid'] }}">
{{ dict_server['uuid']}}
</a>
</td>
<td>{{ dict_server['url']}}</td>
<td>{{ dict_server['description']}}</td>
<td class="text-center">
{% for queue_uuid in dict_server['sync_queues'] %}
<a href="{{ url_for('ail_2_ail_sync.sync_queue_view') }}?uuid={{queue_uuid}}">
{{queue_uuid}}</br>
</a>
{% endfor %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$('#nav_sync').removeClass("text-muted");
$("#nav_ail_servers").addClass("active");
$('#table_servers').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 1, "desc" ]]
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>

View file

@ -0,0 +1,110 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL-Framework</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'settings/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card">
<div class="card-header bg-dark text-white">
<h5 class="card-title">{{ail_uuid}} Register a SYNC Queue</h5>
</div>
<div class="card-body">
<table id="table_sync_queues" class="table table-striped border-primary">
<thead class="bg-dark text-white">
<tr>
<th>name</th>
<th>uuid</th>
<th>description</th>
<th>max size</th>
<th>Register Sync Queue</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for dict_queue in sync_queues %}
<tr class="border-color: blue;">
<td>{{ dict_queue['name']}}</td>
<td>
<a href="{{ url_for('ail_2_ail_sync.sync_queue_view') }}?uuid={{ dict_queue['uuid'] }}">
{{ dict_queue['uuid']}}
</a>
<div>
{% for tag in dict_queue['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
{% endfor %}
</div>
</td>
<td>{{ dict_queue['description']}}</td>
<td>{{ dict_queue['max_size']}}</td>
<td class="text-right">
<a href="{{ url_for('ail_2_ail_sync.ail_server_sync_queues_register') }}?ail_uuid={{ ail_uuid }}&queue_uuid={{ dict_queue['uuid'] }}">
<button type="button" class="btn btn-primary"><i class="fas fa-plus"></i></button>
</a>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$('#nav_sync').removeClass("text-muted");
$('#table_sync_queues').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 0, "desc" ]]
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>

View file

@ -0,0 +1,105 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL-SYNC</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'settings/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="my-4">
<a href="{{ url_for('ail_2_ail_sync.sync_queue_add') }}">
<button type="button" class="btn btn-primary">
<i class="fas fa-plus"></i> Create Sync Queue
</button>
</a>
</div>
<table id="table_servers" class="table table-striped border-primary">
<thead class="bg-dark text-white">
<tr>
<th>name</th>
<th>uuid</th>
<th>description</th>
<th>max size</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for dict_queue in l_queues %}
<tr class="border-color: blue;">
<td>{{ dict_queue['name']}}</td>
<td>
<a href="{{ url_for('ail_2_ail_sync.sync_queue_view') }}?uuid={{ dict_queue['uuid'] }}">
{{ dict_queue['uuid']}}
<div>
{% for tag in dict_queue['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
{% endfor %}
</div>
</a>
</td>
<td>{{ dict_queue['description']}}</td>
<td>{{ dict_queue['max_size']}}</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$('#nav_sync').removeClass("text-muted");
$("#navsync_queues").addClass("active");
$('#table_servers').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 1, "desc" ]]
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>

View file

@ -0,0 +1,168 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL-Framework</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'settings/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card">
<div class="card-header bg-dark text-white">
<h5 class="card-title">{{server_metadata['uuid']}}</h5>
</div>
<div class="card-body">
<table class="table table-borderless">
<tbody>
<tr>
<td class="text-right"><b>URL</b></td>
<td>
{{server_metadata['url']}}
</td>
</tr>
<tr>
<td class="text-right"><b>Api Key</b></td>
<td>
{{server_metadata['api_key']}}
</td>
</tr>
<tr>
<td class="text-right"><b>Description</b></td>
<td>
{{server_metadata['description']}}
</td>
</tr>
<tr>
<td class="text-right"><b>Pull</b></td>
<td>
{{server_metadata['pull']}}
</td>
</tr>
<tr>
<td class="text-right"><b>Push</b></td>
<td>
{{server_metadata['push']}}
</td>
</tr>
</tbody>
</table>
<div class="my-4">
<a href="{{ url_for('ail_2_ail_sync.ail_server_delete') }}?uuid={{server_metadata['uuid']}}">
<button type="button" class="btn btn-danger">
<i class="fas fa-trash-alt"></i> <b>Delete AIL Server</b>
</button>
</a>
</div>
<div class="card">
<div class="card-header bg-secondary text-white">
<h6 class="card-title"><b>SYNC QUEUES:</b></h6>
</div>
<div class="card-body">
<a href="{{ url_for('ail_2_ail_sync.ail_server_sync_queues') }}?uuid={{ server_metadata['uuid'] }}">
<button type="button" class="btn btn-primary my-4">
<i class="fas fa-plus"></i> <b>Register Sync Queue</b>
</button>
</a>
<table id="table_sync_queues" class="table table-striped border-primary">
<thead class="bg-dark text-white">
<tr>
<th>name</th>
<th>uuid</th>
<th>description</th>
<th>max size</th>
<th>Unregister Queue</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for dict_queue in server_metadata['sync_queues'] %}
<tr class="border-color: blue;">
<td>{{ dict_queue['name']}}</td>
<td>
<a href="{{ url_for('ail_2_ail_sync.sync_queue_view') }}?uuid={{ dict_queue['uuid'] }}">
{{ dict_queue['uuid']}}
</a>
<div>
{% for tag in dict_queue['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
{% endfor %}
</div>
</td>
<td>{{ dict_queue['description']}}</td>
<td>{{ dict_queue['max_size']}}</td>
<td class="text-right">
<a href="{{ url_for('ail_2_ail_sync.ail_server_sync_queues_unregister') }}?ail_uuid={{ server_metadata['uuid'] }}&queue_uuid={{ dict_queue['uuid'] }}">
<button type="button" class="btn btn-danger"><i class="fas fa-trash-alt"></i></button>
</a>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$('#nav_sync').removeClass("text-muted");
$('#table_sync_queues').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 0, "asc" ]]
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>

View file

@ -0,0 +1,159 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL-Framework</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'settings/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card">
<div class="card-header bg-dark text-white">
<h4 class="card-title">SYNC Queue: {{queue_metadata['uuid']}}</h4>
</div>
<div class="card-body">
<table class="table table-borderless">
<tbody>
<tr>
<td class="text-right"><b>Name</b></td>
<td>
{{queue_metadata['name']}}
</td>
</tr>
<tr>
<td class="text-right"><b>Tags Filter</b></td>
<td>
<div>
{% for tag in queue_metadata['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
{% endfor %}
</div>
</td>
</tr>
<tr>
<td class="text-right"><b>Description</b></td>
<td>
{{queue_metadata['description']}}
</td>
</tr>
<tr>
<td class="text-right"><b>Max Size</b></td>
<td>
{{queue_metadata['max_size']}}
</td>
</tr>
</tbody>
</table>
<div class="my-4">
<a href="{{ url_for('ail_2_ail_sync.sync_queue_delete') }}?uuid={{queue_metadata['uuid']}}">
<button type="button" class="btn btn-danger">
<i class="fas fa-trash-alt"></i> <b>Delete Sync Queue</b>
</button>
</a>
</div>
<div class="card">
<div class="card-header bg-secondary text-white">
<h5 class="card-title"><b>AIL Server:</b></h5>
</div>
<div class="card-body">
<a href="{{ url_for('ail_2_ail_sync.ail_server_add') }}">
<button type="button" class="btn btn-primary my-4">
<i class="fas fa-plus"></i> <b>Create AIL Server</b>
</button>
</a>
<table id="table_sync_queues" class="table table-striped border-primary">
<thead class="bg-dark text-white">
<tr>
<th>uuid</th>
<th>url</th>
<th>description</th>
<th>pull</th>
<th>push</th>
<th></th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for dict_server in queue_metadata['ail_servers'] %}
<tr class="border-color: blue;">
<td>{{ dict_server['uuid']}}</td>
<td>{{ dict_server['url']}}</td>
<td>{{ dict_server['description']}}</td>
<td>{{ dict_server['pull']}}</td>
<td>{{ dict_server['push']}}</td>
<td class="text-right">
<a href="{{ url_for('ail_2_ail_sync.ail_server_sync_queues_unregister') }}?ail_uuid={{ dict_server['uuid'] }}&queue_uuid={{ queue_metadata['uuid'] }}">
<button type="button" class="btn btn-danger"><i class="fas fa-trash-alt"></i></button>
</a>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$('#nav_sync').removeClass("text-muted");
$('#table_sync_queues').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 0, "asc" ]]
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>

View file

@ -17,6 +17,29 @@
</a> </a>
</li> </li>
</ul> </ul>
<h5 class="d-flex text-muted w-100" id="nav_sync">
<span>AIL SYNC</span>
</h5>
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
<li class="nav-item">
<a class="nav-link" href="{{url_for('ail_2_ail_sync.ail_2_ail_dashboard')}}" id="nav_ail_sync">
<img src="{{ url_for('static', filename='image/ail.png')}}" alt="AIL servers" style="width:25px;">
<span>AIL SYNC</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('ail_2_ail_sync.ail_servers')}}" id="nav_ail_servers">
<i class="fas fa-server"></i>
<span>Servers</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('ail_2_ail_sync.sync_queues')}}" id="navsync_queues">
<i class="fas fa-stream"></i>
<span>Sync queues</span>
</a>
</li>
</ul>
<h5 class="d-flex text-muted w-100" id="nav_settings"> <h5 class="d-flex text-muted w-100" id="nav_settings">
<span>Settings</span> <span>Settings</span>
</h5> </h5>

View file

@ -0,0 +1,108 @@
<div class="input-group" >
<input id="ltags" type="text" class="form-control" autocomplete="off" style="width: 760px" name="taxonomies_tags">
</div>
<div class="dropdown">
<button type="button" class="btn btn-info dropdown-toggle mt-1 mb-3" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" id="dropdown-taxonomie">
Taxonomie Selected
</button>
<div class="dropdown-menu dropdown-menu-right" aria-labelledby="dropdown-taxonomie"> <!-- TODO: make dropdown-scrollable -->
<h6 class="dropdown-header">Taxonomie Tags</h6>
<button class="dropdown-item" type="button" id="all-tags-taxonomies">All Tags <i class="fas fa-tags"></i></button>
<div class="dropdown-divider"></div>
{% for taxo in tags_selector_data['active_taxonomies'] %}
<button class="dropdown-item" type="button" id="{{ taxo }}-id{{ loop.index0 }}">{{ taxo }}</button>
{% endfor %}
</div>
</div>
<div class="input-group">
<input id="ltagsgalaxies" type="text" class="form-control" autocomplete="off" style="width: 760px" name="galaxies_tags">
</div>
<div class="dropdown">
<button type="button" class="btn btn-info dropdown-toggle mt-1" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" id="dropdown-galaxy">
Galaxy Selected
</button>
<div class="dropdown-menu dropdown-menu-right" aria-labelledby="dropdown-galaxy"> <!-- TODO: make dropdown-scrollable -->
<h6 class="dropdown-header">Galaxy Tags</h6>
<button class="dropdown-item" type="button" id="all-tags-galaxies">All Tags <i class="fas fa-tags"></i></button>
<div class="dropdown-divider"></div>
{% for galaxy in tags_selector_data['active_galaxies'] %}
<button class="dropdown-item" type="button" id="{{ galaxy }}-idgalax{{ loop.index0 }}">{{ galaxy }}</button>
{% endfor %}
</div>
</div>
<!--
<button class="btn btn-primary" onclick="tagsSelector()">
<i class="fas fa-plus"></i>
Add Tags
</button>
-->
<script>
var ltags;
var ltagsgalaxies;
$.getJSON("{{ url_for('tags_ui.tag_taxonomies_tags_enabled_json') }}",
function(data) {
ltags = $('#ltags').tagSuggest({
data: data,
maxDropHeight: 200,
name: 'ltags'
});
});
$.getJSON("{{ url_for('tags_ui.tag_galaxies_tags_enabled_json') }}",
function(data) {
ltagsgalaxies = $('#ltagsgalaxies').tagSuggest({
data: data,
maxDropHeight: 200,
name: 'ltagsgalaxies'
});
});
jQuery("#all-tags-taxonomies").click(function(e){
//change input tags list
$.getJSON("{{ url_for('tags_ui.tag_taxonomies_tags_enabled_json') }}",
function(data) {
ltags.setData(data)
});
});
jQuery("#all-tags-galaxies").click(function(e){
$.getJSON("{{ url_for('tags_ui.tag_galaxies_tags_enabled_json') }}",
function(data) {
ltagsgalaxies.setData(data)
});
});
{% for taxo in tags_selector_data['active_taxonomies'] %}
jQuery("#{{ taxo }}-id{{ loop.index0 }}").click(function(e){
$.getJSON("{{ url_for('tags_ui.tag_taxonomie_tags_enabled_json') }}?taxonomie={{ taxo }}",
function(data) {
ltags.setData(data)
});
});
{% endfor %}
{% for galaxy in tags_selector_data['active_galaxies'] %}
jQuery("#{{ galaxy }}-idgalax{{ loop.index0 }}").click(function(e){
$.getJSON("{{ url_for('tags_ui.tag_galaxy_tags_enabled_json') }}?galaxy={{ galaxy }}",
function(data) {
ltagsgalaxies.setData(data)
});
});
{% endfor %}
// function tagsSelector() {
// var tags = ltags.getValue()
// var tagsgalaxy = ltagsgalaxies.getValue()
// window.location.replace("myurl?tags=" + tags + "&tagsgalaxies=" + tagsgalaxy);
//}
</script>