chg: [ail sync UI] restarr/launch/kill sync connections + show sync mode api/pull/push

bin/core/ail_2_ail.py

@@ -121,6 +121,9 @@ def is_server_client_sync_mode_connected(ail_uuid, sync_mode):
     res = r_cache.hexists(f'ail_2_ail:server:client:{ail_uuid}', sync_mode)
     return res == 1
 
+def is_server_client_connected(ail_uuid):
+    return r_cache.sismember('ail_2_ail:server:all_clients', ail_uuid)
+
 def clear_server_connected_clients():
     for ail_uuid in get_server_all_connected_clients():
         r_cache.delete(f'ail_2_ail:server:client:{ail_uuid}')
@@ -144,6 +147,11 @@ def send_command_to_server_controller(command, ail_uuid=None):
     r_cache.sadd('ail_2_ail:server_controller:command', str_command)
 
 ##-- --##
+def get_new_sync_client_id():
+    for new_id in range(120000, 100000, -1):
+        new_id = str(new_id)
+        if not r_cache.exists(f'ail_2_ail:sync_client:{new_id}'):
+            return str(new_id)
 
 def get_all_sync_clients(r_set=False):
     res = r_cache.smembers('ail_2_ail:all_sync_clients')
@@ -155,11 +163,38 @@ def get_all_sync_clients(r_set=False):
 def get_sync_client_ail_uuid(client_id):
     return r_cache.hget(f'ail_2_ail:sync_client:{client_id}', 'ail_uuid')
 
+def get_sync_client_sync_mode(client_id):
+    return r_cache.hget(f'ail_2_ail:sync_client:{client_id}', 'sync_mode')
+
+def set_sync_client_sync_mode(client_id, sync_mode):
+    r_cache.hset(f'ail_2_ail:sync_client:{client_id}', 'sync_mode', sync_mode)
+
+def create_sync_client_cache(ail_uuid, sync_mode, client_id=None):
+    if client_id is None:
+        client_id = get_new_sync_client_id()
+    # save sync client status
+    r_cache.hset(f'ail_2_ail:sync_client:{client_id}', 'ail_uuid', ail_uuid)
+    r_cache.hset(f'ail_2_ail:sync_client:{client_id}', 'launch_time', int(time.time()))
+    set_sync_client_sync_mode(client_id, sync_mode)
+
+    r_cache.sadd('ail_2_ail:all_sync_clients', client_id)
+
+    # create map ail_uuid/queue_uuid
+    r_cache.sadd(f'ail_2_ail:ail_uuid:{ail_uuid}', client_id)
+    return client_id
+
 # current: only one push registred
-def get_client_id_by_ail_uuid(ail_uuid):
+def get_client_id_by_ail_uuid(ail_uuid, filter_push=True):
     res = r_cache.smembers(f'ail_2_ail:ail_uuid:{ail_uuid}')
-    if res:
+    if not filter_push:
-        return int(res.pop())
+        return res
+    else:
+        clients_id = []
+        for client_id in res:
+            client_id = int(client_id)
+            if client_id <= 100000:
+                clients_id.append(client_id)
+        return clients_id
 
 def get_all_running_sync_servers():
     running_ail_servers= []
@@ -168,11 +203,18 @@ def get_all_running_sync_servers():
         running_ail_servers.append(ail_uuid)
     return running_ail_servers
 
+def get_ail_instance_all_running_sync_mode(ail_uuid):
+    clients_id = get_client_id_by_ail_uuid(ail_uuid, filter_push=False)
+    running_sync_mode = {'api': False, 'pull': False, 'push': False}
+    for client_id in clients_id:
+        sync_mode = get_sync_client_sync_mode(client_id)
+        running_sync_mode[sync_mode] = True
+    return running_sync_mode
+
 def delete_sync_client_cache(client_id):
     ail_uuid = get_sync_client_ail_uuid(client_id)
-    # map ail_uuid/queue_uuid
+    # map ail_uuid
     r_cache.srem(f'ail_2_ail:ail_uuid:{ail_uuid}', client_id)
-    r_cache.srem(f'ail_2_ail:queue_uuid:{queue_uuid}', client_id)
 
     r_cache.delete(f'ail_2_ail:sync_client:{client_id}')
     r_cache.srem('ail_2_ail:all_sync_clients', client_id)
@@ -193,9 +235,10 @@ def send_command_to_manager(command, client_id=-1, ail_uuid=None):
     str_command = json.dumps(dict_action)
     r_cache.sadd('ail_2_ail:client_manager:command', str_command)
 
-
 def refresh_ail_instance_connection(ail_uuid):
-    client_id = get_client_id_by_ail_uuid(ail_uuid)
+    clients_id = get_client_id_by_ail_uuid(ail_uuid)
+    if clients_id:
+        clients_id = clients_id[0]
     launch_required = is_ail_instance_push_enabled(ail_uuid)
 
     # relaunch
@@ -253,6 +296,7 @@ class AIL2AILClientManager(object):
 
     def launch_sync_client(self, ail_uuid):
         dir_project = os.environ['AIL_HOME']
+        sync_mode = 'push'
         client_id = self.get_new_sync_client_id()
         script_options = f'-u {ail_uuid} -m push -i {client_id}'
         screen.create_screen(AIL2AILClientManager.SCREEN_NAME)
@@ -262,13 +306,7 @@ class AIL2AILClientManager(object):
                                             AIL2AILClientManager.SCRIPT_NAME,
                                             script_options=script_options, kill_previous_windows=True)
         # save sync client status
-        r_cache.hset(f'ail_2_ail:sync_client:{client_id}', 'ail_uuid', ail_uuid)
+        create_sync_client_cache(ail_uuid, sync_mode, client_id=client_id)
-        r_cache.hset(f'ail_2_ail:sync_client:{client_id}', 'launch_time', int(time.time()))
-
-        r_cache.sadd('ail_2_ail:all_sync_clients', client_id)
-
-        # create map ail_uuid/queue_uuid
-        r_cache.sadd(f'ail_2_ail:ail_uuid:{ail_uuid}', client_id)
 
         self.clients[client_id] = {'ail_uuid': ail_uuid}
 
@@ -439,7 +477,7 @@ def get_ail_server_error(ail_uuid):
     return r_cache.hget(f'ail_2_ail:all_servers:metadata:{ail_uuid}', 'error')
 
 # # TODO: HIDE ADD GLOBAL FILTER (ON BOTH SIDE)
-def get_ail_instance_metadata(ail_uuid, client_sync_mode=False, sync_queues=False):
+def get_ail_instance_metadata(ail_uuid, client_sync_mode=False, server_sync_mode=False, sync_queues=False):
     dict_meta = {}
     dict_meta['uuid'] = ail_uuid
     dict_meta['url'] = get_ail_instance_url(ail_uuid)
@@ -462,6 +500,9 @@ def get_ail_instance_metadata(ail_uuid, client_sync_mode=False, sync_queues=Fals
         dict_meta['client_sync_mode']['push'] = is_server_client_sync_mode_connected(ail_uuid, 'push')
         dict_meta['client_sync_mode']['api'] = is_server_client_sync_mode_connected(ail_uuid, 'api')
 
+    if server_sync_mode:
+        dict_meta['server_sync_mode'] = get_ail_instance_all_running_sync_mode(ail_uuid)
+
     return dict_meta
 
 def get_all_ail_instances_metadata():
@@ -470,10 +511,11 @@ def get_all_ail_instances_metadata():
         l_servers.append(get_ail_instance_metadata(ail_uuid, sync_queues=True))
     return l_servers
 
-def get_ail_instances_metadata(l_ail_servers, sync_queues=True, client_sync_mode=False):
+def get_ail_instances_metadata(l_ail_servers, sync_queues=True, client_sync_mode=False, server_sync_mode=False):
     l_servers = []
     for ail_uuid in l_ail_servers:
-        server_metadata = get_ail_instance_metadata(ail_uuid, sync_queues=sync_queues, client_sync_mode=client_sync_mode)
+        server_metadata = get_ail_instance_metadata(ail_uuid, sync_queues=sync_queues,
+                                client_sync_mode=client_sync_mode, server_sync_mode=server_sync_mode)
         l_servers.append(server_metadata)
     return l_servers
 
@@ -644,6 +686,65 @@ def api_get_remote_ail_server_version(json_dict):
     res = get_remote_ail_server_version(ail_uuid)
     return res, 200
 
+def api_kill_server_connected_clients(json_dict):
+    ail_uuid = json_dict.get('uuid').replace(' ', '')
+    if not is_valid_uuid_v4(ail_uuid):
+        return {"status": "error", "reason": "Invalid ail uuid"}, 400
+    ail_uuid = sanityze_uuid(ail_uuid)
+    if not exists_ail_instance(ail_uuid):
+        return {"status": "error", "reason": "AIL server not found"}, 404
+    if not is_server_client_connected(ail_uuid):
+        return {"status": "error", "reason": "Client not connected"}, 400
+
+    res = send_command_to_server_controller('kill', ail_uuid=ail_uuid)
+    return res, 200
+
+def api_kill_sync_client(json_dict):
+    ail_uuid = json_dict.get('uuid').replace(' ', '')
+    if not is_valid_uuid_v4(ail_uuid):
+        return {"status": "error", "reason": "Invalid ail uuid"}, 400
+    ail_uuid = sanityze_uuid(ail_uuid)
+    if not exists_ail_instance(ail_uuid):
+        return {"status": "error", "reason": "AIL server not found"}, 404
+
+    clients_id = get_client_id_by_ail_uuid(ail_uuid)
+    if not clients_id:
+        return {"status": "error", "reason": "Client not connected"}, 400
+
+    for client_id in clients_id:
+        res = send_command_to_manager('kill', client_id=client_id, ail_uuid=ail_uuid)
+    return res, 200
+
+def api_launch_sync_client(json_dict):
+    ail_uuid = json_dict.get('uuid').replace(' ', '')
+    if not is_valid_uuid_v4(ail_uuid):
+        return {"status": "error", "reason": "Invalid ail uuid"}, 400
+    ail_uuid = sanityze_uuid(ail_uuid)
+    if not exists_ail_instance(ail_uuid):
+        return {"status": "error", "reason": "AIL server not found"}, 404
+
+    clients_id = get_client_id_by_ail_uuid(ail_uuid)
+    if clients_id:
+        return {"status": "error", "reason": "Client already connected"}, 400
+
+    res = send_command_to_manager('launch', ail_uuid=ail_uuid)
+    return res, 200
+
+def api_relaunch_sync_client(json_dict):
+    ail_uuid = json_dict.get('uuid').replace(' ', '')
+    if not is_valid_uuid_v4(ail_uuid):
+        return {"status": "error", "reason": "Invalid ail uuid"}, 400
+    ail_uuid = sanityze_uuid(ail_uuid)
+    if not exists_ail_instance(ail_uuid):
+        return {"status": "error", "reason": "AIL server not found"}, 404
+
+    clients_id = get_client_id_by_ail_uuid(ail_uuid)
+    if not clients_id:
+        return {"status": "error", "reason": "Client not connected"}, 400
+    for client_id in clients_id:
+        res = send_command_to_manager('relaunch', client_id=client_id, ail_uuid=ail_uuid)
+    return res, 200
+
 def api_create_ail_instance(json_dict):
     ail_uuid = json_dict.get('uuid').replace(' ', '')
     if not is_valid_uuid_v4(ail_uuid):

bin/core/ail_2_ail_client.py

@@ -70,7 +70,7 @@ async def push(websocket, ail_uuid):
         else:
             await asyncio.sleep(10)
 
-async def ail_to_ail_client(ail_uuid, sync_mode, api, ail_key=None):
+async def ail_to_ail_client(ail_uuid, sync_mode, api, ail_key=None, client_id=None):
     if not ail_2_ail.exists_ail_instance(ail_uuid):
         print('AIL server not found')
         return
@@ -88,7 +88,8 @@ async def ail_to_ail_client(ail_uuid, sync_mode, api, ail_key=None):
         uri = f"{ail_url}/{sync_mode}/{local_ail_uuid}"
     #print(uri)
 
-    ail_2_ail.clear_save_ail_server_error(ail_uuid)
+    if client_id is None:
+        client_id = ail_2_ail.create_sync_client_cache(ail_uuid, sync_mode)
 
     try:
         async with websockets.client.connect(
@@ -97,6 +98,8 @@ async def ail_to_ail_client(ail_uuid, sync_mode, api, ail_key=None):
             #open_timeout=10, websockers 10.0 /!\ python>=3.7
             extra_headers={"Authorization": f"{ail_key}"}
         ) as websocket:
+            # success
+            ail_2_ail.clear_save_ail_server_error(ail_uuid)
 
             if sync_mode == 'pull':
                 await pull(websocket, ail_uuid)
@@ -155,6 +158,8 @@ async def ail_to_ail_client(ail_uuid, sync_mode, api, ail_key=None):
         redis_logger.critical(f'{ail_uuid}: {error_message}')
         ail_2_ail.save_ail_server_error(ail_uuid, error_message)
 
+    ail_2_ail.delete_sync_client_cache(client_id)
+
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(description='Websocket SYNC Client')
@@ -168,6 +173,7 @@ if __name__ == '__main__':
     ail_uuid = args.ail_uuid
     sync_mode = args.sync_mode
     api = args.api
+    client_id = args.client_id
 
     if ail_uuid is None or sync_mode not in ['api', 'pull', 'push']:
         parser.print_help()
@@ -184,4 +190,4 @@ if __name__ == '__main__':
     ssl_context.verify_mode = ssl.CERT_NONE
     # SELF SIGNED CERTIFICATES
 
-    asyncio.get_event_loop().run_until_complete(ail_to_ail_client(ail_uuid, sync_mode, api))
+    asyncio.get_event_loop().run_until_complete(ail_to_ail_client(ail_uuid, sync_mode, api, client_id=client_id))

bin/core/ail_2_ail_server.py

@@ -73,7 +73,7 @@ async def server_controller():
                 ail_uuid = command_dict.get('ail_uuid')
                 connected_clients = CONNECTED_CLIENTS[ail_uuid].copy()
                 for c_websocket in connected_clients:
-                    await c_websocket.close()
+                    await c_websocket.close(code=1000)
                     redis_logger.info(f'Server Command Connection closed: {ail_uuid}')
                     print(f'Server Command Connection closed: {ail_uuid}')
 

var/www/blueprints/ail_2_ail_sync.py

@@ -55,7 +55,7 @@ def create_json_response(data, status_code):
 def ail_2_ail_dashboard():
     ail_uuid = ail_2_ail.get_ail_uuid()
     l_servers = ail_2_ail.get_all_running_sync_servers()
-    l_servers = ail_2_ail.get_ail_instances_metadata(l_servers)
+    l_servers = ail_2_ail.get_ail_instances_metadata(l_servers, server_sync_mode= True)
     l_clients = ail_2_ail.get_server_all_connected_clients()
     l_clients = ail_2_ail.get_ail_instances_metadata(l_clients, sync_queues=False, client_sync_mode=True)
     return render_template("ail_2_ail_dashboard.html", ail_uuid=ail_uuid,
@@ -79,7 +79,7 @@ def ail_servers():
 @login_admin
 def ail_server_view():
     ail_uuid = request.args.get('uuid')
-    server_metadata = ail_2_ail.get_ail_instance_metadata(ail_uuid,sync_queues=True)
+    server_metadata = ail_2_ail.get_ail_instance_metadata(ail_uuid, client_sync_mode=True, server_sync_mode=True, sync_queues=True)
     server_metadata['sync_queues'] = ail_2_ail.get_queues_metadata(server_metadata['sync_queues'])
 
     return render_template("view_ail_server.html", server_metadata=server_metadata,
@@ -107,6 +107,50 @@ def ail_server_api_version():
         return create_json_response(res[0], res[1])
     return redirect(url_for('ail_2_ail_sync.ail_server_view', uuid=ail_uuid))
 
+@ail_2_ail_sync.route('/settings/ail_2_ail/server/client/kill', methods=['GET'])
+@login_required
+@login_admin
+def ail_server_client_kill():
+    ail_uuid = request.args.get('uuid')
+    input_dict = {"uuid": ail_uuid}
+    res = ail_2_ail.api_kill_server_connected_clients(input_dict)
+    if res[1] != 200:
+        return create_json_response(res[0], res[1])
+    return redirect(url_for('ail_2_ail_sync.ail_2_ail_dashboard'))
+
+@ail_2_ail_sync.route('/settings/ail_2_ail/server/sync_client/kill', methods=['GET'])
+@login_required
+@login_admin
+def ail_server_sync_client_kill():
+    ail_uuid = request.args.get('uuid')
+    input_dict = {"uuid": ail_uuid}
+    res = ail_2_ail.api_kill_sync_client(input_dict)
+    if res[1] != 200:
+        return create_json_response(res[0], res[1])
+    return redirect(url_for('ail_2_ail_sync.ail_2_ail_dashboard'))
+
+@ail_2_ail_sync.route('/settings/ail_2_ail/server/sync_client/relaunch', methods=['GET'])
+@login_required
+@login_admin
+def ail_server_sync_client_relaunch():
+    ail_uuid = request.args.get('uuid')
+    input_dict = {"uuid": ail_uuid}
+    res = ail_2_ail.api_relaunch_sync_client(input_dict)
+    if res[1] != 200:
+        return create_json_response(res[0], res[1])
+    return redirect(url_for('ail_2_ail_sync.ail_2_ail_dashboard'))
+
+@ail_2_ail_sync.route('/settings/ail_2_ail/server/sync_client/launch', methods=['GET'])
+@login_required
+@login_admin
+def ail_server_sync_client_launch():
+    ail_uuid = request.args.get('uuid')
+    input_dict = {"uuid": ail_uuid}
+    res = ail_2_ail.api_launch_sync_client(input_dict)
+    if res[1] != 200:
+        return create_json_response(res[0], res[1])
+    return redirect(url_for('ail_2_ail_sync.ail_2_ail_dashboard'))
+
 @ail_2_ail_sync.route('/settings/ail_2_ail/server/add', methods=['GET', 'POST'])
 @login_required
 @login_admin

var/www/templates/ail_2_ail/ail_2_ail_dashboard.html

@@ -51,7 +51,7 @@
 				</div>
 
 
-				<h1>Connected Servers:</h3>
+				<h1>Connected - AIL Sync:</h3>
 
 				<table id="table_servers" class="table table-striped border-primary">
           <thead class="bg-dark text-white">
@@ -60,6 +60,8 @@
                 <th>url</th>
                 <th>description</th>
                 <th>sync queues</th>
+								<th>sync mode</th>
+								<th></th>
             </tr>
           </thead>
           <tbody style="font-size: 15px;">
@@ -77,15 +79,40 @@
                   	<a href="{{ url_for('ail_2_ail_sync.sync_queue_view') }}?uuid={{queue_uuid}}">
                       {{queue_uuid}}</br>
                     </a>
-                  {%  endfor %}
+                  {% endfor %}
                 </td>
+								<td>
+									{% if dict_server['server_sync_mode']['api'] %}
+                  	API
+                  {% endif %}
+									{% if dict_server['server_sync_mode']['pull'] %}
+                  	PULL
+                  {% endif %}
+									{% if dict_server['server_sync_mode']['push'] %}
+                  	PUSH
+                  {% endif %}
+								</td>
+								<td>
+									{% if dict_server['server_sync_mode']['push'] %}
+										<a href="{{ url_for('ail_2_ail_sync.ail_server_sync_client_kill') }}?uuid={{ dict_server['uuid'] }}">
+											<button type="button" class="btn btn-danger px-1 py-0">
+												<i class="fas fa-times"></i> kill
+											</button>
+											<a href="{{ url_for('ail_2_ail_sync.ail_server_sync_client_relaunch') }}?uuid={{ dict_server['uuid'] }}">
+												<button type="button" class="btn btn-info px-1 py-0">
+													<i class="fas fa-redo-alt"></i> reLaunch
+												</button>
+											</a>
+										</a>
+                  {% endif %}
+								</td>
               </tr>
             {% endfor %}
           </tbody>
         </table>
 
 
-				<h1>Connected Clients:</h3>
+				<h1>Connected - Remote Sync:</h3>
 
 				<table id="table_servers" class="table table-striped border-primary">
           <thead class="bg-dark text-white">
@@ -94,6 +121,7 @@
                 <th>url</th>
                 <th>description</th>
                 <th>sync mode</th>
+								<th></th>
             </tr>
           </thead>
           <tbody style="font-size: 15px;">
@@ -109,14 +137,21 @@
                 <td class="text-center">
 									{% if dict_server['client_sync_mode']['api'] %}
                   	API
-                  {%  endif %}
+                  {% endif %}
 									{% if dict_server['client_sync_mode']['pull'] %}
                   	PULL
-                  {%  endif %}
+                  {% endif %}
 									{% if dict_server['client_sync_mode']['push'] %}
                   	PUSH
-                  {%  endif %}
+                  {% endif %}
                 </td>
+								 <td>
+									 <a href="{{ url_for('ail_2_ail_sync.ail_server_client_kill') }}?uuid={{ dict_server['uuid'] }}">
+										 <button type="button" class="btn btn-danger px-1 py-0">
+											 <i class="fas fa-times"></i> kill
+										 </button>
+									 </a>
+								 </td>
               </tr>
             {% endfor %}
           </tbody>

var/www/templates/ail_2_ail/view_ail_server.html

@@ -125,6 +125,61 @@
 										</a>
 									</td>
 								</tr>
+								<tr>
+									<td class="text-right"><b>AIL Sync</b></td>
+									<td>
+										{% if server_metadata['server_sync_mode']['api'] %}
+	                  	API
+	                  {% endif %}
+										{% if server_metadata['server_sync_mode']['pull'] %}
+	                  	PULL
+	                  {% endif %}
+										{% if server_metadata['server_sync_mode']['push'] %}
+	                  	PUSH
+
+											<a href="{{ url_for('ail_2_ail_sync.ail_server_sync_client_kill') }}?uuid={{ server_metadata['uuid'] }}">
+												<button type="button" class="btn btn-danger px-1 py-0">
+													<i class="fas fa-times"></i> Kill
+												</button>
+											</a>
+											<a href="{{ url_for('ail_2_ail_sync.ail_server_sync_client_relaunch') }}?uuid={{ server_metadata['uuid'] }}">
+												<button type="button" class="btn btn-info px-1 py-0">
+													<i class="fas fa-redo-alt"></i> reLaunch
+												</button>
+											</a>
+										{% else %}
+											{% if (server_metadata['pull'] or server_metadata['push']) and server_metadata['sync_queues']%}
+												<a href="{{ url_for('ail_2_ail_sync.ail_server_sync_client_launch') }}?uuid={{ server_metadata['uuid'] }}">
+													<button type="button" class="btn btn-primary px-1 py-0">
+														<i class="fas fa-play"></i> Launch
+													</button>
+												</a>
+											{% endif %}
+	                  {% endif %}
+									</td>
+								</tr>
+								<tr>
+									<td class="text-right"><b>Remote Sync</b></td>
+									<td>
+										{% if server_metadata['client_sync_mode']['api'] %}
+	                  	API
+	                  {% endif %}
+										{% if server_metadata['client_sync_mode']['pull'] %}
+	                  	PULL
+	                  {% endif %}
+										{% if server_metadata['client_sync_mode']['push'] %}
+	                  	PUSH
+	                  {% endif %}
+
+										{% if server_metadata['client_sync_mode']['api'] or server_metadata['client_sync_mode']['pull'] or server_metadata['client_sync_mode']['push'] %}
+											<a href="{{ url_for('ail_2_ail_sync.ail_server_client_kill') }}?uuid={{ server_metadata['uuid'] }}">
+		 										 <button type="button" class="btn btn-danger px-1 py-0">
+		 											 <i class="fas fa-times"></i> kill
+		 										 </button>
+		 									 </a>
+									 	{% endif %}
+									</td>
+								</tr>
 							</tbody>
 						</table>