diff --git a/HOWTO.md b/HOWTO.md index 3a944239..08f23f6a 100644 --- a/HOWTO.md +++ b/HOWTO.md @@ -78,21 +78,6 @@ If you want to add a new processing or analysis module in AIL, follow these simp 2. Use [./bin/template.py](./bin/template.py) as a sample module and create a new file in bin/ with the module name used in the modules.cfg configuration. -How to create a new webpage ---------------------------- - -If you want to add a new webpage for a module in AIL, follow these simple steps: - -1. Launch [./var/www/create_new_web_module.py](./var/www/create_new_web_module.py) and enter the name to use for your webpage (Usually, your newly created python module). - -2. A template and flask skeleton has been created for your new webpage in [./var/www/modules/](./var/www/modules/) - -3. Edit the created html files under the template folder as well as the Flask_* python script so that they fit your needs. - -4. You can change the order of your module in the top navigation header in the file [./var/www/templates/header_base.html](./var/www/templates/header_base.html) - -5. You can ignore module, and so, not display them in the top navigation header by adding the module name in the file [./var/www/templates/ignored_modules.txt](./var/www/templates/ignored_modules.txt) - How to contribute a module -------------------------- @@ -104,30 +89,6 @@ To contribute your module, feel free to pull your contribution. Additional information ====================== -Manage modules: ModulesInformationV2.py ---------------------------------------- - -You can do a lots of things easily with the [./bin/ModulesInformationV2](./bin/ModulesInformationV2) script: - -- Monitor the health of other modules -- Monitor the ressources comsumption of other modules -- Start one or more modules -- Kill running modules -- Restart automatically stuck modules -- Show the paste currently processed by a module - -### Navigation - -You can navigate into the interface by using arrow keys. In order to perform an action on a selected module, you can either press or to show the dialog box. - -To change list, you can press the key. - -Also, you can quickly stop or start modules by clicking on the ```` or ```` symbol respectively. These are located in the _Action_ column. - -Finally, you can quit this program by pressing either ```` or ````. - - - Crawler --------------------- @@ -145,6 +106,7 @@ In AIL, you can crawl websites and Tor hidden services. Don't forget to review t 1. Lacus URL: In the webinterface, go to ``Crawlers>Settings`` and click on the Edit button + ![Splash Manager Config](./doc/screenshots/lacus_config.png?raw=true "AIL Lacus Config") ![Splash Manager Config](./doc/screenshots/lacus_config_edit.png?raw=true "AIL Lacus Config") @@ -156,9 +118,26 @@ Choose the number of crawlers you want to launch ![Splash Manager Nb Crawlers Config](./doc/screenshots/crawler_nb_captures_edit.png?raw=true "AIL Lacus Nb Crawlers Config") +Kvrocks Migration +--------------------- +**Important Note: +We are currently working on a [migration script](https://github.com/ail-project/ail-framework/blob/master/bin/DB_KVROCKS_MIGRATION.py) to facilitate the migration to Kvrocks. +Once this script is ready, AIL version 5.0 will be released.** -#### Old updates +Please note that the current version of this migration script only supports migrating the database on the same server. +(If you plan to migrate to another server, we will provide additional instructions in this section once the migration script is completed) -##### Python 3 Upgrade - -To upgrade from an existing AIL installation, you have to launch [python3_upgrade.sh](./python3_upgrade.sh), this script will delete and create a new virtual environment. The script **will upgrade the packages but won't keep your previous data** (neverthless the data is copied into a directory called `old`). If you install from scratch, you don't require to launch the [python3_upgrade.sh](./python3_upgrade.sh). +To migrate your database to Kvrocks: +1. Launch ARDB and Kvrocks +2. Pull from remote + ``` + git checkout master + git pull + ``` +3. Launch the migration script: + ``` + git checkout master + git pull + cd bin/ + ./DB_KVROCKS_MIGRATION.py + ``` diff --git a/bin/Update.py b/bin/Update.py index 9c52c691..8e8393a8 100755 --- a/bin/Update.py +++ b/bin/Update.py @@ -37,7 +37,8 @@ def auto_update_enabled(cfg): # check if files are modify locally def check_if_files_modified(): - process = subprocess.run(['git', 'ls-files', '-m'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # return True + process = subprocess.run(['git', 'ls-files' ,'-m'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if process.returncode == 0: modified_files = process.stdout if modified_files: @@ -61,6 +62,7 @@ def check_if_files_modified(): sys.exit(1) def repo_is_fork(): + # return False print('Check if this repository is a fork:') process = subprocess.run(['git', 'remote', '-v'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -234,9 +236,9 @@ def get_git_upper_tags_remote(current_tag, is_fork): except ValueError: continue - if float(current_tag) < 4.2: + if float(current_tag) < 5.0: # add tag with last commit - if float(current_tag_val) <= float(tag_val) < float(4.2): + if float(current_tag_val) <= float(tag_val) < float(5.0): dict_tags_commit[tag] = commit else: # add tag with last commit @@ -270,13 +272,28 @@ def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_for update_submodules() - print('{}git pull:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT)) - process = subprocess.run(['git', 'pull'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + temp_current_tag = current_tag.replace('v', '') + if temp_current_tag.count('.') > 1: + temp_current_tag = temp_current_tag.rsplit('.', 1) + temp_current_tag = ''.join(temp_current_tag) - if process.returncode == 0: - output = process.stdout.decode() - print(output) + if float(temp_current_tag) < 5.0: + roll_back_update('2c65194b94dab95df9b8da19c88d65239f398355') + pulled = True + else: + print('{}git pull:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT)) + process = subprocess.run(['git', 'pull'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if process.returncode == 0: + output = process.stdout.decode() + print(output) + pulled = True + else: + print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) + aborting_update() + pulled = False + sys.exit(1) + if pulled: # CHECK IF UPDATER Update if float(os.stat(UPDATER_FILENAME).st_mtime) > UPDATER_LAST_MODIFICATION: # request updater relaunch @@ -321,10 +338,7 @@ def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_for print(f'{TERMINAL_YELLOW}**************** AIL Successfully Updated *****************{TERMINAL_DEFAULT}') print() sys.exit(0) - else: - print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) - aborting_update() - sys.exit(1) + else: print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT)) aborting_update() diff --git a/var/www/modules/dashboard/templates/index.html b/var/www/modules/dashboard/templates/index.html index 3e86f4fa..db5b4fb4 100644 --- a/var/www/modules/dashboard/templates/index.html +++ b/var/www/modules/dashboard/templates/index.html @@ -74,9 +74,7 @@ {%endif%} - +{# {% include 'dashboard/update_modal.html' %}#}
diff --git a/var/www/templates/dashboard/update_modal.html b/var/www/templates/dashboard/update_modal.html index 54bee18e..a9bef03e 100644 --- a/var/www/templates/dashboard/update_modal.html +++ b/var/www/templates/dashboard/update_modal.html @@ -1,40 +1,49 @@