mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-26 15:57:16 +00:00
chg: [config] save config on DB + default values
This commit is contained in:
parent
2c0ee71845
commit
488604ce4e
3 changed files with 212 additions and 1 deletions
155
bin/lib/Config_DB.py
Executable file
155
bin/lib/Config_DB.py
Executable file
|
@ -0,0 +1,155 @@
|
||||||
|
#!/usr/bin/python3
|
||||||
|
|
||||||
|
"""
|
||||||
|
Config save in DB
|
||||||
|
===================
|
||||||
|
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import redis
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||||
|
import ConfigLoader
|
||||||
|
|
||||||
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||||
|
config_loader = None
|
||||||
|
|
||||||
|
#### TO PUT IN CONFIG
|
||||||
|
# later => module timeout
|
||||||
|
#
|
||||||
|
## data retention
|
||||||
|
#########################
|
||||||
|
|
||||||
|
default_config = {
|
||||||
|
"crawler": {
|
||||||
|
"enable_har_by_default": False,
|
||||||
|
"enable_screenshot_by_default": True,
|
||||||
|
"default_depth_limit": 1,
|
||||||
|
"default_closespider_pagecount": 50,
|
||||||
|
"default_user_agent": "Mozilla/5.0 (Windows NT 10.0; rv:68.0) Gecko/20100101 Firefox/68.0",
|
||||||
|
"default_timeout": 30
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def get_default_config():
|
||||||
|
return default_config
|
||||||
|
|
||||||
|
def get_default_config_value(section, field):
|
||||||
|
return default_config[section][field]
|
||||||
|
|
||||||
|
config_type = {
|
||||||
|
# crawler config
|
||||||
|
"crawler": {
|
||||||
|
"enable_har_by_default": bool,
|
||||||
|
"enable_screenshot_by_default": bool,
|
||||||
|
"default_depth_limit": int,
|
||||||
|
"default_closespider_pagecount": int,
|
||||||
|
"default_user_agent": str,
|
||||||
|
"default_timeout": int
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def get_config_type(section, field):
|
||||||
|
return config_type[section][field]
|
||||||
|
|
||||||
|
# # TODO: add set, dict, list and select_(multiple_)value
|
||||||
|
def is_valid_type(obj, section, field, value_type=None):
|
||||||
|
res = isinstance(obj, get_config_type(section, field))
|
||||||
|
return res
|
||||||
|
|
||||||
|
def reset_default_config():
|
||||||
|
pass
|
||||||
|
|
||||||
|
def set_default_config(section, field):
|
||||||
|
save_config(section, field, get_default_config_value(section, field))
|
||||||
|
|
||||||
|
def get_all_config_sections():
|
||||||
|
return list(get_default_config())
|
||||||
|
|
||||||
|
def get_all_config_fields_by_section(section):
|
||||||
|
return list(get_default_config()[section])
|
||||||
|
|
||||||
|
def get_config(section, field):
|
||||||
|
# config field don't exist
|
||||||
|
if not r_serv_db.hexists(f'config:global:{section}', field):
|
||||||
|
set_default_config(section, field)
|
||||||
|
return get_default_config_value(section, field)
|
||||||
|
|
||||||
|
# load default config section
|
||||||
|
if not r_serv_db.exists('config:global:{}'.format(section)):
|
||||||
|
save_config(section, field, get_default_config_value(section, field))
|
||||||
|
return get_default_config_value(section, field)
|
||||||
|
|
||||||
|
return r_serv_db.hget(f'config:global:{section}', field)
|
||||||
|
|
||||||
|
def get_config_dict_by_section(section):
|
||||||
|
config_dict = {}
|
||||||
|
for field in get_all_config_fields_by_section(section):
|
||||||
|
config_dict[field] = get_config(section, field)
|
||||||
|
return config_dict
|
||||||
|
|
||||||
|
def save_config(section, field, value, value_type=None): ###########################################
|
||||||
|
if section in default_config:
|
||||||
|
if is_valid_type(value, section, field, value_type=value_type):
|
||||||
|
if value_type in ['list', 'set', 'dict']:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
r_serv_db.hset(f'config:global:{section}', field, value)
|
||||||
|
# used by check_integrity
|
||||||
|
r_serv_db.sadd('config:all_global_section', field, value)
|
||||||
|
|
||||||
|
# check config value + type
|
||||||
|
def check_integrity():
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
config_documentation = {
|
||||||
|
"crawler": {
|
||||||
|
"enable_har_by_default": 'Enable HAR by default',
|
||||||
|
"enable_screenshot_by_default": 'Enable screenshot by default',
|
||||||
|
"default_depth_limit": 'Maximum number of url depth',
|
||||||
|
"default_closespider_pagecount": 'Maximum number of pages',
|
||||||
|
"default_user_agent": "User agent used by default",
|
||||||
|
"default_timeout": "Crawler connection timeout"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def get_config_documentation(section, field):
|
||||||
|
return config_documentation[section][field]
|
||||||
|
|
||||||
|
# def conf_view():
|
||||||
|
# class F(MyBaseForm):
|
||||||
|
# pass
|
||||||
|
#
|
||||||
|
# F.username = TextField('username')
|
||||||
|
# for name in iterate_some_model_dynamically():
|
||||||
|
# setattr(F, name, TextField(name.title()))
|
||||||
|
#
|
||||||
|
# form = F(request.POST, ...)
|
||||||
|
|
||||||
|
def get_field_full_config(section, field):
|
||||||
|
dict_config = {}
|
||||||
|
dict_config['value'] = get_config(section, field)
|
||||||
|
dict_config['type'] = get_config_type(section, field)
|
||||||
|
dict_config['info'] = get_config_documentation(section, field)
|
||||||
|
return dict_config
|
||||||
|
|
||||||
|
def get_full_config_by_section(section):
|
||||||
|
dict_config = {}
|
||||||
|
for field in get_all_config_fields_by_section(section):
|
||||||
|
dict_config[field] = get_field_full_config(section, field)
|
||||||
|
return dict_config
|
||||||
|
|
||||||
|
def get_full_config():
|
||||||
|
dict_config = {}
|
||||||
|
for section in get_all_config_sections():
|
||||||
|
dict_config[section] = get_full_config_by_section(section)
|
||||||
|
return dict_config
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
res = get_full_config()
|
||||||
|
print(res)
|
|
@ -26,6 +26,7 @@ import Tag
|
||||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||||
import Domain
|
import Domain
|
||||||
import crawlers
|
import crawlers
|
||||||
|
import Config_DB
|
||||||
|
|
||||||
r_cache = Flask_config.r_cache
|
r_cache = Flask_config.r_cache
|
||||||
r_serv_db = Flask_config.r_serv_db
|
r_serv_db = Flask_config.r_serv_db
|
||||||
|
@ -438,10 +439,12 @@ def crawler_splash_setings():
|
||||||
splash_manager_url = crawlers.get_splash_manager_url()
|
splash_manager_url = crawlers.get_splash_manager_url()
|
||||||
api_key = crawlers.get_hidden_splash_api_key()
|
api_key = crawlers.get_hidden_splash_api_key()
|
||||||
is_manager_connected = crawlers.ping_splash_manager()
|
is_manager_connected = crawlers.ping_splash_manager()
|
||||||
|
crawler_full_config = Config_DB.get_full_config_by_section('crawler')
|
||||||
|
|
||||||
return render_template("settings_splash_crawler.html",
|
return render_template("settings_splash_crawler.html",
|
||||||
is_manager_connected=is_manager_connected,
|
is_manager_connected=is_manager_connected,
|
||||||
splash_manager_url=splash_manager_url, api_key=api_key,
|
splash_manager_url=splash_manager_url, api_key=api_key,
|
||||||
all_splash=all_splash, all_proxies=all_proxies)
|
all_splash=all_splash, all_proxies=all_proxies,
|
||||||
|
crawler_full_config=crawler_full_config)
|
||||||
|
|
||||||
## - - ##
|
## - - ##
|
||||||
|
|
|
@ -102,6 +102,7 @@
|
||||||
<th>
|
<th>
|
||||||
Description
|
Description
|
||||||
</th>
|
</th>
|
||||||
|
<th></th>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
{% for splash_name in all_splash %}
|
{% for splash_name in all_splash %}
|
||||||
|
@ -123,6 +124,13 @@
|
||||||
<td>
|
<td>
|
||||||
{{all_splash[splash_name]['description']}}
|
{{all_splash[splash_name]['description']}}
|
||||||
</td>
|
</td>
|
||||||
|
<td>
|
||||||
|
<div class="d-flex justify-content-end">
|
||||||
|
<button class="btn btn-outline-dark px-1 py-0">
|
||||||
|
<i class="fas fa-pencil-alt"></i>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</tbody>
|
</tbody>
|
||||||
|
@ -153,6 +161,7 @@
|
||||||
<th>
|
<th>
|
||||||
Description
|
Description
|
||||||
</th>
|
</th>
|
||||||
|
<th></th>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
{% for proxy_name in all_proxies %}
|
{% for proxy_name in all_proxies %}
|
||||||
|
@ -180,6 +189,13 @@
|
||||||
<td>
|
<td>
|
||||||
{{all_proxies[proxy_name]['description']}}
|
{{all_proxies[proxy_name]['description']}}
|
||||||
</td>
|
</td>
|
||||||
|
<td>
|
||||||
|
<div class="d-flex justify-content-end">
|
||||||
|
<button class="btn btn-outline-dark px-1 py-0">
|
||||||
|
<i class="fas fa-pencil-alt"></i>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</tbody>
|
</tbody>
|
||||||
|
@ -196,6 +212,43 @@
|
||||||
</div>
|
</div>
|
||||||
<div class="card-body">
|
<div class="card-body">
|
||||||
|
|
||||||
|
<table class="table table-striped table-hover">
|
||||||
|
<thead class="bg-info text-white">
|
||||||
|
<th>
|
||||||
|
Key
|
||||||
|
</th>
|
||||||
|
<th>
|
||||||
|
Description
|
||||||
|
</th>
|
||||||
|
<th>
|
||||||
|
Value
|
||||||
|
</th>
|
||||||
|
<th></th>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{% for config_field in crawler_full_config %}
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
{{config_field}}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{{crawler_full_config[config_field]['info']}}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{{crawler_full_config[config_field]['value']}}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<div class="d-flex justify-content-end">
|
||||||
|
<button class="btn btn-outline-dark px-1 py-0">
|
||||||
|
<i class="fas fa-pencil-alt"></i>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue