From 4660a8e88bfdc4cb540e3e5dcd3e28ee80120cde Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 24 Jun 2022 16:45:35 +0200 Subject: [PATCH] chg: [kvrocks] launch + install --- bin/LAUNCH.sh | 224 ++++++++++------ configs/6383.conf | 650 +++++++++++++++++++++++++++++++++++++++++++++ installing_deps.sh | 25 +- 3 files changed, 806 insertions(+), 93 deletions(-) create mode 100644 configs/6383.conf diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 9fbec934..9e6db45c 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -33,6 +33,7 @@ export PATH=$AIL_FLASK:$PATH isredis=`screen -ls | egrep '[0-9]+.Redis_AIL' | cut -d. -f1` isardb=`screen -ls | egrep '[0-9]+.ARDB_AIL' | cut -d. -f1` +iskvrocks=`screen -ls | egrep '[0-9]+.KVROCKS_AIL' | cut -d. -f1` islogged=`screen -ls | egrep '[0-9]+.Logging_AIL' | cut -d. -f1` isqueued=`screen -ls | egrep '[0-9]+.Queue_AIL' | cut -d. -f1` is_ail_core=`screen -ls | egrep '[0-9]+.Core_AIL' | cut -d. -f1` @@ -61,6 +62,7 @@ function helptext { - All the processing modules. - All Redis in memory servers. - All ARDB on disk servers. + - All KVROCKS servers. "$DEFAULT" (Inside screen Daemons) "$DEFAULT" @@ -104,6 +106,17 @@ function launching_ardb { screen -S "ARDB_AIL" -X screen -t "6382" bash -c 'cd '${AIL_HOME}'; ardb-server '$conf_dir'6382.conf ; read x' } +function launching_kvrocks { + conf_dir="${AIL_HOME}/configs" + + screen -dmS "KVROCKS_AIL" + sleep 0.1 + echo -e $GREEN"\t* Launching KVROCKS servers"$DEFAULT + + sleep 0.1 + screen -S "KVROCKS_AIL" -X screen -t "6383" bash -c 'cd '${AIL_HOME}'; ./kvrocks/build/kvrocks -c '$conf_dir'/6383.conf ; read x' +} + function launching_logs { screen -dmS "Logging_AIL" sleep 0.1 @@ -277,56 +290,65 @@ function launching_scripts { } +function shutting_down_redis_servers { + array=("$@") + redis_dir=${AIL_HOME}/redis/src + for port in "${array[@]}"; + do + bash -c "${redis_dir}/redis-cli -p ${port} SHUTDOWN" + sleep 0.1 + done +} + function shutting_down_redis { - redis_dir=${AIL_HOME}/redis/src/ - bash -c $redis_dir'redis-cli -p 6379 SHUTDOWN' - sleep 0.1 - bash -c $redis_dir'redis-cli -p 6380 SHUTDOWN' - sleep 0.1 - bash -c $redis_dir'redis-cli -p 6381 SHUTDOWN' + ports=("6379" "6380" "6381") + shutting_down_redis_servers "${ports[@]}" } function shutting_down_ardb { - redis_dir=${AIL_HOME}/redis/src/ - bash -c $redis_dir'redis-cli -p 6382 SHUTDOWN' + ports=("6382") + shutting_down_redis_servers "${ports[@]}" +} + +function shutting_down_kvrocks { + ports=("6383") + shutting_down_redis_servers "${ports[@]}" +} + +function checking_redis_servers { + db_name=$1 + shift + array=("$@") + redis_dir="${AIL_HOME}/redis/src" + flag_db=0 + for port in "${array[@]}"; + do + sleep 0.2 + bash -c "${redis_dir}/redis-cli -p ${port} PING | grep "PONG" &> /dev/null" + if [ ! $? == 0 ]; then + echo -e "${RED}\t${port} ${db_name} not ready${DEFAULT}" + flag_db=1 + fi + done + return $flag_db; } function checking_redis { - flag_redis=0 - redis_dir=${AIL_HOME}/redis/src/ - bash -c $redis_dir'redis-cli -p 6379 PING | grep "PONG" &> /dev/null' - if [ ! $? == 0 ]; then - echo -e $RED"\t6379 not ready"$DEFAULT - flag_redis=1 - fi - sleep 0.1 - bash -c $redis_dir'redis-cli -p 6380 PING | grep "PONG" &> /dev/null' - if [ ! $? == 0 ]; then - echo -e $RED"\t6380 not ready"$DEFAULT - flag_redis=1 - fi - sleep 0.1 - bash -c $redis_dir'redis-cli -p 6381 PING | grep "PONG" &> /dev/null' - if [ ! $? == 0 ]; then - echo -e $RED"\t6381 not ready"$DEFAULT - flag_redis=1 - fi - sleep 0.1 - - return $flag_redis; + ports=("6379" "6380" "6381") + checking_redis_servers "Redis" "${ports[@]}" + return $? } function checking_ardb { - flag_ardb=0 - redis_dir=${AIL_HOME}/redis/src/ - sleep 0.2 - bash -c $redis_dir'redis-cli -p 6382 PING | grep "PONG" &> /dev/null' - if [ ! $? == 0 ]; then - echo -e $RED"\t6382 ARDB not ready"$DEFAULT - flag_ardb=1 - fi + ports=("6382") + checking_redis_servers "ARDB" "${ports[@]}" + return $? +} - return $flag_ardb; +function checking_kvrocks { + ports=("6383") + checking_redis_servers "KVROCKS" "${ports[@]}" + return $? } function wait_until_redis_is_ready { @@ -353,6 +375,18 @@ function wait_until_ardb_is_ready { echo -e $YELLOW"\t* ARDB Launched"$DEFAULT } +function wait_until_kvrocks_is_ready { + not_ready=true; + while $not_ready; do + if checking_kvrocks; then + not_ready=false + else + sleep 3 + fi + done + echo -e $YELLOW"\t* KVROCKS Launched"$DEFAULT +} + function launch_redis { if [[ ! $isredis ]]; then launching_redis; @@ -369,6 +403,14 @@ function launch_ardb { fi } +function launch_kvrocks { + if [[ ! $iskvrocks ]]; then + launching_kvrocks; + else + echo -e $RED"\t* A screen is already launched"$DEFAULT + fi +} + function launch_logs { if [[ ! $islogged ]]; then launching_logs; @@ -388,14 +430,14 @@ function launch_queues { function launch_scripts { if [[ ! $isscripted ]]; then ############################# is core sleep 1 - if checking_ardb && checking_redis; then + if checking_ardb && checking_redis && checking_kvrocks; then launching_scripts; else no_script_launched=true while $no_script_launched; do echo -e $YELLOW"\tScript not started, waiting 5 more secondes"$DEFAULT sleep 5 - if checking_redis && checking_ardb; then + if checking_redis && checking_ardb && checking_kvrocks; then launching_scripts; no_script_launched=false else @@ -446,7 +488,7 @@ function killscript { } function killall { - if [[ $isredis || $isardb || $islogged || $isqueued || $is_ail_2_ail || $isscripted || $isflasked || $isfeeded || $iscrawler || $is_ail_core ]]; then + if [[ $isredis || $isardb || $iskvrocks || $islogged || $isqueued || $is_ail_2_ail || $isscripted || $isflasked || $isfeeded || $iscrawler || $is_ail_core || $is_ail_2_ail ]]; then if [[ $isredis ]]; then echo -e $GREEN"Gracefully closing redis servers"$DEFAULT shutting_down_redis; @@ -456,20 +498,20 @@ function killall { echo -e $GREEN"Gracefully closing ardb servers"$DEFAULT shutting_down_ardb; fi + if [[ $iskvrocks ]]; then + echo -e $GREEN"Gracefully closing Kvrocks servers"$DEFAULT + shutting_down_kvrocks; + fi echo -e $GREEN"Killing all"$DEFAULT - kill $isredis $isardb $islogged $isqueued $is_ail_core $isscripted $isflasked $isfeeded $iscrawler $is_ail_2_ail + kill $isredis $isardb $iskvrocks $islogged $isqueued $is_ail_core $isscripted $isflasked $isfeeded $iscrawler $is_ail_2_ail sleep 0.2 echo -e $ROSE`screen -ls`$DEFAULT - echo -e $GREEN"\t* $isredis $isardb $islogged $isqueued $isscripted $is_ail_2_ail $isflasked $isfeeded $iscrawler $is_ail_core killed."$DEFAULT + echo -e $GREEN"\t* $isredis $isardb $iskvrocks $islogged $isqueued $isscripted $is_ail_2_ail $isflasked $isfeeded $iscrawler $is_ail_core killed."$DEFAULT else echo -e $RED"\t* No screen to kill"$DEFAULT fi } -function shutdown { - bash -c "./Shutdown.py" -} - function update() { bin_dir=${AIL_HOME}/bin @@ -526,6 +568,7 @@ function launch_all { update; launch_redis; launch_ardb; + launch_kvrocks; launch_logs; launch_queues; launch_scripts; @@ -534,7 +577,7 @@ function launch_all { function menu_display { - options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Flask" "Killall" "Shutdown" "Update" "Update-config" "Update-thirdparty") + options=("Redis" "Ardb" "Kvrocks" "Logs" "Queues" "Scripts" "Flask" "Killall" "Update" "Update-config" "Update-thirdparty") menu() { echo "What do you want to Launch?:" @@ -565,6 +608,9 @@ function menu_display { Ardb) launch_ardb; ;; + Kvrocks) + launch_kvrocks; + ;; Logs) launch_logs; ;; @@ -580,9 +626,6 @@ function menu_display { Killall) killall; ;; - Shutdown) - shutdown; - ;; Update) checking_configuration; update; @@ -612,42 +655,47 @@ function menu_display { while [ "$1" != "" ]; do case $1 in - -l | --launchAuto ) launch_all "automatic"; - ;; - -lr | --launchRedis ) launch_redis; - ;; - -la | --launchARDB ) launch_ardb; - ;; - -lrv | --launchRedisVerify ) launch_redis; - wait_until_redis_is_ready; - ;; - -lav | --launchARDBVerify ) launch_ardb; - wait_until_ardb_is_ready; - ;; - -k | --killAll ) killall; - ;; - -ks | --killscript ) killscript; - ;; - -m | --menu ) menu_display; - ;; - -u | --update ) checking_configuration; - update "--manual"; - ;; - -t | --test ) launch_tests; - ;; - -ut | --thirdpartyUpdate ) update_thirdparty; - ;; - -rp | --resetPassword ) reset_password; - ;; - -f | --launchFeeder ) launch_feeder; - ;; - -h | --help ) helptext; - exit - ;; - -kh | --khelp ) helptext; - ;; - * ) helptext - exit 1 + -l | --launchAuto ) launch_all "automatic"; + ;; + -lr | --launchRedis ) launch_redis; + ;; + -la | --launchARDB ) launch_ardb; + ;; + -lk | --launchKVROCKS ) launch_kvrocks; + ;; + -lrv | --launchRedisVerify ) launch_redis; + wait_until_redis_is_ready; + ;; + -lav | --launchARDBVerify ) launch_ardb; + wait_until_ardb_is_ready; + ;; + -lkv | --launchKVORCKSVerify ) launch_kvrocks; + wait_until_kvrocks_is_ready; + ;; + -k | --killAll ) killall; + ;; + -ks | --killscript ) killscript; + ;; + -m | --menu ) menu_display; + ;; + -u | --update ) checking_configuration; + update "--manual"; + ;; + -t | --test ) launch_tests; + ;; + -ut | --thirdpartyUpdate ) update_thirdparty; + ;; + -rp | --resetPassword ) reset_password; + ;; + -f | --launchFeeder ) launch_feeder; + ;; + -h | --help ) helptext; + exit + ;; + -kh | --khelp ) helptext; + ;; + * ) helptext + exit 1 esac shift done diff --git a/configs/6383.conf b/configs/6383.conf new file mode 100644 index 00000000..8a3c3aac --- /dev/null +++ b/configs/6383.conf @@ -0,0 +1,650 @@ +################################ GENERAL ##################################### + +# By default kvrocks listens for connections from all the network interfaces +# available on the server. It is possible to listen to just one or multiple +# interfaces using the "bind" configuration directive, followed by one or +# more IP addresses. +# +# Examples: +# +# bind 192.168.1.100 10.0.0.1 +# bind 127.0.0.1 ::1 +bind 127.0.0.1 + +# Unix socket. +# +# Specify the path for the unix socket that will be used to listen for +# incoming connections. There is no default, so kvrocks will not listen +# on a unix socket when not specified. +# +# unixsocket /tmp/kvrocks.sock +# unixsocketperm 777 + +# Accept connections on the specified port, default is 6666. +port 6383 + +# Close the connection after a client is idle for N seconds (0 to disable) +timeout 0 + +# The number of worker's threads, increase or decrease it would effect the performance. +workers 8 + +# By default kvrocks does not run as a daemon. Use 'yes' if you need it. +# Note that kvrocks will write a pid file in /var/run/kvrocks.pid when daemonized. +daemonize no + +# Kvrocks implements cluster solution that is similar with redis cluster solution. +# You can get cluster information by CLUSTER NODES|SLOTS|INFO command, it also is +# adapted to redis-cli, redis-benchmark, redis cluster SDK and redis cluster proxy. +# But kvrocks doesn't support to communicate with each others, so you must set +# cluster topology by CLUSTER SETNODES|SETNODEID commands, more details: #219. +# +# PLEASE NOTE: +# If you enable cluster, kvrocks will encode key with its slot id calculated by +# CRC16 and modulo 16384, endoding key with its slot id makes it efficient to +# migrate keys based on slot. So if you enabled at first time, cluster mode must +# not be disabled after restarting, and vice versa. That is to say, data is not +# compatible between standalone mode with cluster mode, you must migrate data +# if you want to change mode, otherwise, kvrocks will make data corrupt. +# +# Default: no +cluster-enabled no + +# Set the max number of connected clients at the same time. By default +# this limit is set to 10000 clients, however if the server is not +# able to configure the process file limit to allow for the specified limit +# the max number of allowed clients is set to the current file limit +# +# Once the limit is reached the server will close all the new connections sending +# an error 'max number of clients reached'. +# +maxclients 10000 + +# Require clients to issue AUTH before processing any other +# commands. This might be useful in environments in which you do not trust +# others with access to the host running kvrocks. +# +# This should stay commented out for backward compatibility and because most +# people do not need auth (e.g. they run their own servers). +# +# Warning: since kvrocks is pretty fast an outside user can try up to +# 150k passwords per second against a good box. This means that you should +# use a very strong password otherwise it will be very easy to break. +# +# requirepass foobared + +# If the master is password protected (using the "masterauth" configuration +# directive below) it is possible to tell the slave to authenticate before +# starting the replication synchronization process, otherwise the master will +# refuse the slave request. +# +# masterauth foobared + +# Master-Salve replication would check db name is matched. if not, the slave should +# refuse to sync the db from master. Don't use default value, set the db-name to identify +# the cluster. +db-name change.me.db + +# The working directory +# +# The DB will be written inside this directory +# Note that you must specify a directory here, not a file name. +dir /home/aurelien/git/ail-framework/DATA_KVROCKS + +# You can configure where to store your server logs by the log-dir. +# If you don't specify one, we will use the above `dir` as our default log directory. +# We also can send logs to stdout/stderr is as simple as: +# +# log-dir stdout + +# When running daemonized, kvrocks writes a pid file in ${CONFIG_DIR}/kvrocks.pid by +# default. You can specify a custom pid file location here. +# pidfile /var/run/kvrocks.pid +pidfile "" + +# You can configure a slave instance to accept writes or not. Writing against +# a slave instance may be useful to store some ephemeral data (because data +# written on a slave will be easily deleted after resync with the master) but +# may also cause problems if clients are writing to it because of a +# misconfiguration. +slave-read-only yes + +# The slave priority is an integer number published by Kvrocks in the INFO output. +# It is used by Redis Sentinel in order to select a slave to promote into a +# master if the master is no longer working correctly. +# +# A slave with a low priority number is considered better for promotion, so +# for instance if there are three slave with priority 10, 100, 25 Sentinel will +# pick the one with priority 10, that is the lowest. +# +# However a special priority of 0 marks the replica as not able to perform the +# role of master, so a slave with priority of 0 will never be selected by +# Redis Sentinel for promotion. +# +# By default the priority is 100. +slave-priority 100 + +# TCP listen() backlog. +# +# In high requests-per-second environments you need an high backlog in order +# to avoid slow clients connections issues. Note that the Linux kernel +# will silently truncate it to the value of /proc/sys/net/core/somaxconn so +# make sure to raise both the value of somaxconn and tcp_max_syn_backlog +# in order to Get the desired effect. +tcp-backlog 511 + +# If the master is an old version, it may have specified replication threads +# that use 'port + 1' as listening port, but in new versions, we don't use +# extra port to implement replication. In order to allow the new replicas to +# copy old masters, you should indicate that the master uses replication port +# or not. +# If yes, that indicates master uses replication port and replicas will connect +# to 'master's listening port + 1' when synchronization. +# If no, that indicates master doesn't use replication port and replicas will +# connect 'master's listening port' when synchronization. +master-use-repl-port no + +# Currently, master only checks sequence number when replica asks for PSYNC, +# that is not enough since they may have different replication history even +# the replica asking sequence is in the range of the master current WAL. +# +# We design 'Replication Sequence ID' PSYNC, we add unique replication id for +# every write batch (the operation of each command on the storage engine), so +# the combination of replication id and sequence is unique for write batch. +# The master can identify whether the replica has the same replication history +# by checking replication id and sequence. +# +# By default, it is not enabled since this stricter check may easily lead to +# full synchronization. +use-rsid-psync no + +# Master-Slave replication. Use slaveof to make a kvrocks instance a copy of +# another kvrocks server. A few things to understand ASAP about kvrocks replication. +# +# 1) Kvrocks replication is asynchronous, but you can configure a master to +# stop accepting writes if it appears to be not connected with at least +# a given number of slaves. +# 2) Kvrocks slaves are able to perform a partial resynchronization with the +# master if the replication link is lost for a relatively small amount of +# time. You may want to configure the replication backlog size (see the next +# sections of this file) with a sensible value depending on your needs. +# 3) Replication is automatic and does not need user intervention. After a +# network partition slaves automatically try to reconnect to masters +# and resynchronize with them. +# +# slaveof +# slaveof 127.0.0.1 6379 + +# When a slave loses its connection with the master, or when the replication +# is still in progress, the slave can act in two different ways: +# +# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will +# still reply to client requests, possibly with out of date data, or the +# data set may just be empty if this is the first synchronization. +# +# 2) if slave-serve-stale-data is set to 'no' the slave will reply with +# an error "SYNC with master in progress" to all the kind of commands +# but to INFO and SLAVEOF. +# +slave-serve-stale-data yes + +# To guarantee slave's data safe and serve when it is in full synchronization +# state, slave still keep itself data. But this way needs to occupy much disk +# space, so we provide a way to reduce disk occupation, slave will delete itself +# entire database before fetching files from master during full synchronization. +# If you want to enable this way, you can set 'slave-delete-db-before-fullsync' +# to yes, but you must know that database will be lost if master is down during +# full synchronization, unless you have a backup of database. +# +# This option is similar redis replicas RDB diskless load option: +# repl-diskless-load on-empty-db +# +# Default: no +slave-empty-db-before-fullsync no + +# If replicas need full synchronization with master, master need to create +# checkpoint for feeding replicas, and replicas also stage a checkpoint of +# the master. If we also keep the backup, it maybe occupy extra disk space. +# You can enable 'purge-backup-on-fullsync' if disk is not sufficient, but +# that may cause remote backup copy failing. +# +# Default: no +purge-backup-on-fullsync no + +# The maximum allowed rate (in MB/s) that should be used by Replication. +# If the rate exceeds max-replication-mb, replication will slow down. +# Default: 0 (i.e. no limit) +max-replication-mb 0 + +# The maximum allowed aggregated write rate of flush and compaction (in MB/s). +# If the rate exceeds max-io-mb, io will slow down. +# 0 is no limit +# Default: 500 +max-io-mb 500 + +# The maximum allowed space (in GB) that should be used by RocksDB. +# If the total size of the SST files exceeds max_allowed_space, writes to RocksDB will fail. +# Please see: https://github.com/facebook/rocksdb/wiki/Managing-Disk-Space-Utilization +# Default: 0 (i.e. no limit) +max-db-size 0 + +# The maximum backup to keep, server cron would run every minutes to check the num of current +# backup, and purge the old backup if exceed the max backup num to keep. If max-backup-to-keep +# is 0, no backup would be keep. But now, we only support 0 or 1. +max-backup-to-keep 1 + +# The maximum hours to keep the backup. If max-backup-keep-hours is 0, wouldn't purge any backup. +# default: 1 day +max-backup-keep-hours 24 + +# max-bitmap-to-string-mb use to limit the max size of bitmap to string transformation(MB). +# +# Default: 16 +max-bitmap-to-string-mb 16 + +################################## SLOW LOG ################################### + +# The Kvrocks Slow Log is a mechanism to log queries that exceeded a specified +# execution time. The execution time does not include the I/O operations +# like talking with the client, sending the reply and so forth, +# but just the time needed to actually execute the command (this is the only +# stage of command execution where the thread is blocked and can not serve +# other requests in the meantime). +# +# You can configure the slow log with two parameters: one tells Kvrocks +# what is the execution time, in microseconds, to exceed in order for the +# command to get logged, and the other parameter is the length of the +# slow log. When a new command is logged the oldest one is removed from the +# queue of logged commands. + +# The following time is expressed in microseconds, so 1000000 is equivalent +# to one second. Note that -1 value disables the slow log, while +# a value of zero forces the logging of every command. +slowlog-log-slower-than 100000 + +# There is no limit to this length. Just be aware that it will consume memory. +# You can reclaim memory used by the slow log with SLOWLOG RESET. +slowlog-max-len 128 + +# If you run kvrocks from upstart or systemd, kvrocks can interact with your +# supervision tree. Options: +# supervised no - no supervision interaction +# supervised upstart - signal upstart by putting kvrocks into SIGSTOP mode +# supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET +# supervised auto - detect upstart or systemd method based on +# UPSTART_JOB or NOTIFY_SOCKET environment variables +# Note: these supervision methods only signal "process is ready." +# They do not enable continuous liveness pings back to your supervisor. +supervised no + +################################## PERF LOG ################################### + +# The Kvrocks Perf Log is a mechanism to log queries' performance context that +# exceeded a specified execution time. This mechanism uses rocksdb's +# Perf Context and IO Stats Context, Please see: +# https://github.com/facebook/rocksdb/wiki/Perf-Context-and-IO-Stats-Context +# +# This mechanism is enabled when profiling-sample-commands is not empty and +# profiling-sample-ratio greater than 0. +# It is important to note that this mechanism affects performance, but it is +# useful for troubleshooting performance bottlenecks, so it should only be +# enabled when performance problems occur. + +# The name of the commands you want to record. Must be original name of +# commands supported by Kvrocks. Use ',' to separate multiple commands and +# use '*' to record all commands supported by Kvrocks. +# Example: +# - Single command: profiling-sample-commands get +# - Multiple commands: profiling-sample-commands get,mget,hget +# +# Default: empty +# profiling-sample-commands "" + +# Ratio of the samples would be recorded. We simply use the rand to determine +# whether to record the sample or not. +# +# Default: 0 +profiling-sample-ratio 0 + +# There is no limit to this length. Just be aware that it will consume memory. +# You can reclaim memory used by the perf log with PERFLOG RESET. +# +# Default: 256 +profiling-sample-record-max-len 256 + +# profiling-sample-record-threshold-ms use to tell the kvrocks when to record. +# +# Default: 100 millisecond +profiling-sample-record-threshold-ms 100 + +################################## CRON ################################### + +# Compact Scheduler, auto compact at schedule time +# time expression format is the same as crontab(currently only support * and int) +# e.g. compact-cron 0 3 * * * 0 4 * * * +# would compact the db at 3am and 4am everyday +# compact-cron 0 3 * * * + +# The hour range that compaction checker would be active +# e.g. compaction-checker-range 0-7 means compaction checker would be worker between +# 0-7am every day. +compaction-checker-range 0-7 + +# Bgsave scheduler, auto bgsave at schedule time +# time expression format is the same as crontab(currently only support * and int) +# e.g. bgsave-cron 0 3 * * * 0 4 * * * +# would bgsave the db at 3am and 4am everyday + +# Command renaming. +# +# It is possible to change the name of dangerous commands in a shared +# environment. For instance the KEYS command may be renamed into something +# hard to guess so that it will still be available for internal-use tools +# but not available for general clients. +# +# Example: +# +# rename-command KEYS b840fc02d524045429941cc15f59e41cb7be6c52 +# +# It is also possible to completely kill a command by renaming it into +# an empty string: +# +# rename-command KEYS "" + +# The key-value size may so be quite different in many scenes, and use 256MiB as SST file size +# may cause data loading(large index/filter block) ineffective when the key-value was too small. +# kvrocks supports user-defined SST file in config(rocksdb.target_file_size_base), +# but it still too trivial and inconvenient to adjust the different sizes for different instances. +# so we want to periodic auto-adjust the SST size in-flight with user avg key-value size. +# +# If enabled, kvrocks will auto resize rocksdb.target_file_size_base +# and rocksdb.write_buffer_size in-flight with user avg key-value size. +# Please see #118. +# +# Default: yes +auto-resize-block-and-sst yes + +################################ MIGRATE ##################################### +# If the network bandwidth is completely consumed by the migration task, +# it will affect the availability of kvrocks. To avoid this situation, +# migrate-speed is adpoted to limit the migrating speed. +# Migrating speed is limited by controling the duraiton between sending data, +# the duation is calculated by: 1000000 * migrate-pipeline-size / migrate-speed (us). +# Value: [0,INT_MAX], 0 means no limit +# +# Default: 4096 +migrate-speed 4096 + +# In order to reduce data transimission times and improve the efficiency of data migration, +# pipeline is adopted to send multiple data at once. Pipeline size can be set by this option. +# Value: [1, INT_MAX], it can't be 0 +# +# Default: 16 +migrate-pipeline-size 16 + +# In order to reduce the write forbidden time during migrating slot, we will migrate the incremetal +# data sevral times to reduce the amount of incremetal data. Until the quantity of incremetal +# data is reduced to a certain threshold, slot will be forbidden write. The threshold is set by +# this option. +# Value: [1, INT_MAX], it can't be 0 +# +# Default: 10000 +migrate-sequence-gap 10000 + +################################ ROCKSDB ##################################### + +# Specify the capacity of metadata column family block cache. Larger block cache +# may make request faster while more keys would be cached. Max Size is 200*1024. +# Default: 2048MB +rocksdb.metadata_block_cache_size 2048 + +# Specify the capacity of subkey column family block cache. Larger block cache +# may make request faster while more keys would be cached. Max Size is 200*1024. +# Default: 2048MB +rocksdb.subkey_block_cache_size 2048 + +# Metadata column family and subkey column family will share a single block cache +# if set 'yes'. The capacity of shared block cache is +# metadata_block_cache_size + subkey_block_cache_size +# +# Default: yes +rocksdb.share_metadata_and_subkey_block_cache yes + +# A global cache for table-level rows in RocksDB. If almost always point +# lookups, enlarging row cache may improve read performance. Otherwise, +# if we enlarge this value, we can lessen metadata/subkey block cache size. +# +# Default: 0 (disabled) +rocksdb.row_cache_size 0 + +# Number of open files that can be used by the DB. You may need to +# increase this if your database has a large working set. Value -1 means +# files opened are always kept open. You can estimate number of files based +# on target_file_size_base and target_file_size_multiplier for level-based +# compaction. For universal-style compaction, you can usually set it to -1. +# Default: 4096 +rocksdb.max_open_files 8096 + +# Amount of data to build up in memory (backed by an unsorted log +# on disk) before converting to a sorted on-disk file. +# +# Larger values increase performance, especially during bulk loads. +# Up to max_write_buffer_number write buffers may be held in memory +# at the same time, +# so you may wish to adjust this parameter to control memory usage. +# Also, a larger write buffer will result in a longer recovery time +# the next time the database is opened. +# +# Note that write_buffer_size is enforced per column family. +# See db_write_buffer_size for sharing memory across column families. + +# default is 64MB +rocksdb.write_buffer_size 64 + +# Target file size for compaction, target file size for Leve N can be caculated +# by target_file_size_base * (target_file_size_multiplier ^ (L-1)) +# +# Default: 128MB +rocksdb.target_file_size_base 128 + +# The maximum number of write buffers that are built up in memory. +# The default and the minimum number is 2, so that when 1 write buffer +# is being flushed to storage, new writes can continue to the other +# write buffer. +# If max_write_buffer_number > 3, writing will be slowed down to +# options.delayed_write_rate if we are writing to the last write buffer +# allowed. +rocksdb.max_write_buffer_number 4 + +# Maximum number of concurrent background compaction jobs, submitted to +# the default LOW priority thread pool. +rocksdb.max_background_compactions 4 + +# Maximum number of concurrent background memtable flush jobs, submitted by +# default to the HIGH priority thread pool. If the HIGH priority thread pool +# is configured to have zero threads, flush jobs will share the LOW priority +# thread pool with compaction jobs. +rocksdb.max_background_flushes 4 + +# This value represents the maximum number of threads that will +# concurrently perform a compaction job by breaking it into multiple, +# smaller ones that are run simultaneously. +# Default: 2 (i.e. no subcompactions) +rocksdb.max_sub_compactions 2 + +# In order to limit the size of WALs, RocksDB uses DBOptions::max_total_wal_size +# as the trigger of column family flush. Once WALs exceed this size, RocksDB +# will start forcing the flush of column families to allow deletion of some +# oldest WALs. This config can be useful when column families are updated at +# non-uniform frequencies. If there's no size limit, users may need to keep +# really old WALs when the infrequently-updated column families hasn't flushed +# for a while. +# +# In kvrocks, we use multiple column families to store metadata, subkeys, etc. +# If users always use string type, but use list, hash and other complex data types +# infrequently, there will be a lot of old WALs if we don't set size limit +# (0 by default in rocksdb), because rocksdb will dynamically choose the WAL size +# limit to be [sum of all write_buffer_size * max_write_buffer_number] * 4 if set to 0. +# +# Moreover, you should increase this value if you already set rocksdb.write_buffer_size +# to a big value, to avoid influencing the effect of rocksdb.write_buffer_size and +# rocksdb.max_write_buffer_number. +# +# default is 512MB +rocksdb.max_total_wal_size 512 + +# We impl the repliction with rocksdb WAL, it would trigger full sync when the seq was out of range. +# wal_ttl_seconds and wal_size_limit_mb would affect how archived logswill be deleted. +# If WAL_ttl_seconds is not 0, then WAL files will be checked every WAL_ttl_seconds / 2 and those that +# are older than WAL_ttl_seconds will be deleted# +# +# Default: 3 Hours +rocksdb.wal_ttl_seconds 10800 + +# If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0, +# WAL files will be checked every 10 min and if total size is greater +# then WAL_size_limit_MB, they will be deleted starting with the +# earliest until size_limit is met. All empty files will be deleted +# Default: 16GB +rocksdb.wal_size_limit_mb 16384 + +# Approximate size of user data packed per block. Note that the +# block size specified here corresponds to uncompressed data. The +# actual size of the unit read from disk may be smaller if +# compression is enabled. +# +# Default: 4KB +rocksdb.block_size 16384 + +# Indicating if we'd put index/filter blocks to the block cache +# +# Default: no +rocksdb.cache_index_and_filter_blocks yes + +# Specify the compression to use. Only compress level greater +# than 2 to improve performance. +# Accept value: "no", "snappy" +# default snappy +rocksdb.compression snappy + +# If non-zero, we perform bigger reads when doing compaction. If you're +# running RocksDB on spinning disks, you should set this to at least 2MB. +# That way RocksDB's compaction is doing sequential instead of random reads. +# When non-zero, we also force new_table_reader_for_compaction_inputs to +# true. +# +# Default: 2 MB +rocksdb.compaction_readahead_size 2097152 + +# he limited write rate to DB if soft_pending_compaction_bytes_limit or +# level0_slowdown_writes_trigger is triggered. + +# If the value is 0, we will infer a value from `rater_limiter` value +# if it is not empty, or 16MB if `rater_limiter` is empty. Note that +# if users change the rate in `rate_limiter` after DB is opened, +# `delayed_write_rate` won't be adjusted. +# +rocksdb.delayed_write_rate 0 +# If enable_pipelined_write is true, separate write thread queue is +# maintained for WAL write and memtable write. +# +# Default: no +rocksdb.enable_pipelined_write no + +# Soft limit on number of level-0 files. We start slowing down writes at this +# point. A value <0 means that no writing slow down will be triggered by +# number of files in level-0. +# +# Default: 20 +rocksdb.level0_slowdown_writes_trigger 20 + +# Maximum number of level-0 files. We stop writes at this point. +# +# Default: 40 +rocksdb.level0_stop_writes_trigger 40 + +# Number of files to trigger level-0 compaction. +# +# Default: 4 +rocksdb.level0_file_num_compaction_trigger 4 + +# if not zero, dump rocksdb.stats to LOG every stats_dump_period_sec +# +# Default: 0 +rocksdb.stats_dump_period_sec 0 + +# if yes, the auto compaction would be disabled, but the manual compaction remain works +# +# Default: no +rocksdb.disable_auto_compactions no + +# BlobDB(key-value separation) is essentially RocksDB for large-value use cases. +# Since 6.18.0, The new implementation is integrated into the RocksDB core. +# When set, large values (blobs) are written to separate blob files, and only +# pointers to them are stored in SST files. This can reduce write amplification +# for large-value use cases at the cost of introducing a level of indirection +# for reads. Please see: https://github.com/facebook/rocksdb/wiki/BlobDB. +# +# Note that when enable_blob_files is set to yes, BlobDB-related configuration +# items will take effect. +# +# Default: no +rocksdb.enable_blob_files no + +# The size of the smallest value to be stored separately in a blob file. Values +# which have an uncompressed size smaller than this threshold are stored alongside +# the keys in SST files in the usual fashion. +# +# Default: 4096 byte, 0 means that all values are stored in blob files +rocksdb.min_blob_size 4096 + +# The size limit for blob files. When writing blob files, a new file is +# opened once this limit is reached. +# +# Default: 268435456 bytes +rocksdb.blob_file_size 268435456 + +# Enables garbage collection of blobs. Valid blobs residing in blob files +# older than a cutoff get relocated to new files as they are encountered +# during compaction, which makes it possible to clean up blob files once +# they contain nothing but obsolete/garbage blobs. +# See also rocksdb.blob_garbage_collection_age_cutoff below. +# +# Default: yes +rocksdb.enable_blob_garbage_collection yes + +# The percentage cutoff in terms of blob file age for garbage collection. +# Blobs in the oldest N blob files will be relocated when encountered during +# compaction, where N = (garbage_collection_cutoff/100) * number_of_blob_files. +# Note that this value must belong to [0, 100]. +# +# Default: 25 +rocksdb.blob_garbage_collection_age_cutoff 25 + + +# The purpose of following three options are to dynamically adjust the upper limit of +# the data that each layer can store according to the size of the different +# layers of the LSM. Enabling this option will bring some improvements in +# deletion efficiency and space amplification, but it will lose a certain +# amount of read performance. +# If you want know more details about Levels' Target Size, you can read RocksDB wiki: +# https://github.com/facebook/rocksdb/wiki/Leveled-Compaction#levels-target-size +# +# Default: no +rocksdb.level_compaction_dynamic_level_bytes no + +# The total file size of level-1 sst. +# +# Default: 268435456 bytes +rocksdb.max_bytes_for_level_base 268435456 + +# Multiplication factor for the total file size of L(n+1) layers. +# This option is a double type number in RocksDB, but kvrocks is +# not support double data type number yet, so we use int data +# number instead of double currently. +# +# Default: 10 +rocksdb.max_bytes_for_level_multiplier 10 + +################################ NAMESPACE ##################################### +# namespace.test change.me diff --git a/installing_deps.sh b/installing_deps.sh index 24a3b7b1..9b8f4d32 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -11,6 +11,11 @@ sudo apt-get update sudo apt-get install python3-pip virtualenv python3-dev python3-tk libfreetype6-dev \ screen g++ python-tk unzip libsnappy-dev cmake -qq +# update virtualenv +# python3-dev | python3-nose + +# python3-tk | python3-numpy?????? + #Needed for downloading jemalloc sudo apt-get install wget -qq @@ -84,11 +89,21 @@ pushd ardb/ make popd -# KVROCKS # -# test ! -d kvrocks/ && git clone https://github.com/bitleak/kvrocks.git -# pushd kvrocks/ -# make -j4 -# popd +DEFAULT_HOME=$(pwd) + +#### KVROCKS #### +test ! -d kvrocks/ && git clone https://github.com/apache/incubator-kvrocks.git kvrocks +pushd kvrocks +./build.sh build +popd + +DEFAULT_KVROCKS_DATA=$DEFAULT_HOME/DATA_KVROCKS +mkdir -p $DEFAULT_KVROCKS_DATA + +sed -i "s|dir /tmp/kvrocks|dir ${DEFAULT_KVROCKS_DATA}|1" $DEFAULT_HOME/configs/6383.conf +##-- KVROCKS --## + + # Config File if [ ! -f configs/core.cfg ]; then