chg: [chats] add heatmap nb week messages by hour

This commit is contained in:
terrtia 2023-11-13 14:10:24 +01:00
parent abc10a1203
commit 7bf0fe8992
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
6 changed files with 395 additions and 1 deletions

View file

@ -313,6 +313,14 @@ def api_get_chat(chat_id, chat_instance_uuid):
meta['messages'], meta['tags_messages'] = chat.get_messages() meta['messages'], meta['tags_messages'] = chat.get_messages()
return meta, 200 return meta, 200
def api_get_nb_message_by_week(chat_id, chat_instance_uuid):
chat = Chats.Chat(chat_id, chat_instance_uuid)
if not chat.exists():
return {"status": "error", "reason": "Unknown chat"}, 404
week = chat.get_nb_message_this_week()
# week = chat.get_nb_message_by_week('20231109')
return week, 200
def api_get_subchannel(chat_id, chat_instance_uuid): def api_get_subchannel(chat_id, chat_instance_uuid):
subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid) subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid)
if not subchannel.exists(): if not subchannel.exists():

View file

@ -8,6 +8,7 @@ Base Class for AIL Objects
################################## ##################################
import os import os
import sys import sys
import time
from abc import ABC from abc import ABC
from datetime import datetime from datetime import datetime
@ -21,6 +22,7 @@ from lib.objects.abstract_subtype_object import AbstractSubtypeObject
from lib.ail_core import get_object_all_subtypes, zscan_iter ################ from lib.ail_core import get_object_all_subtypes, zscan_iter ################
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib.objects import Messages from lib.objects import Messages
from packages import Date
# from lib.data_retention_engine import update_obj_date # from lib.data_retention_engine import update_obj_date
@ -141,6 +143,30 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
def get_last_message(self): def get_last_message(self):
return r_object.zrevrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, 0) return r_object.zrevrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, 0)
def get_nb_message_by_hours(self, date_day, nb_day):
hours = []
# start=0, end=23
timestamp = time.mktime(datetime.strptime(date_day, "%Y%m%d").timetuple())
for i in range(24):
timestamp_end = timestamp + 3600
nb_messages = r_object.zcount(f'messages:{self.type}:{self.subtype}:{self.id}', timestamp, timestamp_end)
timestamp = timestamp_end
hours.append({'date': f'{date_day[0:4]}-{date_day[4:6]}-{date_day[6:8]}', 'day': nb_day, 'hour': i, 'count': nb_messages})
return hours
def get_nb_message_by_week(self, date_day):
date_day = Date.get_date_week_by_date(date_day)
week_messages = []
i = 0
for date in Date.daterange_add_days(date_day, 6):
week_messages = week_messages + self.get_nb_message_by_hours(date, i)
i += 1
return week_messages
def get_nb_message_this_week(self):
week_date = Date.get_current_week_day()
return self.get_nb_message_by_week(week_date)
def get_message_meta(self, message, timestamp=None): # TODO handle file message def get_message_meta(self, message, timestamp=None): # TODO handle file message
message = Messages.Message(message[9:]) message = Messages.Message(message[9:])
meta = message.get_meta(options={'content', 'link', 'parent', 'parent_meta', 'user-account'}, timestamp=timestamp) meta = message.get_meta(options={'content', 'link', 'parent', 'parent_meta', 'user-account'}, timestamp=timestamp)
@ -205,6 +231,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
self.add_obj_children(obj_global_id, mess_id) self.add_obj_children(obj_global_id, mess_id)
# get_messages_meta ???? # get_messages_meta ????
# TODO move me to abstract subtype # TODO move me to abstract subtype

View file

@ -85,11 +85,25 @@ def get_today_date_str(separator=False):
else: else:
return datetime.date.today().strftime("%Y%m%d") return datetime.date.today().strftime("%Y%m%d")
def get_current_week_day():
dt = datetime.date.today()
start = dt - datetime.timedelta(days=dt.weekday())
return start.strftime("%Y%m%d")
def get_date_week_by_date(date):
dt = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8]))
start = dt - datetime.timedelta(days=dt.weekday())
return start.strftime("%Y%m%d")
def date_add_day(date, num_day=1): def date_add_day(date, num_day=1):
new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day) new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day)
new_date = str(new_date).replace('-', '') new_date = str(new_date).replace('-', '')
return new_date return new_date
def daterange_add_days(date, nb_days):
end_date = date_add_day(date, num_day=nb_days)
return get_daterange(date, end_date)
def date_substract_day(date, num_day=1): def date_substract_day(date, num_day=1):
new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) - datetime.timedelta(num_day) new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) - datetime.timedelta(num_day)
new_date = str(new_date).replace('-', '') new_date = str(new_date).replace('-', '')

View file

@ -87,6 +87,18 @@ def chats_explorer_chat():
chat = chat[0] chat = chat[0]
return render_template('chat_viewer.html', chat=chat, bootstrap_label=bootstrap_label) return render_template('chat_viewer.html', chat=chat, bootstrap_label=bootstrap_label)
@chats_explorer.route("chats/explorer/messages/stats/week", methods=['GET'])
@login_required
@login_read_only
def chats_explorer_messages_stats_week():
chat_id = request.args.get('id')
instance_uuid = request.args.get('uuid')
week = chats_viewer.api_get_nb_message_by_week(chat_id, instance_uuid)
if week[1] != 200:
return create_json_response(week[0], week[1])
else:
return jsonify(week[0])
@chats_explorer.route("/chats/explorer/subchannel", methods=['GET']) @chats_explorer.route("/chats/explorer/subchannel", methods=['GET'])
@login_required @login_required
@login_read_only @login_read_only

View file

@ -16,6 +16,7 @@
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script> <script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script> <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script> <script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/d3.min.js')}}"></script>
<style> <style>
.chat-message-left, .chat-message-left,
@ -142,6 +143,8 @@
{% endif %} {% endif %}
<div id="heatmapweekhour"></div>
{% if chat['messages'] %} {% if chat['messages'] %}
<div class="position-relative"> <div class="position-relative">
@ -244,6 +247,336 @@ function toggle_sidebar(){
} }
</script> </script>
<script>
/*
// set the dimensions and margins of the graph
const margin = {top: 30, right: 30, bottom: 30, left: 30},
width = 450 - margin.left - margin.right,
height = 450 - margin.top - margin.bottom;
// append the svg object to the body of the page
const svg = d3.select("#my_dataviz")
.append("svg")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append("g")
.attr("transform", `translate(${margin.left},${margin.top})`);
// Labels of row and columns
const myGroups = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"]
const myVars = ["v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10"]
//Read the data
d3.csv("").then( function(data) {
// Labels of row and columns -> unique identifier of the column called 'group' and 'variable'
const myGroups = Array.from(new Set(data.map(d => d.group)))
const myVars = Array.from(new Set(data.map(d => d.variable)))
// Build X scales and axis:
const x = d3.scaleBand()
.range([ 0, width ])
.domain(myGroups)
.padding(0.05);
svg.append("g")
.style("font-size", 15)
.attr("transform", `translate(0, ${height})`)
.call(d3.axisBottom(x).tickSize(0))
.select(".domain").remove()
// Build Y scales and axis:
const y = d3.scaleBand()
.range([ height, 0 ])
.domain(myVars)
.padding(0.01);
svg.append("g")
.style("font-size", 15)
.call(d3.axisLeft(y).tickSize(0))
.select(".domain").remove()
// Build color scale
const myColor = d3.scaleSequential()
.interpolator(d3.interpolateInferno)
.domain([1,100])
// create a tooltip
const tooltip = d3.select("#my_dataviz")
.append("div")
.style("opacity", 0)
.attr("class", "tooltip")
.style("background-color", "white")
.style("border", "solid")
.style("border-width", "2px")
.style("border-radius", "5px")
.style("padding", "5px")
// Three function that change the tooltip when user hover / move / leave a cell
const mouseover = function(event,d) {
tooltip.style("opacity", 1)
d3.select(this)
.style("stroke", "black")
.style("opacity", 1)
}
const mousemove = function(event,d) {
tooltip.html("The exact value of<br>this cell is: " + d)
.style("left", (event.x)/2 + "px")
.style("top", (event.y)/2 + "px")
}
const mouseleave = function(d) {
tooltip.style("opacity", 0)
d3.select(this)
.style("stroke", "none")
.style("opacity", 0.8)
}
svg.selectAll()
.data(data, function(d) {return d.group+':'+d.variable;})
.join("rect")
.attr("x", function(d) { return x(d.group) })
.attr("y", function(d) { return y(d.variable) })
.attr("rx", 4)
.attr("ry", 4)
.attr("width", x.bandwidth() )
.attr("height", y.bandwidth() )
.style("fill", function(d) { return myColor(d.value)} )
.style("stroke-width", 4)
.style("stroke", "none")
.style("opacity", 0.8)
.on("mouseover", mouseover)
.on("mousemove", mousemove)
.on("mouseleave", mouseleave)
})
*/
</script>
<script>
// based on gist nbremer/62cf60e116ae821c06602793d265eaf6
d3.json("{{ url_for('chats_explorer.chats_explorer_messages_stats_week') }}?uuid={{ chat['subtype'] }}&id={{ chat['id'] }}")
.then(function(data) {
var days = ["Mo", "Tu", "We", "Th", "Fr", "Sa", "Su"],
times = d3.range(24);
var margin = {
top: 80,
right: 50,
bottom: 20,
left: 50
};
var width = Math.max(Math.min(window.innerWidth, 1000), 500) - margin.left - margin.right - 20,
gridSize = Math.floor(width / times.length),
height = gridSize * (days.length + 2);
var heatmap_font_size = width * 62.5 / 900;
//SVG container
var svg = d3.select('#heatmapweekhour')
.append("svg")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
// create a tooltip
const tooltip = d3.select("#heatmapweekhour")
.append("div")
.style("opacity", 0)
.attr("class", "tooltip")
.style("background-color", "white")
.style("border", "solid")
.style("border-width", "2px")
.style("border-radius", "5px")
.style("padding", "5px")
// Three function that change the tooltip when user hover / move / leave a cell
const mouseover = function(d) {
tooltip.style("opacity", 1)
d3.select(this)
.style("stroke", "black")
.style("opacity", 1)
tooltip.html(d.date + " " + d.hour + "-" + (d.hour + 1) + "h: <b>" + d.count + "</b> messages")
}
const mouseleave = function(d) {
console.log(d)
console.log(d.hour)
console.log(d.day)
tooltip.style("opacity", 0)
d3.select(this)
.style("stroke", "none")
.style("opacity", 0.8)
}
///////////////////////////////////////////////////////////////////////////
//////////////////////////// Draw Heatmap /////////////////////////////////
///////////////////////////////////////////////////////////////////////////
var colorScale = d3.scaleLinear()
.domain([0, d3.max(data, function (d) {
return d.count;
}) / 2, d3.max(data, function (d) {
return d.count;
})])
.range(["#FFFFF6", "#3E9583", "#1F2D86"])
//.interpolate(d3.interpolateHcl);
var dayLabels = svg.selectAll(".dayLabel")
.data(days)
.enter().append("text")
.text(function (d) {
return d;
})
.attr("x", 0)
.attr("y", function (d, i) {
return i * gridSize;
})
.style("text-anchor", "end")
.attr("transform", "translate(-36, -11)")
.attr("class", function (d, i) {
return ((i >= 0 && i <= 4) ? "dayLabel mono axis axis-workweek" : "dayLabel mono axis");
})
.style("font-size", heatmap_font_size + "%");
var timeLabels = svg.selectAll(".timeLabel")
.data(times)
.enter().append("text")
.text(function (d) {
return d;
})
.attr("x", function (d, i) {
return i * gridSize;
})
.attr("y", 0)
.style("text-anchor", "middle")
.attr("transform", "translate(-" + gridSize / 2 + ", -36)")
.attr("class", function (d, i) {
return ((i >= 8 && i <= 17) ? "timeLabel mono axis axis-worktime" : "timeLabel mono axis");
})
.style("font-size", heatmap_font_size + "%");
var heatMap = svg.selectAll(".hour")
.data(data)
.enter().append("rect")
.attr("x", function (d) {
return (d.hour - 1) * gridSize;
})
.attr("y", function (d) {
return (d.day - 1) * gridSize;
})
.attr("class", "hour bordered")
.attr("width", gridSize)
.attr("height", gridSize)
.style("stroke", "white")
.style("stroke-opacity", 0.6)
.style("fill", function (d) {
return colorScale(d.count);
})
.on("mouseover", mouseover)
.on("mouseleave", mouseleave);
//Append title to the top
svg.append("text")
.attr("class", "title")
.attr("x", width / 2)
.attr("y", -60)
.style("text-anchor", "middle")
.text("Chat Messages");
///////////////////////////////////////////////////////////////////////////
//////////////// Create the gradient for the legend ///////////////////////
///////////////////////////////////////////////////////////////////////////
//Extra scale since the color scale is interpolated
var countScale = d3.scaleLinear()
.domain([0, d3.max(data, function (d) {
return d.count;
})])
.range([0, width])
//Calculate the variables for the temp gradient
var numStops = 10;
countRange = countScale.domain();
countRange[2] = countRange[1] - countRange[0];
countPoint = [];
for (var i = 0; i < numStops; i++) {
countPoint.push(i * countRange[2] / (numStops - 1) + countRange[0]);
}//for i
//Create the gradient
svg.append("defs")
.append("linearGradient")
.attr("id", "legend-heatmap")
.attr("x1", "0%").attr("y1", "0%")
.attr("x2", "100%").attr("y2", "0%")
.selectAll("stop")
.data(d3.range(numStops))
.enter().append("stop")
.attr("offset", function (d, i) {
return countScale(countPoint[i]) / width;
})
.attr("stop-color", function (d, i) {
return colorScale(countPoint[i]);
});
///////////////////////////////////////////////////////////////////////////
////////////////////////// Draw the legend ////////////////////////////////
///////////////////////////////////////////////////////////////////////////
var legendWidth = Math.min(width * 0.8, 400);
//Color Legend container
var legendsvg = svg.append("g")
.attr("class", "legendWrapper")
.attr("transform", "translate(" + (width / 2) + "," + (gridSize * days.length) + ")"); // 319
//Draw the Rectangle
legendsvg.append("rect")
.attr("class", "legendRect")
.attr("x", -legendWidth / 2)
.attr("y", 0)
//.attr("rx", hexRadius*1.25/2)
.attr("width", legendWidth)
.attr("height", 10)
.style("fill", "url(#legend-heatmap)");
//Append title
legendsvg.append("text")
.attr("class", "legendTitle")
.attr("x", 0)
.attr("y", -10)
.style("text-anchor", "middle")
.text("Number of Messages");
//Set scale for x-axis
var xScale = d3.scaleLinear()
.range([-legendWidth / 2, legendWidth / 2])
.domain([0, d3.max(data, function (d) {
return d.count;
})]);
//Define x-axis
var xAxis = d3.axisBottom(xScale)
//.orient("bottom")
.ticks(5);
//.tickFormat(formatPercent)
//Set up X axis
legendsvg.append("g")
.attr("class", "axis")
.attr("transform", "translate(0," + (10) + ")")
.call(xAxis);
})
</script>
</body> </body>
</html> </html>

View file

@ -10,7 +10,7 @@ wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.
SBADMIN_VERSION='3.3.7' SBADMIN_VERSION='3.3.7'
BOOTSTRAP_VERSION='4.2.1' BOOTSTRAP_VERSION='4.2.1'
FONT_AWESOME_VERSION='5.7.1' FONT_AWESOME_VERSION='5.7.1'
D3_JS_VERSION='5.5.0' D3_JS_VERSION='5.16.0'
rm -rf temp rm -rf temp
mkdir temp mkdir temp