chg: [chats] add heatmap nb week messages by hour

bin/lib/chats_viewer.py

@@ -313,6 +313,14 @@ def api_get_chat(chat_id, chat_instance_uuid):
         meta['messages'], meta['tags_messages'] = chat.get_messages()
     return meta, 200
 
+def api_get_nb_message_by_week(chat_id, chat_instance_uuid):
+    chat = Chats.Chat(chat_id, chat_instance_uuid)
+    if not chat.exists():
+        return {"status": "error", "reason": "Unknown chat"}, 404
+    week = chat.get_nb_message_this_week()
+    # week = chat.get_nb_message_by_week('20231109')
+    return week, 200
+
 def api_get_subchannel(chat_id, chat_instance_uuid):
     subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid)
     if not subchannel.exists():

bin/lib/objects/abstract_chat_object.py

@@ -8,6 +8,7 @@ Base Class for AIL Objects
 ##################################
 import os
 import sys
+import time
 from abc import ABC
 
 from datetime import datetime
@@ -21,6 +22,7 @@ from lib.objects.abstract_subtype_object import AbstractSubtypeObject
 from lib.ail_core import get_object_all_subtypes, zscan_iter ################
 from lib.ConfigLoader import ConfigLoader
 from lib.objects import Messages
+from packages import Date
 
 # from lib.data_retention_engine import update_obj_date
 
@@ -141,6 +143,30 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
     def get_last_message(self):
         return r_object.zrevrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, 0)
 
+    def get_nb_message_by_hours(self, date_day, nb_day):
+        hours = []
+        # start=0, end=23
+        timestamp = time.mktime(datetime.strptime(date_day, "%Y%m%d").timetuple())
+        for i in range(24):
+            timestamp_end = timestamp + 3600
+            nb_messages = r_object.zcount(f'messages:{self.type}:{self.subtype}:{self.id}', timestamp, timestamp_end)
+            timestamp = timestamp_end
+            hours.append({'date': f'{date_day[0:4]}-{date_day[4:6]}-{date_day[6:8]}', 'day': nb_day, 'hour': i, 'count': nb_messages})
+        return hours
+
+    def get_nb_message_by_week(self, date_day):
+        date_day = Date.get_date_week_by_date(date_day)
+        week_messages = []
+        i = 0
+        for date in Date.daterange_add_days(date_day, 6):
+            week_messages = week_messages + self.get_nb_message_by_hours(date, i)
+            i += 1
+        return week_messages
+
+    def get_nb_message_this_week(self):
+        week_date = Date.get_current_week_day()
+        return self.get_nb_message_by_week(week_date)
+
     def get_message_meta(self, message, timestamp=None):  # TODO handle file message
         message = Messages.Message(message[9:])
         meta = message.get_meta(options={'content', 'link', 'parent', 'parent_meta', 'user-account'}, timestamp=timestamp)
@@ -205,6 +231,7 @@ class AbstractChatObject(AbstractSubtypeObject, ABC):
             self.add_obj_children(obj_global_id, mess_id)
 
 
+
     # get_messages_meta ????
 
 # TODO move me to abstract subtype

bin/packages/Date.py

@@ -85,11 +85,25 @@ def get_today_date_str(separator=False):
     else:
         return datetime.date.today().strftime("%Y%m%d")
 
+def get_current_week_day():
+    dt = datetime.date.today()
+    start = dt - datetime.timedelta(days=dt.weekday())
+    return start.strftime("%Y%m%d")
+
+def get_date_week_by_date(date):
+    dt = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8]))
+    start = dt - datetime.timedelta(days=dt.weekday())
+    return start.strftime("%Y%m%d")
+
 def date_add_day(date, num_day=1):
     new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day)
     new_date = str(new_date).replace('-', '')
     return new_date
 
+def daterange_add_days(date, nb_days):
+    end_date = date_add_day(date, num_day=nb_days)
+    return get_daterange(date, end_date)
+
 def date_substract_day(date, num_day=1):
     new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) - datetime.timedelta(num_day)
     new_date = str(new_date).replace('-', '')

var/www/blueprints/chats_explorer.py

@@ -87,6 +87,18 @@ def chats_explorer_chat():
         chat = chat[0]
         return render_template('chat_viewer.html', chat=chat, bootstrap_label=bootstrap_label)
 
+@chats_explorer.route("chats/explorer/messages/stats/week", methods=['GET'])
+@login_required
+@login_read_only
+def chats_explorer_messages_stats_week():
+    chat_id = request.args.get('id')
+    instance_uuid = request.args.get('uuid')
+    week = chats_viewer.api_get_nb_message_by_week(chat_id, instance_uuid)
+    if week[1] != 200:
+        return create_json_response(week[0], week[1])
+    else:
+        return jsonify(week[0])
+
 @chats_explorer.route("/chats/explorer/subchannel", methods=['GET'])
 @login_required
 @login_read_only

var/www/templates/chats_explorer/chat_viewer.html

@@ -16,6 +16,7 @@
 	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
 	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
 	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/d3.min.js')}}"></script>
 
     <style>
         .chat-message-left,
@@ -142,6 +143,8 @@
 
                 {% endif %}
 
+                <div id="heatmapweekhour"></div>
+
                 {% if chat['messages'] %}
 
                     <div class="position-relative">
@@ -244,6 +247,336 @@ function toggle_sidebar(){
 }
 </script>
 
+
+<script>
+/*
+
+// set the dimensions and margins of the graph
+const margin = {top: 30, right: 30, bottom: 30, left: 30},
+  width = 450 - margin.left - margin.right,
+  height = 450 - margin.top - margin.bottom;
+
+// append the svg object to the body of the page
+const svg = d3.select("#my_dataviz")
+.append("svg")
+  .attr("width", width + margin.left + margin.right)
+  .attr("height", height + margin.top + margin.bottom)
+.append("g")
+  .attr("transform", `translate(${margin.left},${margin.top})`);
+
+// Labels of row and columns
+const myGroups = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"]
+const myVars = ["v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10"]
+
+//Read the data
+d3.csv("").then( function(data) {
+
+    // Labels of row and columns -> unique identifier of the column called 'group' and 'variable'
+    const myGroups = Array.from(new Set(data.map(d => d.group)))
+    const myVars = Array.from(new Set(data.map(d => d.variable)))
+
+    // Build X scales and axis:
+    const x = d3.scaleBand()
+        .range([ 0, width ])
+        .domain(myGroups)
+        .padding(0.05);
+    svg.append("g")
+        .style("font-size", 15)
+        .attr("transform", `translate(0, ${height})`)
+        .call(d3.axisBottom(x).tickSize(0))
+        .select(".domain").remove()
+
+    // Build Y scales and axis:
+    const y = d3.scaleBand()
+        .range([ height, 0 ])
+        .domain(myVars)
+        .padding(0.01);
+    svg.append("g")
+        .style("font-size", 15)
+        .call(d3.axisLeft(y).tickSize(0))
+        .select(".domain").remove()
+
+    // Build color scale
+    const myColor = d3.scaleSequential()
+        .interpolator(d3.interpolateInferno)
+        .domain([1,100])
+
+    // create a tooltip
+    const tooltip = d3.select("#my_dataviz")
+        .append("div")
+        .style("opacity", 0)
+        .attr("class", "tooltip")
+        .style("background-color", "white")
+        .style("border", "solid")
+        .style("border-width", "2px")
+        .style("border-radius", "5px")
+        .style("padding", "5px")
+
+    // Three function that change the tooltip when user hover / move / leave a cell
+    const mouseover = function(event,d) {
+        tooltip.style("opacity", 1)
+        d3.select(this)
+            .style("stroke", "black")
+            .style("opacity", 1)
+    }
+    const mousemove = function(event,d) {
+        tooltip.html("The exact value of<br>this cell is: " + d)
+            .style("left", (event.x)/2 + "px")
+            .style("top", (event.y)/2 + "px")
+    }
+    const mouseleave = function(d) {
+        tooltip.style("opacity", 0)
+        d3.select(this)
+            .style("stroke", "none")
+            .style("opacity", 0.8)
+    }
+
+
+    svg.selectAll()
+        .data(data, function(d) {return d.group+':'+d.variable;})
+        .join("rect")
+            .attr("x", function(d) { return x(d.group) })
+            .attr("y", function(d) { return y(d.variable) })
+            .attr("rx", 4)
+            .attr("ry", 4)
+            .attr("width", x.bandwidth() )
+            .attr("height", y.bandwidth() )
+            .style("fill", function(d) { return myColor(d.value)} )
+            .style("stroke-width", 4)
+            .style("stroke", "none")
+            .style("opacity", 0.8)
+        .on("mouseover", mouseover)
+        .on("mousemove", mousemove)
+        .on("mouseleave", mouseleave)
+
+})
+
+ */
+</script>
+
+<script>
+
+// based on gist nbremer/62cf60e116ae821c06602793d265eaf6
+d3.json("{{ url_for('chats_explorer.chats_explorer_messages_stats_week') }}?uuid={{ chat['subtype'] }}&id={{ chat['id'] }}")
+.then(function(data) {
+
+    var days = ["Mo", "Tu", "We", "Th", "Fr", "Sa", "Su"],
+        times = d3.range(24);
+
+    var margin = {
+        top: 80,
+        right: 50,
+        bottom: 20,
+        left: 50
+    };
+
+    var width = Math.max(Math.min(window.innerWidth, 1000), 500) - margin.left - margin.right - 20,
+        gridSize = Math.floor(width / times.length),
+        height = gridSize * (days.length + 2);
+
+    var heatmap_font_size = width * 62.5 / 900;
+
+//SVG container
+    var svg = d3.select('#heatmapweekhour')
+        .append("svg")
+        .attr("width", width + margin.left + margin.right)
+        .attr("height", height + margin.top + margin.bottom)
+        .append("g")
+        .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+    // create a tooltip
+    const tooltip = d3.select("#heatmapweekhour")
+        .append("div")
+        .style("opacity", 0)
+        .attr("class", "tooltip")
+        .style("background-color", "white")
+        .style("border", "solid")
+        .style("border-width", "2px")
+        .style("border-radius", "5px")
+        .style("padding", "5px")
+
+    // Three function that change the tooltip when user hover / move / leave a cell
+    const mouseover = function(d) {
+        tooltip.style("opacity", 1)
+        d3.select(this)
+            .style("stroke", "black")
+            .style("opacity", 1)
+
+        tooltip.html(d.date + " " + d.hour + "-" + (d.hour + 1) + "h: <b>" + d.count + "</b> messages")
+    }
+    const mouseleave = function(d) {
+        console.log(d)
+        console.log(d.hour)
+        console.log(d.day)
+        tooltip.style("opacity", 0)
+        d3.select(this)
+            .style("stroke", "none")
+            .style("opacity", 0.8)
+    }
+
+///////////////////////////////////////////////////////////////////////////
+//////////////////////////// Draw Heatmap /////////////////////////////////
+///////////////////////////////////////////////////////////////////////////
+
+    var colorScale = d3.scaleLinear()
+        .domain([0, d3.max(data, function (d) {
+            return d.count;
+        }) / 2, d3.max(data, function (d) {
+            return d.count;
+        })])
+        .range(["#FFFFF6", "#3E9583", "#1F2D86"])
+    //.interpolate(d3.interpolateHcl);
+
+    var dayLabels = svg.selectAll(".dayLabel")
+        .data(days)
+        .enter().append("text")
+        .text(function (d) {
+            return d;
+        })
+        .attr("x", 0)
+        .attr("y", function (d, i) {
+            return i * gridSize;
+        })
+        .style("text-anchor", "end")
+        .attr("transform", "translate(-36, -11)")
+        .attr("class", function (d, i) {
+            return ((i >= 0 && i <= 4) ? "dayLabel mono axis axis-workweek" : "dayLabel mono axis");
+        })
+        .style("font-size", heatmap_font_size + "%");
+
+    var timeLabels = svg.selectAll(".timeLabel")
+        .data(times)
+        .enter().append("text")
+        .text(function (d) {
+            return d;
+        })
+        .attr("x", function (d, i) {
+            return i * gridSize;
+        })
+        .attr("y", 0)
+        .style("text-anchor", "middle")
+        .attr("transform", "translate(-" + gridSize / 2 + ", -36)")
+        .attr("class", function (d, i) {
+            return ((i >= 8 && i <= 17) ? "timeLabel mono axis axis-worktime" : "timeLabel mono axis");
+        })
+        .style("font-size", heatmap_font_size + "%");
+
+    var heatMap = svg.selectAll(".hour")
+        .data(data)
+        .enter().append("rect")
+        .attr("x", function (d) {
+            return (d.hour - 1) * gridSize;
+        })
+        .attr("y", function (d) {
+            return (d.day - 1) * gridSize;
+        })
+        .attr("class", "hour bordered")
+        .attr("width", gridSize)
+        .attr("height", gridSize)
+        .style("stroke", "white")
+        .style("stroke-opacity", 0.6)
+        .style("fill", function (d) {
+            return colorScale(d.count);
+        })
+        .on("mouseover", mouseover)
+        .on("mouseleave", mouseleave);
+
+//Append title to the top
+    svg.append("text")
+        .attr("class", "title")
+        .attr("x", width / 2)
+        .attr("y", -60)
+        .style("text-anchor", "middle")
+        .text("Chat Messages");
+
+///////////////////////////////////////////////////////////////////////////
+//////////////// Create the gradient for the legend ///////////////////////
+///////////////////////////////////////////////////////////////////////////
+
+//Extra scale since the color scale is interpolated
+    var countScale = d3.scaleLinear()
+        .domain([0, d3.max(data, function (d) {
+            return d.count;
+        })])
+        .range([0, width])
+
+//Calculate the variables for the temp gradient
+    var numStops = 10;
+    countRange = countScale.domain();
+    countRange[2] = countRange[1] - countRange[0];
+    countPoint = [];
+    for (var i = 0; i < numStops; i++) {
+        countPoint.push(i * countRange[2] / (numStops - 1) + countRange[0]);
+    }//for i
+
+//Create the gradient
+    svg.append("defs")
+        .append("linearGradient")
+        .attr("id", "legend-heatmap")
+        .attr("x1", "0%").attr("y1", "0%")
+        .attr("x2", "100%").attr("y2", "0%")
+        .selectAll("stop")
+        .data(d3.range(numStops))
+        .enter().append("stop")
+        .attr("offset", function (d, i) {
+            return countScale(countPoint[i]) / width;
+        })
+        .attr("stop-color", function (d, i) {
+            return colorScale(countPoint[i]);
+        });
+
+///////////////////////////////////////////////////////////////////////////
+////////////////////////// Draw the legend ////////////////////////////////
+///////////////////////////////////////////////////////////////////////////
+
+    var legendWidth = Math.min(width * 0.8, 400);
+//Color Legend container
+    var legendsvg = svg.append("g")
+        .attr("class", "legendWrapper")
+        .attr("transform", "translate(" + (width / 2) + "," + (gridSize * days.length) + ")");  // 319
+
+//Draw the Rectangle
+    legendsvg.append("rect")
+        .attr("class", "legendRect")
+        .attr("x", -legendWidth / 2)
+        .attr("y", 0)
+        //.attr("rx", hexRadius*1.25/2)
+        .attr("width", legendWidth)
+        .attr("height", 10)
+        .style("fill", "url(#legend-heatmap)");
+
+//Append title
+    legendsvg.append("text")
+        .attr("class", "legendTitle")
+        .attr("x", 0)
+        .attr("y", -10)
+        .style("text-anchor", "middle")
+        .text("Number of Messages");
+
+//Set scale for x-axis
+    var xScale = d3.scaleLinear()
+        .range([-legendWidth / 2, legendWidth / 2])
+        .domain([0, d3.max(data, function (d) {
+            return d.count;
+        })]);
+
+//Define x-axis
+    var xAxis = d3.axisBottom(xScale)
+        //.orient("bottom")
+        .ticks(5);
+    //.tickFormat(formatPercent)
+
+//Set up X axis
+    legendsvg.append("g")
+        .attr("class", "axis")
+        .attr("transform", "translate(0," + (10) + ")")
+        .call(xAxis);
+
+
+})
+
+</script>
+
 </body>
 
 </html>

var/www/update_thirdparty.sh

@@ -10,7 +10,7 @@ wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.
 SBADMIN_VERSION='3.3.7'
 BOOTSTRAP_VERSION='4.2.1'
 FONT_AWESOME_VERSION='5.7.1'
-D3_JS_VERSION='5.5.0'
+D3_JS_VERSION='5.16.0'
 
 rm -rf temp
 mkdir temp