chg: [user-account viewer] add messages heatmap by year

bin/lib/chats_viewer.py

@@ -11,7 +11,7 @@ import sys
 import time
 import uuid
 
-from datetime import datetime
+from datetime import datetime, timezone
 
 sys.path.append(os.environ['AIL_BIN'])
 ##################################
@@ -501,6 +501,29 @@ def get_user_account_nb_all_week_messages(user_id, chats, subchannels):
         nb_day += 1
     return stats
 
+
+def get_user_account_nb_year_messages(user_id, chats, year):
+    nb_year = {}
+    nb_max = 0
+    start = int(datetime(year, 1, 1, 0, 0, 0, tzinfo=timezone.utc).timestamp())
+    end = int(datetime(year, 12, 31, 23, 59, 59, tzinfo=timezone.utc).timestamp())
+
+    for chat_g_id in chats:
+        c_subtype, c_id = chat_g_id.split(':', 1)
+        chat = Chats.Chat(c_id, c_subtype)
+        for message in chat.get_user_messages(user_id):
+            timestamp = int(message.split('/', 2)[1])
+            if start <= timestamp <= end:
+                timestamp = datetime.utcfromtimestamp(timestamp)
+                date = timestamp.strftime('%Y-%m-%d')
+                if date not in nb_year:
+                    nb_year[date] = 0
+                nb_year[date] += 1
+                nb_max = max(nb_max, nb_year[date])
+
+    return nb_max, nb_year
+
+
 def get_user_account_usernames_timeline(subtype, user_id):
     user_account = UsersAccount.UserAccount(user_id, subtype)
     usernames = user_account.get_usernames_history()
@@ -904,7 +927,7 @@ def api_get_user_account(user_id, instance_uuid, translation_target=None):
     user_account = UsersAccount.UserAccount(user_id, instance_uuid)
     if not user_account.exists():
         return {"status": "error", "reason": "Unknown user-account"}, 404
-    meta = user_account.get_meta({'chats', 'icon', 'info', 'subchannels', 'threads', 'translation', 'username', 'usernames', 'username_meta'}, translation_target=translation_target)
+    meta = user_account.get_meta({'chats', 'icon', 'info', 'subchannels', 'threads', 'translation', 'username', 'usernames', 'username_meta', 'years'}, translation_target=translation_target)
     if meta['chats']:
         meta['chats'] = get_user_account_chats_meta(user_id, meta['chats'], meta['subchannels'])
     return meta, 200
@@ -928,6 +951,18 @@ def api_get_user_account_nb_all_week_messages(user_id, instance_uuid):
     week = get_user_account_nb_all_week_messages(user_account.id, user_account.get_chats(), user_account.get_chat_subchannels())
     return week, 200
 
+def api_get_user_account_nb_year_messages(user_id, instance_uuid, year):
+    user_account = UsersAccount.UserAccount(user_id, instance_uuid)
+    if not user_account.exists():
+        return {"status": "error", "reason": "Unknown user-account"}, 404
+    try:
+        year = int(year)
+    except (TypeError, ValueError):
+        year = datetime.now().year
+    nb_max, nb = get_user_account_nb_year_messages(user_account.id, user_account.get_chats(), year)
+    nb = [[date, value] for date, value in nb.items()]
+    return {'max': nb_max, 'nb': nb, 'year': year}, 200
+
 def api_chat_messages(subtype, chat_id):
     chat = Chats.Chat(chat_id, subtype)
     if not chat.exists():

bin/lib/objects/UsersAccount.py

@@ -5,7 +5,6 @@ import os
 import sys
 # import re
 
-# from datetime import datetime
 from flask import url_for
 from pymisp import MISPObject
 
@@ -73,6 +72,11 @@ class UserAccount(AbstractSubtypeObject):
         elif last_name:
             return last_name
 
+    def get_years(self):
+        year_start = int(self.get_first_seen()[0:4])
+        year_end = int(self.get_last_seen()[0:4])
+        return list(range(year_start, year_end + 1))
+
     def get_phone(self):
         return self._get_field('phone')
 
@@ -189,6 +193,8 @@ class UserAccount(AbstractSubtypeObject):
             meta['subchannels'] = self.get_chat_subchannels()
         if 'threads' in options:
             meta['threads'] = self.get_chat_threads()
+        if 'years' in options:
+            meta['years'] = self.get_years()
         return meta
 
     def get_misp_object(self):
@@ -258,8 +264,8 @@ class UserAccounts(AbstractSubtypeObjects):
         return name_to_search
 
 
-if __name__ == '__main__':
-    from lib.objects import Chats
-    chat = Chats.Chat('', '00098785-7e70-5d12-a120-c5cdc1252b2b')
-    account = UserAccount('', '00098785-7e70-5d12-a120-c5cdc1252b2b')
-    print(account.get_messages_by_chat_obj(chat))
+# if __name__ == '__main__':
+#     from lib.objects import Chats
+#     chat = Chats.Chat('', '00098785-7e70-5d12-a120-c5cdc1252b2b')
+#     account = UserAccount('', '00098785-7e70-5d12-a120-c5cdc1252b2b')
+#     print(account.get_messages_by_chat_obj(chat))

var/www/blueprints/chats_explorer.py

@@ -408,5 +408,15 @@ def user_account_messages_stats_week_all():
     else:
         return jsonify(week[0])
 
-
-
+@chats_explorer.route("objects/user-account/messages/stats/year", methods=['GET'])
+@login_required
+@login_read_only
+def user_account_messages_stats_year():
+    instance_uuid = request.args.get('subtype')
+    user_id = request.args.get('id')
+    year = request.args.get('year')
+    stats = chats_viewer.api_get_user_account_nb_year_messages(user_id, instance_uuid, year)
+    if stats[1] != 200:
+        return create_json_response(stats[0], stats[1])
+    else:
+        return jsonify(stats[0])

var/www/templates/chats_explorer/chat_viewer.html

@@ -244,7 +244,6 @@ optionheatmap = {
   tooltip: {
     position: 'top',
     formatter: function (p) {
-      //const format = echarts.time.format(p.data[0], '{yyyy}-{MM}-{dd}', false);
       return p.data[0] + ': ' + p.data[1];
     }
   },

var/www/templates/chats_explorer/user_account.html

@@ -21,6 +21,7 @@
     <script src="{{ url_for('static', filename='js/d3/heatmap_week_hour.js')}}"></script>
     <script src="{{ url_for('static', filename='js/d3/chord_directed_diagram.js')}}"></script>
     <script src="{{ url_for('static', filename='js/d3/timeline_basic.js')}}"></script>
+    <script src="{{ url_for('static', filename='js/echarts.min.js')}}"></script>
 
 
 </head>
@@ -48,6 +49,20 @@
                     <h4 class="mx-5 mt-2 text-secondary">User All Messages:</h4>
                     <div id="heatmapweekhourall"></div>
 
+                    <h5>Messages by year:</h5>
+                    <div>
+                        <div class="row">
+                            <div class="col-12 col-lg-11">
+                                <div id="heatmapyear" style="width: 100%;height: 300px;"></div>
+                            </div>
+                            <div class="col-12 col-lg-1">
+                                {% for year in meta['years'] %}
+                                    <div><button class="btn btn-info mt-1" onclick="update_heatmap_year({{ year }})">{{ year }}</button></div>
+                                {% endfor %}
+                            </div>
+                        </div>
+                    </div>
+
                     <h4>Usernames:</h4>
                     <div id="timeline_user_usernames" style="max-width: 900px"></div>
 
@@ -111,6 +126,58 @@ d3.json(url2).then(function(data) {
     create_directed_chord_diagram('#chord_mentions', data, 0, -1, mouseover_tooltip_ail_obj, mouseout_tooltip_ail_obj);
 });
 
+
+var heatyearChart = echarts.init(document.getElementById('heatmapyear'));
+window.addEventListener('resize', function() {
+    heatyearChart.resize();
+});
+var optionheatmap;
+
+optionheatmap = {
+  tooltip: {
+    position: 'top',
+    formatter: function (p) {
+      return p.data[0] + ': ' + p.data[1];
+    }
+  },
+  visualMap: {
+    min: 0,
+    max: 100,
+    calculable: true,
+    orient: 'horizontal',
+    left: '500',
+    top: '-10'
+  },
+  calendar: [
+    {
+      orient: 'horizontal',
+      range: new Date().getFullYear(),
+    },
+  ],
+  series: [
+    {
+      type: 'heatmap',
+      coordinateSystem: 'calendar',
+      data: []
+    },
+
+  ]
+};
+heatyearChart.setOption(optionheatmap);
+
+update_heatmap_year(null);
+function update_heatmap_year(year) {
+    $.getJSON("{{ url_for('chats_explorer.user_account_messages_stats_year') }}?type=chat&subtype={{ meta['subtype'] }}&id={{ meta['id'] }}&year=" + year)
+        .done(function(data) {
+            optionheatmap['visualMap']['max'] = data['max']
+            optionheatmap['calendar'][0]['range'] = data['year']
+            optionheatmap['series'][0]['data'] = data['nb']
+            heatyearChart.setOption(optionheatmap)
+
+        }
+    );
+}
+
 </script>