Merge pull request #370 from CIRCL/api_v1

Api v1

.gitignore

@@ -31,6 +31,8 @@ var/www/static/
 !var/www/static/js/trendingchart.js
 var/www/templates/header.html
 var/www/submitted
+var/www/server.crt
+var/www/server.key
 
 # Local config
 bin/packages/config.cfg
@@ -40,6 +42,11 @@ configs/update.cfg
 update/current_version
 files
 
+# Helper
+bin/helper/gen_cert/rootCA.*
+bin/helper/gen_cert/server.*
+
+
 # Pystemon archives
 pystemon/archives
 

OVERVIEW.md

@@ -38,6 +38,21 @@ Redis and ARDB overview
 | failed_login_ip:**ip**           | **nb login failed** | TTL
 | failed_login_user_id:**user_id** | **nb login failed** | TTL
 
+##### Item Import:
+
+| Key | Value |
+| ------ | ------ |
+| **uuid**:nb_total   | **nb total**      | TTL *(if imported)*
+| **uuid**:nb_end     | **nb**            | TTL *(if imported)*
+| **uuid**:nb_sucess  | **nb success**    | TTL *(if imported)*
+| **uuid**:end        | **0 (in progress) or (item imported)** | TTL *(if imported)*
+| **uuid**:processing | **process status: 0 or 1**             | TTL *(if imported)*
+| **uuid**:error      | **error message** | TTL *(if imported)*
+
+| Set Key | Value |
+| ------ | ------ |
+| **uuid**:paste_submit_link | **item_path** | TTL *(if imported)*
+
 ## DB0 - Core:
 
 ##### Update keys:
@@ -82,6 +97,18 @@ Redis and ARDB overview
 | ------ | ------ | ------ |
 | ail:all_role | **role** | **int, role priority (1=admin)** |
 
+##### Item Import:
+| Key | Value |
+| ------ | ------ |
+| **uuid**:isfile   | **boolean** |
+| **uuid**:paste_content | **item_content** |
+
+| Set Key | Value |
+| ------ | ------ |
+| submitted:uuid | **uuid** |
+| **uuid**:ltags | **tag** |
+| **uuid**:ltagsgalaxies | **tag** |
+
 ## DB2 - TermFreq:
 
 ##### Set:

bin/LAUNCH.sh

@@ -66,8 +66,8 @@ function helptext {
     "$DEFAULT"
     This script launch:
     "$CYAN"
-    - All the ZMQ queuing modules.
+    - All the queuing modules.
-    - All the ZMQ processing modules.
+    - All the processing modules.
     - All Redis in memory servers.
     - All ARDB on disk servers.
     "$DEFAULT"
@@ -143,7 +143,7 @@ function launching_scripts {
 
     screen -dmS "Script_AIL"
     sleep 0.1
-    echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
+    echo -e $GREEN"\t* Launching scripts"$DEFAULT
 
     screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x"
     sleep 0.1

bin/packages/Date.py

@@ -40,3 +40,13 @@ class Date(object):
         comp_month = str(computed_date.month).zfill(2)
         comp_day = str(computed_date.day).zfill(2)
         return comp_year + comp_month + comp_day
+
+def date_add_day(date, num_day=1):
+    new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day)
+    new_date = str(new_date).replace('-', '')
+    return new_date
+
+def date_substract_day(date, num_day=1):
+    new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) - datetime.timedelta(num_day)
+    new_date = str(new_date).replace('-', '')
+    return new_date

bin/packages/Import_helper.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import uuid
+import redis
+
+import Flask_config
+
+r_serv_db = Flask_config.r_serv_db
+r_serv_log_submit = Flask_config.r_serv_log_submit
+
+def is_valid_uuid_v4(UUID):
+    UUID = UUID.replace('-', '')
+    try:
+        uuid_test = uuid.UUID(hex=UUID, version=4)
+        return uuid_test.hex == UUID
+    except:
+        return False
+
+def create_import_queue(tags, galaxy, paste_content, UUID,  password=None, isfile = False):
+
+    # save temp value on disk
+    for tag in tags:
+        r_serv_db.sadd(UUID + ':ltags', tag)
+    for tag in galaxy:
+        r_serv_db.sadd(UUID + ':ltagsgalaxies', tag)
+
+    r_serv_db.set(UUID + ':paste_content', paste_content)
+
+    if password:
+        r_serv_db.set(UUID + ':password', password)
+
+    r_serv_db.set(UUID + ':isfile', isfile)
+
+    r_serv_log_submit.set(UUID + ':end', 0)
+    r_serv_log_submit.set(UUID + ':processing', 0)
+    r_serv_log_submit.set(UUID + ':nb_total', -1)
+    r_serv_log_submit.set(UUID + ':nb_end', 0)
+    r_serv_log_submit.set(UUID + ':nb_sucess', 0)
+
+    # save UUID on disk
+    r_serv_db.sadd('submitted:uuid', UUID)
+    return UUID
+
+def check_import_status(UUID):
+    if not is_valid_uuid_v4(UUID):
+        return ({'status': 'error', 'reason': 'Invalid uuid'}, 400)
+
+    processing = r_serv_log_submit.get(UUID + ':processing')
+    if not processing:
+        return ({'status': 'error', 'reason': 'Unknown uuid'}, 404)
+
+    # nb_total = r_serv_log_submit.get(UUID + ':nb_total')
+    # nb_sucess = r_serv_log_submit.get(UUID + ':nb_sucess')
+    # nb_end = r_serv_log_submit.get(UUID + ':nb_end')
+    items_id = list(r_serv_log_submit.smembers(UUID + ':paste_submit_link'))
+    error = r_serv_log_submit.get(UUID + ':error')
+    end = r_serv_log_submit.get(UUID + ':end')
+
+    dict_import_status = {}
+    if items_id:
+        dict_import_status['items'] = items_id
+    if error:
+        dict_import_status['error'] = error
+
+    if processing == '0':
+        status = 'in queue'
+    else:
+        if end == '0':
+            status = 'in progress'
+        else:
+            status = 'imported'
+    dict_import_status['status'] = status
+
+    return (dict_import_status, 200)

bin/packages/Item.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import gzip
+import redis
+
+import Flask_config
+import Date
+import Tag
+
+PASTES_FOLDER = Flask_config.PASTES_FOLDER
+r_cache = Flask_config.r_cache
+
+def exist_item(item_id):
+    if os.path.isfile(os.path.join(PASTES_FOLDER, item_id)):
+        return True
+    else:
+        return False
+
+def get_item_date(item_id):
+    l_directory = item_id.split('/')
+    return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
+
+def get_item_size(item_id):
+    return round(os.path.getsize(os.path.join(PASTES_FOLDER, item_id))/1024.0, 2)
+
+def get_lines_info(item_id, item_content=None):
+    if not item_content:
+        item_content = get_item_content(item_id)
+    max_length = 0
+    line_id = 0
+    nb_line = 0
+    for line in item_content.splitlines():
+        length = len(line)
+        if length > max_length:
+            max_length = length
+        nb_line += 1
+    return {'nb': nb_line, 'max_length': max_length}
+
+
+def get_item_content(item_id):
+    item_full_path = os.path.join(PASTES_FOLDER, item_id)
+    try:
+        item_content = r_cache.get(item_full_path)
+    except UnicodeDecodeError:
+        item_content = None
+    except Exception as e:
+        item_content = None
+    if item_content is None:
+        try:
+            with gzip.open(item_full_path, 'r') as f:
+                item_content = f.read().decode()
+                r_cache.set(item_full_path, item_content)
+                r_cache.expire(item_full_path, 300)
+        except:
+            item_content = ''
+    return str(item_content)
+
+# API
+def get_item(request_dict):
+    if not request_dict:
+        return Response({'status': 'error', 'reason': 'Malformed JSON'}, 400)
+
+    item_id = request_dict.get('id', None)
+    if not item_id:
+        return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 )
+    if not exist_item(item_id):
+        return ( {'status': 'error', 'reason': 'Item not found'}, 404 )
+
+    dict_item = {}
+    dict_item['id'] = item_id
+    date = request_dict.get('date', True)
+    if date:
+        dict_item['date'] = get_item_date(item_id)
+    tags = request_dict.get('tags', True)
+    if tags:
+        dict_item['tags'] = Tag.get_item_tags(item_id)
+
+    size = request_dict.get('size', False)
+    if size:
+        dict_item['size'] = get_item_size(item_id)
+
+    content = request_dict.get('content', False)
+    if content:
+        # UTF-8 outpout, # TODO: use base64
+        dict_item['content'] = get_item_content(item_id)
+
+    lines_info = request_dict.get('lines', False)
+    if lines_info:
+        dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None'))
+
+    return (dict_item, 200)

bin/packages/Paste.py

@@ -115,6 +115,17 @@ class Paste(object):
         self.p_duplicate = None
         self.p_tags = None
 
+    def get_item_dict(self):
+        dict_item = {}
+        dict_item['id'] = self.p_rel_path
+        dict_item['date'] = str(self.p_date)
+        dict_item['content'] = self.get_p_content()
+        tags = self._get_p_tags()
+        if tags:
+            dict_item['tags'] = tags
+        return dict_item
+
+
     def get_p_content(self):
         """
         Returning the content of the Paste
@@ -321,8 +332,8 @@ class Paste(object):
         return self.store_metadata.scard('dup:'+self.p_path) + self.store_metadata.scard('dup:'+self.p_rel_path)
 
     def _get_p_tags(self):
-        self.p_tags = self.store_metadata.smembers('tag:'+path, tag)
+        self.p_tags = self.store_metadata.smembers('tag:'+self.p_rel_path)
-        if self.self.p_tags is not None:
+        if self.p_tags is not None:
             return list(self.p_tags)
         else:
             return '[]'

bin/packages/Tag.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import redis
+
+import Flask_config
+import Date
+import Item
+
+from pytaxonomies import Taxonomies
+from pymispgalaxies import Galaxies, Clusters
+
+r_serv_tags = Flask_config.r_serv_tags
+r_serv_metadata = Flask_config.r_serv_metadata
+
+def get_taxonomie_from_tag(tag):
+    return tag.split(':')[0]
+
+def get_galaxy_from_tag(tag):
+    galaxy = tag.split(':')[1]
+    galaxy = galaxy.split('=')[0]
+    return galaxy
+
+def get_active_taxonomies():
+    return r_serv_tags.smembers('active_taxonomies')
+
+def get_active_galaxies():
+    return r_serv_tags.smembers('active_galaxies')
+
+def is_taxonomie_tag_enabled(taxonomie, tag):
+    if tag in r_serv_tags.smembers('active_tag_' + taxonomie):
+        return True
+    else:
+        return False
+
+def is_galaxy_tag_enabled(galaxy, tag):
+    if tag in r_serv_tags.smembers('active_tag_galaxies_' + galaxy):
+        return True
+    else:
+        return False
+
+# Check if tags are enabled in AIL
+def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy):
+    if list_tags:
+        active_taxonomies = get_active_taxonomies()
+
+        for tag in list_tags:
+            taxonomie = get_taxonomie_from_tag(tag)
+            if taxonomie not in active_taxonomies:
+                return False
+            if not is_taxonomie_tag_enabled(taxonomie, tag):
+                return False
+
+    if list_tags_galaxy:
+        active_galaxies = get_active_galaxies()
+
+        for tag in list_tags_galaxy:
+            galaxy = get_galaxy_from_tag(tag)
+            if galaxy not in active_galaxies:
+                return False
+            if not is_galaxy_tag_enabled(galaxy, tag):
+                return False
+    return True
+
+def get_tag_metadata(tag):
+    first_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'first_seen')
+    last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
+    return {'tag': tag, 'first_seen': first_seen, 'last_seen': last_seen}
+
+def is_tag_in_all_tag(tag):
+    if r_serv_tags.sismember('list_tags', tag):
+        return True
+    else:
+        return False
+
+def get_all_tags():
+    return list(r_serv_tags.smembers('list_tags'))
+
+def get_item_tags(item_id):
+    tags = r_serv_metadata.smembers('tag:'+item_id)
+    if tags:
+        return list(tags)
+    else:
+        return '[]'
+
+# TEMPLATE + API QUERY
+def add_items_tag(tags=[], galaxy_tags=[], item_id=None):
+    res_dict = {}
+    if item_id == None:
+        return ({'status': 'error', 'reason': 'Item id not found'}, 404)
+    if not tags and not galaxy_tags:
+        return ({'status': 'error', 'reason': 'Tags or Galaxy not specified'}, 400)
+
+    res_dict['tags'] = []
+    for tag in tags:
+        taxonomie = get_taxonomie_from_tag(tag)
+        if is_taxonomie_tag_enabled(taxonomie, tag):
+            add_item_tag(tag, item_id)
+            res_dict['tags'].append(tag)
+        else:
+            return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400)
+
+    for tag in galaxy_tags:
+        galaxy = get_galaxy_from_tag(tag)
+        if is_galaxy_tag_enabled(galaxy, tag):
+            add_item_tag(tag, item_id)
+            res_dict['tags'].append(tag)
+        else:
+            return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400)
+
+    res_dict['id'] = item_id
+    return (res_dict, 200)
+
+
+def add_item_tag(tag, item_path):
+
+    item_date = int(Item.get_item_date(item_path))
+
+    #add tag
+    r_serv_metadata.sadd('tag:{}'.format(item_path), tag)
+    r_serv_tags.sadd('{}:{}'.format(tag, item_date), item_path)
+
+    r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, 1)
+
+    tag_first_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
+    if tag_first_seen is None:
+        tag_first_seen = 99999999
+    else:
+        tag_first_seen = int(tag_first_seen)
+    tag_last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
+    if tag_last_seen is None:
+        tag_last_seen = 0
+    else:
+        tag_last_seen = int(tag_last_seen)
+
+    #add new tag in list of all used tags
+    r_serv_tags.sadd('list_tags', tag)
+
+    # update fisrt_seen/last_seen
+    if item_date < tag_first_seen:
+        r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', item_date)
+
+    # update metadata last_seen
+    if item_date > tag_last_seen:
+        r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', item_date)
+
+# API QUERY
+def remove_item_tags(tags=[], item_id=None):
+    if item_id == None:
+        return ({'status': 'error', 'reason': 'Item id not found'}, 404)
+    if not tags:
+        return ({'status': 'error', 'reason': 'No Tag(s) specified'}, 400)
+
+    dict_res = {}
+    dict_res['tags'] = []
+    for tag in tags:
+        res = remove_item_tag(tag, item_id)
+        if res[1] != 200:
+            return res
+        else:
+            dict_res['tags'].append(tag)
+    dict_res['id'] = item_id
+    return (dict_res, 200)
+
+# TEMPLATE + API QUERY
+def remove_item_tag(tag, item_id):
+    item_date = int(Item.get_item_date(item_id))
+
+    #remove tag
+    r_serv_metadata.srem('tag:{}'.format(item_id), tag)
+    res = r_serv_tags.srem('{}:{}'.format(tag, item_date), item_id)
+
+    if res ==1:
+        # no tag for this day
+        if int(r_serv_tags.hget('daily_tags:{}'.format(item_date), tag)) == 1:
+            r_serv_tags.hdel('daily_tags:{}'.format(item_date), tag)
+        else:
+            r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, -1)
+
+        tag_first_seen = int(r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen'))
+        tag_last_seen = int(r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen'))
+        # update fisrt_seen/last_seen
+        if item_date == tag_first_seen:
+            update_tag_first_seen(tag, tag_first_seen, tag_last_seen)
+        if item_date == tag_last_seen:
+            update_tag_last_seen(tag, tag_first_seen, tag_last_seen)
+        return ({'status': 'success'}, 200)
+    else:
+        return ({'status': 'error', 'reason': 'Item id or tag not found'}, 400)
+
+def update_tag_first_seen(tag, tag_first_seen, tag_last_seen):
+    if tag_first_seen == tag_last_seen:
+        if r_serv_tags.scard('{}:{}'.format(tag, tag_first_seen)) > 0:
+            r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_first_seen)
+        # no tag in db
+        else:
+            r_serv_tags.srem('list_tags', tag)
+            r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'first_seen')
+            r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'last_seen')
+    else:
+        if r_serv_tags.scard('{}:{}'.format(tag, tag_first_seen)) > 0:
+            r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_first_seen)
+        else:
+            tag_first_seen = Date.date_add_day(tag_first_seen)
+            update_tag_first_seen(tag, tag_first_seen, tag_last_seen)
+
+def update_tag_last_seen(tag, tag_first_seen, tag_last_seen):
+    if tag_first_seen == tag_last_seen:
+        if r_serv_tags.scard('{}:{}'.format(tag, tag_last_seen)) > 0:
+            r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_last_seen)
+        # no tag in db
+        else:
+            r_serv_tags.srem('list_tags', tag)
+            r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'first_seen')
+            r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'last_seen')
+    else:
+        if r_serv_tags.scard('{}:{}'.format(tag, tag_last_seen)) > 0:
+            r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_last_seen)
+        else:
+            tag_last_seen = Date.date_substract_day(tag_last_seen)
+            update_tag_last_seen(tag, tag_first_seen, tag_last_seen)

bin/submit_paste.py

@@ -47,7 +47,11 @@ def create_paste(uuid, paste_content, ltags, ltagsgalaxies, name):
     r_serv_log_submit.hincrby("mixer_cache:list_feeder", "submitted", 1)
 
     # add tags
-    add_tags(ltags, ltagsgalaxies, rel_item_path)
+    for tag in ltags:
+        add_item_tag(tag, rel_item_path)
+
+    for tag in ltagsgalaxies:
+        add_item_tag(tag, rel_item_path)
 
     r_serv_log_submit.incr(uuid + ':nb_end')
     r_serv_log_submit.incr(uuid + ':nb_sucess')
@@ -92,7 +96,6 @@ def remove_submit_uuid(uuid):
     r_serv_log_submit.expire(uuid + ':nb_sucess', expire_time)
     r_serv_log_submit.expire(uuid + ':nb_end', expire_time)
     r_serv_log_submit.expire(uuid + ':error', expire_time)
-    r_serv_log_submit.srem(uuid + ':paste_submit_link', '')
     r_serv_log_submit.expire(uuid + ':paste_submit_link', expire_time)
 
     # delete uuid
@@ -134,18 +137,6 @@ def add_item_tag(tag, item_path):
     if item_date > tag_last_seen:
         r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', item_date)
 
-def add_tags(tags, tagsgalaxies, path):
-    list_tag = tags.split(',')
-    list_tag_galaxies = tagsgalaxies.split(',')
-
-    if list_tag != ['']:
-        for tag in list_tag:
-            add_item_tag(tag, path)
-
-    if list_tag_galaxies != ['']:
-        for tag in list_tag_galaxies:
-            add_item_tag(tag, path)
-
 def verify_extention_filename(filename):
     if not '.' in filename:
         return True
@@ -218,8 +209,8 @@ if __name__ == "__main__":
             uuid = r_serv_db.srandmember('submitted:uuid')
 
             # get temp value save on disk
-            ltags = r_serv_db.get(uuid + ':ltags')
+            ltags = r_serv_db.smembers(uuid + ':ltags')
-            ltagsgalaxies = r_serv_db.get(uuid + ':ltagsgalaxies')
+            ltagsgalaxies = r_serv_db.smembers(uuid + ':ltagsgalaxies')
             paste_content = r_serv_db.get(uuid + ':paste_content')
             isfile = r_serv_db.get(uuid + ':isfile')
             password = r_serv_db.get(uuid + ':password')
@@ -230,8 +221,6 @@ if __name__ == "__main__":
             r_serv_log_submit.set(uuid + ':nb_total', -1)
             r_serv_log_submit.set(uuid + ':nb_end', 0)
             r_serv_log_submit.set(uuid + ':nb_sucess', 0)
-            r_serv_log_submit.set(uuid + ':error', 'error:')
-            r_serv_log_submit.sadd(uuid + ':paste_submit_link', '')
 
 
             r_serv_log_submit.set(uuid + ':processing', 1)
@@ -275,7 +264,7 @@ if __name__ == "__main__":
                         else:
                             #decompress file
                             try:
-                                if password == '':
+                                if password == None:
                                     files = unpack(file_full_path.encode())
                                     #print(files.children)
                                 else:

doc/README.md

@@ -0,0 +1,806 @@
+#  API DOCUMENTATION
+
+## General
+
+### Automation key
+
+The authentication of the automation is performed via a secure key available in the AIL UI interface. Make sure you keep that key secret. It gives access to the entire database! The API key is available in the ``Server Management`` menu under ``My Profile``.
+
+The authorization is performed by using the following header:
+
+~~~~
+Authorization: YOUR_API_KEY
+~~~~
+### Accept and Content-Type headers
+
+When submitting data in a POST, PUT or DELETE operation you need to specify in what content-type you encoded the payload. This is done by setting the below Content-Type headers:
+
+~~~~
+Content-Type: application/json
+~~~~
+
+Example:
+
+~~~~
+curl --header "Authorization: YOUR_API_KEY" --header "Content-Type: application/json" https://AIL_URL/
+~~~~
+
+## Item management
+
+### Get item: `api/v1/get/item/default`<a name="get_item_default"></a>
+
+#### Description
+Get item default info.
+
+**Method** : `POST`
+
+#### Parameters
+- `id`
+  - item id
+  - *str - relative item path*
+  - mandatory
+
+#### JSON response
+- `content`
+  - item content
+  - *str*
+- `id`
+  - item id
+  - *str*
+- `date`
+  - item date
+  - *str - YYMMDD*
+- `tags`
+  - item tags list
+  - *list*
+
+#### Example
+```
+curl https://127.0.0.1:7000/api/v1/get/item/default --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
+```
+
+#### input.json Example
+```json
+  {
+    "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz"
+  }
+```
+
+#### Expected Success Response
+**HTTP Status Code** : `200`
+
+```json
+  {
+    "content": "item content test",
+    "date": "20190726",
+    "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
+    "tags":
+      [
+        "misp-galaxy:backdoor=\"Rosenbridge\"",
+        "infoleak:automatic-detection=\"pgp-message\"",
+        "infoleak:automatic-detection=\"encrypted-private-key\"",
+        "infoleak:submission=\"manual\"",
+        "misp-galaxy:backdoor=\"SLUB\""
+      ]
+  }
+```
+
+#### Expected Fail Response
+
+**HTTP Status Code** : `400`
+```json
+  {"status": "error", "reason": "Mandatory parameter(s) not provided"}
+```
+**HTTP Status Code** : `404`
+```json
+  {"status": "error", "reason": "Item not found"}
+```
+
+
+
+
+### Get item content: `api/v1/get/item/content`<a name="get_item_content"></a>
+
+#### Description
+Get a specific item content.
+
+**Method** : `POST`
+
+#### Parameters
+- `id`
+  - item id
+  - *str - relative item path*
+  - mandatory
+
+#### JSON response
+- `content`
+  - item content
+  - *str*
+- `id`
+  - item id
+  - *str*
+
+#### Example
+```
+curl https://127.0.0.1:7000/api/v1/get/item/content --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
+```
+
+#### input.json Example
+```json
+  {
+    "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz"
+  }
+```
+
+#### Expected Success Response
+**HTTP Status Code** : `200`
+
+```json
+  {
+    "content": "item content test",
+    "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz"
+  }
+```
+
+#### Expected Fail Response
+
+**HTTP Status Code** : `400`
+```json
+  {"status": "error", "reason": "Mandatory parameter(s) not provided"}
+```
+**HTTP Status Code** : `404`
+```json
+  {"status": "error", "reason": "Item not found"}
+```
+
+
+
+### Get item content: `api/v1/get/item/tag`<a name="get_item_tag"></a>
+
+#### Description
+Get all tags from an item.
+
+**Method** : `POST`
+
+#### Parameters
+- `id`
+  - item id
+  - *str - relative item path*
+  - mandatory
+
+#### JSON response
+- `content`
+  - item content
+  - *str*
+- `tags`
+  - item tags list
+  - *list*
+
+#### Example
+```
+curl https://127.0.0.1:7000/api/v1/get/item/tag --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
+```
+
+#### input.json Example
+```json
+  {
+    "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz"
+  }
+```
+
+#### Expected Success Response
+**HTTP Status Code** : `200`
+
+```json
+  {
+    "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
+    "tags":
+      [
+        "misp-galaxy:backdoor=\"Rosenbridge\"",
+        "infoleak:automatic-detection=\"pgp-message\"",
+        "infoleak:automatic-detection=\"encrypted-private-key\"",
+        "infoleak:submission=\"manual\"",
+        "misp-galaxy:backdoor=\"SLUB\""
+      ]
+  }
+```
+
+#### Expected Fail Response
+
+**HTTP Status Code** : `400`
+```json
+  {"status": "error", "reason": "Mandatory parameter(s) not provided"}
+```
+**HTTP Status Code** : `404`
+```json
+  {"status": "error", "reason": "Item not found"}
+```
+
+
+
+### Advanced Get item: `api/v1/get/item`<a name="get_item"></a>
+
+#### Description
+Get item. Filter requested field.
+
+**Method** : `POST`
+
+#### Parameters
+- `id`
+  - item id
+  - *str - relative item path*
+  - mandatory
+- `date`
+  - get item date
+  - *boolean*
+  - default: `true`
+- `tags`
+  - get item tags
+  - *boolean*
+  - default: `true`
+- `content`
+  - get item content
+  - *boolean*
+  - default: `false`
+- `size`
+  - get item size
+  - *boolean*
+  - default: `false`
+- `lines`
+  - get item lines info
+  - *boolean*
+  - default: `false`
+
+#### JSON response
+- `content`
+  - item content
+  - *str*
+- `id`
+  - item id
+  - *str*
+- `date`
+  - item date
+  - *str - YYMMDD*
+- `tags`
+  - item tags list
+  - *list*
+- `size`
+  - item size (Kb)
+  - *int*
+- `lines`
+  - item lines info
+  - *{}*
+      - `max_length`
+        -  line max length line
+        - *int*
+      - `nb`
+        - nb lines item
+        - *int*
+
+
+#### Example
+```
+curl https://127.0.0.1:7000/api/v1/get/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
+```
+
+#### input.json Example
+```json
+{
+  "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
+  "content": true,
+  "lines_info": true,
+  "tags": true,
+  "size": true
+}
+```
+
+#### Expected Success Response
+**HTTP Status Code** : `200`
+```json
+  {
+    "content": "dsvcdsvcdsc vvvv",
+    "date": "20190726",
+    "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
+    "lines": {
+      "max_length": 19,
+      "nb": 1
+    },
+    "size": 0.03,
+    "tags": [
+      "misp-galaxy:stealer=\"Vidar\"",
+      "infoleak:submission=\"manual\""
+    ]
+  }
+```
+
+#### Expected Fail Response
+**HTTP Status Code** : `400`
+```json
+  {"status": "error", "reason": "Mandatory parameter(s) not provided"}
+```
+**HTTP Status Code** : `404`
+```json
+  {"status": "error", "reason": "Item not found"}
+```
+
+
+
+
+
+### Add item tags: `api/v1/add/item/tag`<a name="add_item_tag"></a>
+
+#### Description
+Add tags to an item.
+
+**Method** : `POST`
+
+#### Parameters
+- `id`
+  - item id
+  - *str - relative item path*
+  - mandatory
+- `tags`
+  - list of tags
+  - *list*
+  - default: `[]`
+- `galaxy`
+  - list of galaxy
+  - *list*
+  - default: `[]`
+
+#### JSON response
+- `id`
+  - item id
+  - *str - relative item path*
+- `tags`
+  - list of item tags added
+  - *list*
+
+#### Example
+```
+curl https://127.0.0.1:7000/api/v1/import/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
+```
+
+#### input.json Example
+```json
+  {
+    "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
+    "tags": [
+      "infoleak:analyst-detection=\"private-key\"",
+      "infoleak:analyst-detection=\"api-key\""
+    ],
+    "galaxy": [
+      "misp-galaxy:stealer=\"Vidar\""
+    ]
+  }
+```
+
+#### Expected Success Response
+**HTTP Status Code** : `200`
+
+```json
+  {
+    "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
+    "tags": [
+      "infoleak:analyst-detection=\"private-key\"",
+      "infoleak:analyst-detection=\"api-key\"",
+      "misp-galaxy:stealer=\"Vidar\""
+    ]
+  }
+```
+
+#### Expected Fail Response
+**HTTP Status Code** : `400`
+
+```json
+  {"status": "error", "reason": "Item id not found"}
+  {"status": "error", "reason": "Tags or Galaxy not specified"}
+  {"status": "error", "reason": "Tags or Galaxy not enabled"}
+```
+
+
+
+
+### Delete item tags: `api/v1/delete/item/tag`<a name="delete_item_tag"></a>
+
+#### Description
+Delete tags from an item.
+
+**Method** : `DELETE`
+
+#### Parameters
+- `id`
+  - item id
+  - *str - relative item path*
+  - mandatory
+- `tags`
+  - list of tags
+  - *list*
+  - default: `[]`
+
+#### JSON response
+- `id`
+  - item id
+  - *str - relative item path*
+- `tags`
+  - list of item tags deleted
+  - *list*
+
+#### Example
+```
+curl https://127.0.0.1:7000/api/v1/delete/item/tag --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X DELETE
+```
+
+#### input.json Example
+```json
+  {
+    "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
+    "tags": [
+      "infoleak:analyst-detection=\"private-key\"",
+      "infoleak:analyst-detection=\"api-key\"",
+      "misp-galaxy:stealer=\"Vidar\""
+    ]
+  }
+```
+
+#### Expected Success Response
+**HTTP Status Code** : `200`
+
+```json
+  {
+    "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
+    "tags": [
+      "infoleak:analyst-detection=\"private-key\"",
+      "infoleak:analyst-detection=\"api-key\"",
+      "misp-galaxy:stealer=\"Vidar\""
+    ]
+  }
+```
+
+#### Expected Fail Response
+**HTTP Status Code** : `400`
+
+```json
+  {"status": "error", "reason": "Item id not found"}
+  {"status": "error", "reason": "No Tag(s) specified"}
+```
+
+
+
+
+
+
+## Tag management
+
+
+### Get all AIL tags: `api/v1/get/tag/all`<a name="get_tag_all"></a>
+
+#### Description
+Get all tags used in AIL.
+
+**Method** : `GET`
+
+#### JSON response
+- `tags`
+  - list of tag
+  - *list*
+#### Example
+```
+curl https://127.0.0.1:7000/api/v1/get/tag/all --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
+```
+
+#### Expected Success Response
+**HTTP Status Code** : `200`
+```json
+  {
+    "tags": [
+      "misp-galaxy:backdoor=\"Rosenbridge\"",
+      "infoleak:automatic-detection=\"pgp-private-key\"",
+      "infoleak:automatic-detection=\"pgp-signature\"",
+      "infoleak:automatic-detection=\"base64\"",
+      "infoleak:automatic-detection=\"encrypted-private-key\"",
+      "infoleak:submission=\"crawler\"",
+      "infoleak:automatic-detection=\"binary\"",
+      "infoleak:automatic-detection=\"pgp-public-key-block\"",
+      "infoleak:automatic-detection=\"hexadecimal\"",
+      "infoleak:analyst-detection=\"private-key\"",
+      "infoleak:submission=\"manual\"",
+      "infoleak:automatic-detection=\"private-ssh-key\"",
+      "infoleak:automatic-detection=\"iban\"",
+      "infoleak:automatic-detection=\"pgp-message\"",
+      "infoleak:automatic-detection=\"certificate\"",
+      "infoleak:automatic-detection=\"credential\"",
+      "infoleak:automatic-detection=\"cve\"",
+      "infoleak:automatic-detection=\"google-api-key\"",
+      "infoleak:automatic-detection=\"phone-number\"",
+      "infoleak:automatic-detection=\"rsa-private-key\"",
+      "misp-galaxy:backdoor=\"SLUB\"",
+      "infoleak:automatic-detection=\"credit-card\"",
+      "misp-galaxy:stealer=\"Vidar\"",
+      "infoleak:automatic-detection=\"private-key\"",
+      "infoleak:automatic-detection=\"api-key\"",
+      "infoleak:automatic-detection=\"mail\""
+    ]
+  }
+```
+
+
+
+
+### Get tag metadata: `api/v1/get/tag/metadata`<a name="get_tag_metadata"></a>
+
+#### Description
+Get tag metadata.
+
+**Method** : `POST`
+
+#### Parameters
+- `tag`
+  - tag name
+  - *str*
+  - mandatory
+
+#### JSON response
+- `tag`
+  - tag name
+  - *str*
+- `first_seen`
+  - date: first seen
+  - *str - YYYYMMDD*
+- `last_seen`
+  - date: last seen
+  - *str - YYYYMMDD*
+#### Example
+```
+curl https://127.0.0.1:7000/api/v1/get/tag/metadata --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
+```
+
+#### input.json Example
+```json
+  {
+    "tag": "infoleak:submission=\"manual\""
+  }
+```
+
+#### Expected Success Response
+**HTTP Status Code** : `200`
+```json
+  {
+    "first_seen": "20190605",
+    "last_seen": "20190726",
+    "tag": "infoleak:submission=\"manual\""
+  }
+```
+
+#### Expected Fail Response
+**HTTP Status Code** : `404`
+```json
+  {"status": "error", "reason": "Tag not found"}
+```
+
+
+
+
+
+
+## Import management
+
+
+
+### Import item (currently: text only): `api/v1/import/item`<a name="import_item"></a>
+
+#### Description
+Allows users to import new items. asynchronous function.
+
+**Method** : `POST`
+
+#### Parameters
+- `type`
+  - import type
+  - *str*
+  - default: `text`
+- `text`
+  - text to import
+  - *str*
+  - mandatory if type = text
+- `default_tags`
+  - add default import tag
+  - *boolean*
+  - default: True
+- `tags`
+  - list of tags
+  - *list*
+  - default: `[]`
+- `galaxy`
+  - list of galaxy
+  - *list*
+  - default: `[]`
+
+#### JSON response
+- `uuid`
+  - import uuid
+  - *uuid4*
+
+#### Example
+```
+curl https://127.0.0.1:7000/api/v1/import/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
+```
+
+#### input.json Example
+```json
+  {
+    "type": "text",
+    "tags": [
+      "infoleak:analyst-detection=\"private-key\""
+    ],
+    "text": "text to import"
+  }
+```
+
+#### Expected Success Response
+**HTTP Status Code** : `200`
+
+```json
+  {
+    "uuid": "0c3d7b34-936e-4f01-9cdf-2070184b6016"
+  }
+```
+
+#### Expected Fail Response
+**HTTP Status Code** : `400`
+
+```json
+  {"status": "error", "reason": "Malformed JSON"}
+  {"status": "error", "reason": "No text supplied"}
+  {"status": "error", "reason": "Tags or Galaxy not enabled"}
+  {"status": "error", "reason": "Size exceeds default"}
+```
+
+
+
+
+
+### GET Import item info: `api/v1/get/import/item/`<a name="get_import_item"></a>
+
+#### Description
+
+Get import status and all items imported by uuid
+
+**Method** : `POST`
+
+#### Parameters
+
+- `uuid`
+  - import uuid
+  - *uuid4*
+  - mandatory
+
+#### JSON response
+
+- `status`
+  - import status
+  - *str*
+  - values: `in queue`, `in progress`, `imported`
+- `items`
+  - list of imported items id
+  - *list*
+  - The full list of imported items is not complete until `status` = `"imported"`
+
+#### Example
+
+```
+curl -k https://127.0.0.1:7000/api/v1/get/import/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
+```
+
+#### input.json Example
+```json
+  {
+    "uuid": "0c3d7b34-936e-4f01-9cdf-2070184b6016"
+  }
+```
+
+#### Expected Success Response
+
+**HTTP Status Code** : `200`
+
+```json
+  {
+    "items": [
+      "submitted/2019/07/26/b20a69f1-99ad-4cb3-b212-7ce24b763b50.gz"
+    ],
+    "status": "imported"
+  }
+```
+
+#### Expected Fail Response
+
+**HTTP Status Code** : `400`
+
+```json
+  {"status": "error", "reason": "Invalid uuid"}
+  {"status": "error", "reason": "Unknown uuid"}
+```
+
+
+
+
+
+
+
+
+# FUTURE endpoints
+
+### Text search by daterange
+##### ``api/search/textIndexer/item`` POST
+
+### Get tagged items by daterange
+##### ``api/search/tag/item`` POST
+
+### Submit a domain to crawl
+##### ``api/add/crawler/domain`` POST
+
+### Create a term/set/regex tracker
+##### ``api/add/termTracker/`` POST
+
+### Get tracker items list
+##### ``api/get/termTracker/item`` POST
+
+-----
+
+### Check if a tor/regular domain have been crawled
+##### ``api/get/crawler/domain/`` POST
+
+### Check if a tor/regular domain have been crawled
+##### ``api/get/crawler/domain/metadata/ <domain><port>`` POST
+
+### Get domain tags
+##### ``api/get/crawler/domain/tag/ <domain><port>`` POST
+
+### Get domain history
+##### ``api/get/crawler/domain/history/ <domain><port>`` POST
+
+### Get domain list of items
+##### ``api/get/crawler/domain/item/ <domain><port>`` POST
+
+-----
+
+### Create auto-crawlers
+##### ``api/add/crawler/autoCrawler/`` POST
+
+-----
+
+### get item by mime type/ decoded type
+##### ``api/get/decoded`` POST
+
+### Check if a decoded item exists (via sha1)
+##### ``api/get/decoded/exist/<sha1>`` POST
+
+### Get decoded item metadata
+### Check if a decoded item exists (via sha1)
+##### ``api/get/decoded/metadata/<sha1>`` POST
+
+### Get decoded item correlation (1 depth)
+##### ``api/get/decoded/metadata/<sha1>`` POST
+
+-----
+
+
+-----
+##### ``api/get/cryptocurrency`` POST
+
+### Check if a cryptocurrency address (bitcoin, ..) exists
+##### ``api/get/cryptocurrency/exist/<bitcoin_address>`` POST
+
+### Get cryptocurrency address metadata
+##### ``api/get/cryptocurrency/metadata/<bitcoin_address>`` POST
+
+-----
+
+### Item correlation (1 depth)
+##### ``api/get/item/correlation/`` POST
+
+### Create MISP event from item
+##### ``api/export/item/misp`` POST
+
+### Create TheHive case from item
+##### ``api/export/item/thehive`` POST

doc/api/submit_paste.py

@@ -1,53 +0,0 @@
-#!/usr/bin/env python3
-# -*-coding:UTF-8 -*
-
-'''
-submit your own pastes in AIL
-
-empty values must be initialized
-'''
-
-import requests
-
-if __name__ == '__main__':
-
-    #AIL url
-    url = 'http://localhost:7000'
-
-    ail_url = url + '/PasteSubmit/submit'
-
-    # MIPS TAXONOMIE, need to be initialized (tags_taxonomies = '')
-    tags_taxonomies = 'CERT-XLM:malicious-code=\"ransomware\",CERT-XLM:conformity=\"standard\"'
-
-    # MISP GALAXY, need to be initialized (tags_galaxies = '')
-    tags_galaxies = 'misp-galaxy:cert-seu-gocsector=\"Constituency\",misp-galaxy:cert-seu-gocsector=\"EU-Centric\"'
-
-    # user paste input, need to be initialized (paste_content = '')
-    paste_content = 'paste content test'
-
-    #file full or relative path
-    file_to_submit = 'test_file.zip'
-
-    #compress file password, need to be initialized (password = '')
-    password = ''
-
-    '''
-    submit user text
-    '''
-    r = requests.post(ail_url, data={   'password': password,
-                                        'paste_content': paste_content,
-                                        'tags_taxonomies': tags_taxonomies,
-                                        'tags_galaxies': tags_galaxies})
-    print(r.status_code, r.reason)
-
-
-    '''
-    submit a file
-    '''
-    with open(file_submit,'rb') as f:
-
-        r = requests.post(ail_url, data={   'password': password,
-                                            'paste_content': paste_content,
-                                            'tags_taxonomies': tags_taxonomies,
-                                            'tags_galaxies': tags_galaxies}, files={'file': (file_to_submit, f.read() )})
-        print(r.status_code, r.reason)

tests/testApi.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import time
+import unittest
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
+sys.path.append(os.environ['AIL_FLASK'])
+sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules'))
+
+import Import_helper
+import Tag
+
+from Flask_server import app
+
+def parse_response(obj, ail_response):
+    res_json = ail_response.get_json()
+    if 'status' in res_json:
+        if res_json['status'] == 'error':
+            return obj.fail('{}: {}: {}'.format(ail_response.status_code, res_json['status'], res_json['reason']))
+    return res_json
+
+def get_api_key():
+    with open(os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD'), 'r') as f:
+        content = f.read()
+        content = content.splitlines()
+        apikey = content[-1]
+        apikey = apikey.replace('API_Key=', '', 1)
+        return apikey
+
+APIKEY = get_api_key()
+
+class TestApiV1(unittest.TestCase):
+    import_uuid = None
+    item_id = None
+
+
+    def setUp(self):
+        self.app = app
+        self.app.config['TESTING'] = True
+        self.client = self.app.test_client()
+        self.apikey = APIKEY
+        self.item_content = "text to import"
+        self.item_tags = ["infoleak:analyst-detection=\"private-key\""]
+        self.expected_tags = ["infoleak:analyst-detection=\"private-key\"", 'infoleak:submission="manual"']
+
+    # POST /api/v1/import/item
+    def test_0001_api_import_item(self):
+        input_json = {"type": "text","tags": self.item_tags,"text": self.item_content}
+        req = self.client.post('/api/v1/import/item', json=input_json ,headers={ 'Authorization': self.apikey })
+        req_json = parse_response(self, req)
+        import_uuid = req_json['uuid']
+        self.__class__.import_uuid = import_uuid
+        self.assertTrue(Import_helper.is_valid_uuid_v4(import_uuid))
+
+    # POST /api/v1/get/import/item
+    def test_0002_api_get_import_item(self):
+        input_json = {"uuid": self.__class__.import_uuid}
+        item_not_imported = True
+        import_timout = 30
+        start = time.time()
+
+        while item_not_imported:
+            req = self.client.post('/api/v1/get/import/item', json=input_json ,headers={ 'Authorization': self.apikey })
+            req_json = parse_response(self, req)
+            if req_json['status'] == 'imported':
+                try:
+                    item_id = req_json['items'][0]
+                    item_not_imported = False
+                except Exception as e:
+                    if time.time() - start > import_timout:
+                        item_not_imported = False
+                        self.fail("Import error: {}".format(req_json))
+            else:
+                if time.time() - start > import_timout:
+                    item_not_imported = False
+                    self.fail("Import Timeout, import status: {}".format(req_json['status']))
+        self.__class__.item_id = item_id
+
+        # Process item
+        time.sleep(5)
+
+    # POST /api/v1/get/item/content
+    def test_0003_api_get_item_content(self):
+        input_json = {"id": self.__class__.item_id}
+        req = self.client.post('/api/v1/get/item/content', json=input_json ,headers={ 'Authorization': self.apikey })
+        req_json = parse_response(self, req)
+        item_content = req_json['content']
+        self.assertEqual(item_content, self.item_content)
+
+    # POST /api/v1/get/item/tag
+    def test_0004_api_get_item_tag(self):
+        input_json = {"id": self.__class__.item_id}
+        req = self.client.post('/api/v1/get/item/tag', json=input_json ,headers={ 'Authorization': self.apikey })
+        req_json = parse_response(self, req)
+        item_tags = req_json['tags']
+        self.assertCountEqual(item_tags, self.expected_tags)
+
+    # POST /api/v1/get/item/tag
+    def test_0005_api_get_item_default(self):
+        input_json = {"id": self.__class__.item_id}
+        req = self.client.post('/api/v1/get/item/default', json=input_json ,headers={ 'Authorization': self.apikey })
+        req_json = parse_response(self, req)
+        item_tags = req_json['tags']
+        self.assertCountEqual(item_tags, self.expected_tags)
+        item_content = req_json['content']
+        self.assertEqual(item_content, self.item_content)
+
+    # POST /api/v1/get/item/tag
+    # # TODO: add more test
+    def test_0006_api_get_item(self):
+        input_json = {"id": self.__class__.item_id, "content": True}
+        req = self.client.post('/api/v1/get/item', json=input_json ,headers={ 'Authorization': self.apikey })
+        req_json = parse_response(self, req)
+        item_tags = req_json['tags']
+        self.assertCountEqual(item_tags, self.expected_tags)
+        item_content = req_json['content']
+        self.assertEqual(item_content, self.item_content)
+
+    # POST api/v1/add/item/tag
+    def test_0007_api_add_item_tag(self):
+        tags_to_add = ["infoleak:analyst-detection=\"api-key\""]
+        current_item_tag = Tag.get_item_tags(self.__class__.item_id)
+        current_item_tag.append(tags_to_add[0])
+
+        #galaxy_to_add = ["misp-galaxy:stealer=\"Vidar\""]
+        input_json = {"id": self.__class__.item_id, "tags": tags_to_add}
+        req = self.client.post('/api/v1/add/item/tag', json=input_json ,headers={ 'Authorization': self.apikey })
+        req_json = parse_response(self, req)
+        item_tags = req_json['tags']
+        self.assertEqual(item_tags, tags_to_add)
+
+        new_item_tag = Tag.get_item_tags(self.__class__.item_id)
+        self.assertCountEqual(new_item_tag, current_item_tag)
+
+    # DELETE api/v1/delete/item/tag
+    def test_0008_api_add_item_tag(self):
+        tags_to_delete = ["infoleak:analyst-detection=\"api-key\""]
+        input_json = {"id": self.__class__.item_id, "tags": tags_to_delete}
+        req = self.client.delete('/api/v1/delete/item/tag', json=input_json ,headers={ 'Authorization': self.apikey })
+        req_json = parse_response(self, req)
+        item_tags = req_json['tags']
+        self.assertCountEqual(item_tags, tags_to_delete)
+        current_item_tag = Tag.get_item_tags(self.__class__.item_id)
+        if tags_to_delete[0] in current_item_tag:
+            self.fail('Tag no deleted')
+
+    # POST api/v1/get/tag/metadata
+    def test_0009_api_add_item_tag(self):
+        input_json = {"tag": self.item_tags[0]}
+        req = self.client.post('/api/v1/get/tag/metadata', json=input_json ,headers={ 'Authorization': self.apikey })
+        req_json = parse_response(self, req)
+        self.assertEqual(req_json['tag'], self.item_tags[0])
+
+    # GET api/v1/get/tag/all
+    def test_0010_api_add_item_tag(self):
+        input_json = {"tag": self.item_tags[0]}
+        req = self.client.get('/api/v1/get/tag/all', json=input_json ,headers={ 'Authorization': self.apikey })
+        req_json = parse_response(self, req)
+        self.assertTrue(req_json['tags'])
+
+if __name__ == "__main__":
+    unittest.main()

var/www/Flask_server.py

@@ -5,6 +5,7 @@ import os
 import re
 import sys
 import ssl
+import json
 import time
 
 import redis
@@ -13,7 +14,7 @@ import logging
 import logging.handlers
 import configparser
 
-from flask import Flask, render_template, jsonify, request, Request, session, redirect, url_for
+from flask import Flask, render_template, jsonify, request, Request, Response, session, redirect, url_for
 from flask_login import LoginManager, current_user, login_user, logout_user, login_required
 
 import bcrypt
@@ -37,6 +38,8 @@ import Flask_config
 from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
 from Role_Manager import login_admin, login_analyst
 
+Flask_dir = os.environ['AIL_FLASK']
+
 # CONFIG #
 cfg = Flask_config.cfg
 baseUrl = cfg.get("Flask", "baseurl")
@@ -67,21 +70,21 @@ log_dir = os.path.join(os.environ['AIL_HOME'], 'logs')
 if not os.path.isdir(log_dir):
     os.makedirs(logs_dir)
 
-log_filename = os.path.join(log_dir, 'flask_server.logs')
+# log_filename = os.path.join(log_dir, 'flask_server.logs')
-logger = logging.getLogger()
+# logger = logging.getLogger()
-formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+# formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
-handler_log = logging.handlers.TimedRotatingFileHandler(log_filename, when="midnight", interval=1)
+# handler_log = logging.handlers.TimedRotatingFileHandler(log_filename, when="midnight", interval=1)
-handler_log.suffix = '%Y-%m-%d.log'
+# handler_log.suffix = '%Y-%m-%d.log'
-handler_log.setFormatter(formatter)
+# handler_log.setFormatter(formatter)
-handler_log.setLevel(30)
+# handler_log.setLevel(30)
-logger.addHandler(handler_log)
+# logger.addHandler(handler_log)
-logger.setLevel(30)
+# logger.setLevel(30)
 
 # =========       =========#
 
 # =========  TLS  =========#
 ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
-ssl_context.load_cert_chain(certfile='server.crt', keyfile='server.key')
+ssl_context.load_cert_chain(certfile=os.path.join(Flask_dir, 'server.crt'), keyfile=os.path.join(Flask_dir, 'server.key'))
 #print(ssl_context.get_ciphers())
 # =========       =========#
 
@@ -112,13 +115,12 @@ try:
             toIgnoreModule.add(line)
 
 except IOError:
-    f = open('templates/ignored_modules.txt', 'w')
+    pass
-    f.close()
 
 # Dynamically import routes and functions from modules
 # Also, prepare header.html
 to_add_to_header_dico = {}
-for root, dirs, files in os.walk('modules/'):
+for root, dirs, files in os.walk(os.path.join(Flask_dir, 'modules')):
     sys.path.append(join(root))
 
     # Ignore the module
@@ -140,7 +142,7 @@ for root, dirs, files in os.walk('modules/'):
 
 #create header.html
 complete_header = ""
-with open('templates/header_base.html', 'r') as f:
+with open(os.path.join(Flask_dir, 'templates', 'header_base.html'), 'r') as f:
     complete_header = f.read()
 modified_header = complete_header
 
@@ -159,7 +161,7 @@ for module_name, txt in to_add_to_header_dico.items():
 modified_header = modified_header.replace('<!--insert here-->', '\n'.join(to_add_to_header))
 
 #Write the header.html file
-with open('templates/header.html', 'w') as f:
+with open(os.path.join(Flask_dir, 'templates', 'header.html'), 'w') as f:
     f.write(modified_header)
 
 # ========= JINJA2 FUNCTIONS ========
@@ -226,7 +228,7 @@ def login():
             # login failed
             else:
                 # set brute force protection
-                logger.warning("Login failed, ip={}, username={}".format(current_ip, username))
+                #logger.warning("Login failed, ip={}, username={}".format(current_ip, username))
                 r_cache.incr('failed_login_ip:{}'.format(current_ip))
                 r_cache.expire('failed_login_ip:{}'.format(current_ip), 300)
                 r_cache.incr('failed_login_user_id:{}'.format(username))
@@ -289,7 +291,26 @@ def searchbox():
 
 # ========== ERROR HANDLER ============
 
+@app.errorhandler(405)
+def _handle_client_error(e):
+    if request.path.startswith('/api/'):
+        res_dict = {"status": "error", "reason": "Method Not Allowed: The method is not allowed for the requested URL"}
+        anchor_id = request.path[8:]
+        anchor_id = anchor_id.replace('/', '_')
+        api_doc_url = 'https://github.com/CIRCL/AIL-framework/tree/master/doc#{}'.format(anchor_id)
+        res_dict['documentation'] = api_doc_url
+        return Response(json.dumps(res_dict, indent=2, sort_keys=True), mimetype='application/json'), 405
+    else:
+        return e
+
 @app.errorhandler(404)
+def error_page_not_found(e):
+    if request.path.startswith('/api/'):
+        return Response(json.dumps({"status": "error", "reason": "404 Not Found"}, indent=2, sort_keys=True), mimetype='application/json'), 404
+    else:
+        # avoid endpoint enumeration
+        return page_not_found(e)
+
 @login_required
 def page_not_found(e):
     # avoid endpoint enumeration

var/www/modules/Flask_config.py

@@ -12,7 +12,6 @@ import sys
 
 # FLASK #
 app = None
-#secret_key = 'ail-super-secret_key01C'
 
 # CONFIG #
 configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
@@ -146,7 +145,7 @@ if HiveApi != False:
         HiveApi = False
         print('The Hive not connected')
 
-# VARIABLES #
+#### VARIABLES ####
 baseUrl = cfg.get("Flask", "baseurl")
 baseUrl = baseUrl.replace('/', '')
 if baseUrl != '':
@@ -179,6 +178,8 @@ crawler_enabled = cfg.getboolean("Crawler", "activate_crawler")
 email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
 email_regex = re.compile(email_regex)
 
+IMPORT_MAX_TEXT_SIZE = 900000 # size in bytes
+
 # VT
 try:
     from virusTotalKEYS import vt_key

var/www/modules/PasteSubmit/Flask_PasteSubmit.py

@@ -23,6 +23,9 @@ import json
 
 import Paste
 
+import Import_helper
+import Tag
+
 from pytaxonomies import Taxonomies
 from pymispgalaxies import Galaxies, Clusters
 
@@ -87,65 +90,6 @@ def clean_filename(filename, whitelist=valid_filename_chars, replace=' '):
     # keep only whitelisted chars
     return ''.join(c for c in cleaned_filename if c in whitelist)
 
-def launch_submit(ltags, ltagsgalaxies, paste_content, UUID,  password, isfile = False):
-
-    # save temp value on disk
-    r_serv_db.set(UUID + ':ltags', ltags)
-    r_serv_db.set(UUID + ':ltagsgalaxies', ltagsgalaxies)
-    r_serv_db.set(UUID + ':paste_content', paste_content)
-    r_serv_db.set(UUID + ':password', password)
-    r_serv_db.set(UUID + ':isfile', isfile)
-
-    r_serv_log_submit.set(UUID + ':end', 0)
-    r_serv_log_submit.set(UUID + ':processing', 0)
-    r_serv_log_submit.set(UUID + ':nb_total', -1)
-    r_serv_log_submit.set(UUID + ':nb_end', 0)
-    r_serv_log_submit.set(UUID + ':nb_sucess', 0)
-    r_serv_log_submit.set(UUID + ':error', 'error:')
-    r_serv_log_submit.sadd(UUID + ':paste_submit_link', '')
-
-
-    # save UUID on disk
-    r_serv_db.sadd('submitted:uuid', UUID)
-
-
-def addTagsVerification(tags, tagsgalaxies):
-
-    list_tag = tags.split(',')
-    list_tag_galaxies = tagsgalaxies.split(',')
-
-    taxonomies = Taxonomies()
-    active_taxonomies = r_serv_tags.smembers('active_taxonomies')
-
-    active_galaxies = r_serv_tags.smembers('active_galaxies')
-
-    if list_tag != ['']:
-        for tag in list_tag:
-            # verify input
-            tax = tag.split(':')[0]
-            if tax in active_taxonomies:
-                if tag in r_serv_tags.smembers('active_tag_' + tax):
-                    pass
-                else:
-                    return False
-            else:
-                return False
-
-    if list_tag_galaxies != ['']:
-        for tag in list_tag_galaxies:
-            # verify input
-            gal = tag.split(':')[1]
-            gal = gal.split('=')[0]
-
-            if gal in active_galaxies:
-                if tag in r_serv_tags.smembers('active_tag_galaxies_' + gal):
-                    pass
-                else:
-                    return False
-            else:
-                return False
-    return True
-
 def date_to_str(date):
     return "{0}-{1}-{2}".format(date.year, date.month, date.day)
 
@@ -279,11 +223,9 @@ def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title,
 @login_required
 @login_analyst
 def PasteSubmit_page():
-    #active taxonomies
+    # Get all active tags/galaxy
-    active_taxonomies = r_serv_tags.smembers('active_taxonomies')
+    active_taxonomies = Tag.get_active_taxonomies()
-
+    active_galaxies = Tag.get_active_galaxies()
-    #active galaxies
-    active_galaxies = r_serv_tags.smembers('active_galaxies')
 
     return render_template("submit_items.html",
                             active_taxonomies = active_taxonomies,
@@ -311,21 +253,27 @@ def submit():
     submitted_tag = 'infoleak:submission="manual"'
 
     #active taxonomies
-    active_taxonomies = r_serv_tags.smembers('active_taxonomies')
+    active_taxonomies = Tag.get_active_taxonomies()
     #active galaxies
-    active_galaxies = r_serv_tags.smembers('active_galaxies')
+    active_galaxies = Tag.get_active_galaxies()
 
     if ltags or ltagsgalaxies:
-        if not addTagsVerification(ltags, ltagsgalaxies):
+
+        ltags = ltags.split(',')
+        ltagsgalaxies = ltagsgalaxies.split(',')
+
+        print(ltags)
+        print(ltagsgalaxies)
+
+        if not Tags.is_valid_tags_taxonomies_galaxy(ltags, ltagsgalaxies):
             content = 'INVALID TAGS'
             print(content)
             return content, 400
 
     # add submitted tags
-    if(ltags != ''):
+    if not ltags:
-        ltags = ltags + ',' + submitted_tag
+        ltags = []
-    else:
+    ltags.append(submitted_tag)
-        ltags = submitted_tag
 
     if is_file:
         if file:
@@ -358,7 +306,7 @@ def submit():
 
                 paste_content = full_path
 
-                launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password ,True)
+                Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password ,True)
 
                 return render_template("submit_items.html",
                                             active_taxonomies = active_taxonomies,
@@ -376,12 +324,7 @@ def submit():
 
             # get id
             UUID = str(uuid.uuid4())
-
+            Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password)
-            #if paste_name:
-                # clean file name
-                #id = clean_filename(paste_name)
-
-            launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password)
 
             return render_template("submit_items.html",
                                         active_taxonomies = active_taxonomies,
@@ -415,7 +358,7 @@ def submit_status():
         nb_sucess = r_serv_log_submit.get(UUID + ':nb_sucess')
         paste_submit_link = list(r_serv_log_submit.smembers(UUID + ':paste_submit_link'))
 
-        if (end != None) and (nb_total != None) and (nb_end != None) and (error != None) and (processing != None) and (paste_submit_link != None):
+        if (end != None) and (nb_total != None) and (nb_end != None) and (processing != None):
 
             link = ''
             if paste_submit_link:
@@ -433,10 +376,10 @@ def submit_status():
             else:
                 prog = 0
 
-            if error == 'error:':
+            if error:
-                isError = False
-            else:
                 isError = True
+            else:
+                isError = False
 
             if end == '0':
                 end = False

var/www/modules/Tags/Flask_Tags.py

@@ -20,6 +20,7 @@ from pymispgalaxies import Galaxies, Clusters
 
 # ============ VARIABLES ============
 import Flask_config
+import Tag
 
 app = Flask_config.app
 cfg = Flask_config.cfg
@@ -59,16 +60,6 @@ for name, tags in clusters.items(): #galaxie name + tags
 def one():
     return 1
 
-def date_substract_day(date, num_day=1):
-    new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) - datetime.timedelta(num_day)
-    new_date = str(new_date).replace('-', '')
-    return new_date
-
-def date_add_day(date, num_day=1):
-    new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day)
-    new_date = str(new_date).replace('-', '')
-    return new_date
-
 def get_tags_with_synonyms(tag):
     str_synonyms = ' - synonyms: '
     synonyms = r_serv_tags.smembers('synonym_tag_' + tag)
@@ -131,93 +122,6 @@ def get_last_seen_from_tags_list(list_tags):
                 min_last_seen = tag_last_seen
     return str(min_last_seen)
 
-def add_item_tag(tag, item_path):
-    item_date = int(get_item_date(item_path))
-
-    #add tag
-    r_serv_metadata.sadd('tag:{}'.format(item_path), tag)
-    r_serv_tags.sadd('{}:{}'.format(tag, item_date), item_path)
-
-    r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, 1)
-
-    tag_first_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
-    if tag_first_seen is None:
-        tag_first_seen = 99999999
-    else:
-        tag_first_seen = int(tag_first_seen)
-    tag_last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
-    if tag_last_seen is None:
-        tag_last_seen = 0
-    else:
-        tag_last_seen = int(tag_last_seen)
-
-    #add new tag in list of all used tags
-    r_serv_tags.sadd('list_tags', tag)
-
-    # update fisrt_seen/last_seen
-    if item_date < tag_first_seen:
-        r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', item_date)
-
-    # update metadata last_seen
-    if item_date > tag_last_seen:
-        r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', item_date)
-
-def remove_item_tag(tag, item_path):
-    item_date = int(get_item_date(item_path))
-
-    #remove tag
-    r_serv_metadata.srem('tag:{}'.format(item_path), tag)
-    res = r_serv_tags.srem('{}:{}'.format(tag, item_date), item_path)
-
-    if res ==1:
-        # no tag for this day
-        if int(r_serv_tags.hget('daily_tags:{}'.format(item_date), tag)) == 1:
-            r_serv_tags.hdel('daily_tags:{}'.format(item_date), tag)
-        else:
-            r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, -1)
-
-        tag_first_seen = int(r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen'))
-        tag_last_seen = int(r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen'))
-        # update fisrt_seen/last_seen
-        if item_date == tag_first_seen:
-            update_tag_first_seen(tag, tag_first_seen, tag_last_seen)
-        if item_date == tag_last_seen:
-            update_tag_last_seen(tag, tag_first_seen, tag_last_seen)
-    else:
-        return 'Error incorrect tag'
-
-def update_tag_first_seen(tag, tag_first_seen, tag_last_seen):
-    if tag_first_seen == tag_last_seen:
-        if r_serv_tags.scard('{}:{}'.format(tag, tag_first_seen)) > 0:
-            r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_first_seen)
-        # no tag in db
-        else:
-            r_serv_tags.srem('list_tags', tag)
-            r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'first_seen')
-            r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'last_seen')
-    else:
-        if r_serv_tags.scard('{}:{}'.format(tag, tag_first_seen)) > 0:
-            r_serv_tags.hset('tag_metadata:{}'.format(tag), 'first_seen', tag_first_seen)
-        else:
-            tag_first_seen = date_add_day(tag_first_seen)
-            update_tag_first_seen(tag, tag_first_seen, tag_last_seen)
-
-def update_tag_last_seen(tag, tag_first_seen, tag_last_seen):
-    if tag_first_seen == tag_last_seen:
-        if r_serv_tags.scard('{}:{}'.format(tag, tag_last_seen)) > 0:
-            r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_last_seen)
-        # no tag in db
-        else:
-            r_serv_tags.srem('list_tags', tag)
-            r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'first_seen')
-            r_serv_tags.hdel('tag_metadata:{}'.format(tag), 'last_seen')
-    else:
-        if r_serv_tags.scard('{}:{}'.format(tag, tag_last_seen)) > 0:
-            r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', tag_last_seen)
-        else:
-            tag_last_seen = date_substract_day(tag_last_seen)
-            update_tag_last_seen(tag, tag_first_seen, tag_last_seen)
-
 # ============= ROUTES ==============
 
 @Tags.route("/tags/", methods=['GET'])
@@ -472,8 +376,9 @@ def remove_tag():
     path = request.args.get('paste')
     tag = request.args.get('tag')
 
-    remove_item_tag(tag, path)
+    res = Tag.remove_item_tag(tag, path)
-
+    if res[1] != 200:
+        str(res[0])
     return redirect(url_for('showsavedpastes.showsavedpaste', paste=path))
 
 @Tags.route("/Tags/confirm_tag")
@@ -486,11 +391,11 @@ def confirm_tag():
     tag = request.args.get('tag')
 
     if(tag[9:28] == 'automatic-detection'):
-        remove_item_tag(tag, path)
+        Tag.remove_item_tag(tag, path)
 
         tag = tag.replace('automatic-detection','analyst-detection', 1)
         #add analyst tag
-        add_item_tag(tag, path)
+        Tag.add_item_tag(tag, path)
 
         return redirect(url_for('showsavedpastes.showsavedpaste', paste=path))
 
@@ -530,42 +435,12 @@ def addTags():
     list_tag = tags.split(',')
     list_tag_galaxies = tagsgalaxies.split(',')
 
-    taxonomies = Taxonomies()
+    res = Tag.add_items_tag(list_tag, list_tag_galaxies, path)
-    active_taxonomies = r_serv_tags.smembers('active_taxonomies')
+    print(res)
-
+    # error
-    active_galaxies = r_serv_tags.smembers('active_galaxies')
+    if res[1] != 200:
-
+        return str(res[0])
-    if not path:
+    # success
-        return 'INCORRECT INPUT0'
-
-    if list_tag != ['']:
-        for tag in list_tag:
-            # verify input
-            tax = tag.split(':')[0]
-            if tax in active_taxonomies:
-                if tag in r_serv_tags.smembers('active_tag_' + tax):
-                    add_item_tag(tag, path)
-
-                else:
-                    return 'INCORRECT INPUT1'
-            else:
-                return 'INCORRECT INPUT2'
-
-    if list_tag_galaxies != ['']:
-        for tag in list_tag_galaxies:
-            # verify input
-            gal = tag.split(':')[1]
-            gal = gal.split('=')[0]
-
-            if gal in active_galaxies:
-                if tag in r_serv_tags.smembers('active_tag_galaxies_' + gal):
-                    add_item_tag(tag, path)
-
-                else:
-                    return 'INCORRECT INPUT3'
-            else:
-                return 'INCORRECT INPUT4'
-
     return redirect(url_for('showsavedpastes.showsavedpaste', paste=path))
 
 

var/www/modules/restApi/Flask_restApi.py

@@ -8,10 +8,16 @@
 import os
 import re
 import sys
+import uuid
 import json
 import redis
 import datetime
 
+import Import_helper
+import Item
+import Paste
+import Tag
+
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
 from flask_login import login_required
 
@@ -20,6 +26,7 @@ from functools import wraps
 # ============ VARIABLES ============
 import Flask_config
 
+
 app = Flask_config.app
 cfg = Flask_config.cfg
 baseUrl = Flask_config.baseUrl
@@ -28,6 +35,7 @@ r_serv_db = Flask_config.r_serv_db
 r_serv_onion = Flask_config.r_serv_onion
 r_serv_metadata = Flask_config.r_serv_metadata
 
+
 restApi = Blueprint('restApi', __name__, template_folder='templates')
 
 # ============ AUTH FUNCTIONS ============
@@ -36,7 +44,7 @@ def check_token_format(strg, search=re.compile(r'[^a-zA-Z0-9_-]').search):
     return not bool(search(strg))
 
 def verify_token(token):
-    if len(token) != 55:
+    if len(token) != 41:
         return False
 
     if not check_token_format(token):
@@ -47,23 +55,41 @@ def verify_token(token):
     else:
         return False
 
+def verify_user_role(role, token):
+    user_id = r_serv_db.hget('user:tokens', token)
+    if user_id:
+        if is_in_role(user_id, role):
+            return True
+        else:
+            return False
+    else:
+        return False
+
+def is_in_role(user_id, role):
+    if r_serv_db.sismember('user_role:{}'.format(role), user_id):
+        return True
+    else:
+        return False
+
 # ============ DECORATOR ============
 
-def token_required(funct):
+def token_required(user_role):
-    @wraps(funct)
+    def actual_decorator(funct):
-    def api_token(*args, **kwargs):
+        @wraps(funct)
-        data = authErrors()
+        def api_token(*args, **kwargs):
-        if data:
+            data = authErrors(user_role)
-            return Response(json.dumps(data[0], indent=2, sort_keys=True), mimetype='application/json'), data[1]
+            if data:
-        else:
+                return Response(json.dumps(data[0], indent=2, sort_keys=True), mimetype='application/json'), data[1]
-            return funct(*args, **kwargs)
+            else:
-    return api_token
+                return funct(*args, **kwargs)
+        return api_token
+    return actual_decorator
 
 def get_auth_from_header():
     token = request.headers.get('Authorization').replace(' ', '') # remove space
     return token
 
-def authErrors():
+def authErrors(user_role):
     # Check auth
     if not request.headers.get('Authorization'):
         return ({'status': 'error', 'reason': 'Authentication needed'}, 401)
@@ -71,12 +97,27 @@ def authErrors():
     data = None
     # verify token format
 
+    # brute force protection
+    current_ip = request.remote_addr
+    login_failed_ip = r_cache.get('failed_login_ip_api:{}'.format(current_ip))
+    # brute force by ip
+    if login_failed_ip:
+        login_failed_ip = int(login_failed_ip)
+        if login_failed_ip >= 5:
+            return ({'status': 'error', 'reason': 'Max Connection Attempts reached, Please wait {}s'.format(r_cache.ttl('failed_login_ip_api:{}'.format(current_ip)))}, 401)
+
     try:
         authenticated = False
         if verify_token(token):
             authenticated = True
 
+            # check user role
+            if not verify_user_role(user_role, token):
+                data = ({'status': 'error', 'reason': 'Access Forbidden'}, 403)
+
         if not authenticated:
+            r_cache.incr('failed_login_ip_api:{}'.format(current_ip))
+            r_cache.expire('failed_login_ip_api:{}'.format(current_ip), 300)
             data = ({'status': 'error', 'reason': 'Authentication failed'}, 401)
     except Exception as e:
         print(e)
@@ -86,8 +127,18 @@ def authErrors():
     else:
         return None
 
+# ============ API CORE =============
+
 # ============ FUNCTIONS ============
 
+def is_valid_uuid_v4(header_uuid):
+    try:
+        header_uuid=header_uuid.replace('-', '')
+        uuid_test = uuid.UUID(hex=header_uuid, version=4)
+        return uuid_test.hex == header_uuid
+    except:
+        return False
+
 def one():
     return 1
 
@@ -98,12 +149,249 @@ def one():
 # def api():
 #     return 'api doc'
 
-@restApi.route("api/items", methods=['POST'])
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-@token_required
+# POST
-def items():
+#
-    item = request.args.get('id')
+# {
+#   "id": item_id,      mandatory
+#   "content": true,
+#   "tags": true,
+#
+#
+# }
+#
+# response: {
+#               "id": "item_id",
+#               "tags": [],
+#           }
+#
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+@restApi.route("api/v1/get/item", methods=['POST'])
+@token_required('analyst')
+def get_item_id():
+    data = request.get_json()
+    res = Item.get_item(data)
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
-    return Response(json.dumps({'test': 2}), mimetype='application/json')
+@restApi.route("api/v1/get/item/default", methods=['POST'])
+@token_required('analyst')
+def get_item_id_basic():
+
+    data = request.get_json()
+    item_id = data.get('id', None)
+    req_data = {'id': item_id, 'date': True, 'content': True, 'tags': True}
+    res = Item.get_item(req_data)
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# GET
+#
+# {
+#   "id": item_id,      mandatory
+# }
+#
+# response: {
+#               "id": "item_id",
+#               "tags": [],
+#           }
+#
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+@restApi.route("api/v1/get/item/tag", methods=['POST'])
+@token_required('analyst')
+def get_item_tag():
+
+    data = request.get_json()
+    item_id = data.get('id', None)
+    req_data = {'id': item_id, 'date': False, 'tags': True}
+    res = Item.get_item(req_data)
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# POST
+#
+# {
+#   "id": item_id,      mandatory
+#   "tags": [tags to add],
+#   "galaxy": [galaxy to add],
+# }
+#
+# response: {
+#               "id": "item_id",
+#               "tags": [tags added],
+#           }
+#
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+@restApi.route("api/v1/add/item/tag", methods=['POST'])
+@token_required('analyst')
+def add_item_tags():
+
+    data = request.get_json()
+    if not data:
+        return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400
+
+    item_id = data.get('id', None)
+    tags = data.get('tags', [])
+    galaxy = data.get('galaxy', [])
+
+    res = Tag.add_items_tag(tags, galaxy, item_id)
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# DELETE
+#
+# {
+#   "id": item_id,      mandatory
+#   "tags": [tags to delete],
+# }
+#
+# response: {
+#               "id": "item_id",
+#               "tags": [tags deleted],
+#           }
+#
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+@restApi.route("api/v1/delete/item/tag", methods=['DELETE'])
+@token_required('analyst')
+def delete_item_tags():
+
+    data = request.get_json()
+    if not data:
+        return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400
+
+    item_id = data.get('id', None)
+    tags = data.get('tags', [])
+
+    res = Tag.remove_item_tags(tags, item_id)
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# GET
+#
+# {
+#   "id": item_id,      mandatory
+# }
+#
+# response: {
+#               "id": "item_id",
+#               "content": "item content"
+#           }
+#
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+@restApi.route("api/v1/get/item/content", methods=['POST'])
+@token_required('analyst')
+def get_item_content():
+
+    data = request.get_json()
+    item_id = data.get('id', None)
+    req_data = {'id': item_id, 'date': False, 'content': True, 'tags': False}
+    res = Item.get_item(req_data)
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# # # # # # # # # # # # # #        TAGS       # # # # # # # # # # # # # # # # #
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+@restApi.route("api/v1/get/tag/metadata", methods=['POST'])
+@token_required('analyst')
+def get_tag_metadata():
+    data = request.get_json()
+    tag = data.get('tag', None)
+    if not Tag.is_tag_in_all_tag(tag):
+        return Response(json.dumps({'status': 'error', 'reason':'Tag not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404
+    metadata = Tag.get_tag_metadata(tag)
+    return Response(json.dumps(metadata, indent=2, sort_keys=True), mimetype='application/json'), 200
+
+@restApi.route("api/v1/get/tag/all", methods=['GET'])
+@token_required('analyst')
+def get_all_tags():
+    res = {'tags': Tag.get_all_tags()}
+    return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# # # # # # # # # # # # #        IMPORT     # # # # # # # # # # # # # # # # # #
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+#
+# POST JSON FORMAT
+#
+# {
+#   "type": "text",         (default value)
+#   "tags": [],             (default value)
+#   "default_tags": True,    (default value)
+#   "galaxy" [],            (default value)
+#   "text": "",             mandatory if type = text
+# }
+#
+# response: {"uuid": "uuid"}
+#
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+@restApi.route("api/v1/import/item", methods=['POST'])
+@token_required('analyst')
+def import_item():
+
+    data = request.get_json()
+    if not data:
+        return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400
+
+    # unpack json
+    text_to_import = data.get('text', None)
+    if not text_to_import:
+        return Response(json.dumps({'status': 'error', 'reason': 'No text supplied'}, indent=2, sort_keys=True), mimetype='application/json'), 400
+
+    tags = data.get('tags', [])
+    if not type(tags) is list:
+        tags = []
+    galaxy = data.get('galaxy', [])
+    if not type(galaxy) is list:
+        galaxy = []
+
+    if not Tag.is_valid_tags_taxonomies_galaxy(tags, galaxy):
+        return Response(json.dumps({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, indent=2, sort_keys=True), mimetype='application/json'), 400
+
+    default_tags = data.get('default_tags', True)
+    if default_tags:
+        tags.append('infoleak:submission="manual"')
+
+    if sys.getsizeof(text_to_import) > 900000:
+        return Response(json.dumps({'status': 'error', 'reason': 'Size exceeds default'}, indent=2, sort_keys=True), mimetype='application/json'), 413
+
+    UUID = str(uuid.uuid4())
+    Import_helper.create_import_queue(tags, galaxy, text_to_import, UUID)
+
+    return Response(json.dumps({'uuid': UUID}, indent=2, sort_keys=True), mimetype='application/json')
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# GET
+#
+# {
+#   "uuid": "uuid",      mandatory
+# }
+#
+# response: {
+#               "status": "in queue"/"in progress"/"imported",
+#               "items": [all item id]
+#           }
+#
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+@restApi.route("api/v1/get/import/item", methods=['POST'])
+@token_required('analyst')
+def import_item_uuid():
+    data = request.get_json()
+    UUID = data.get('uuid', None)
+
+    # Verify uuid
+    if not is_valid_uuid_v4(UUID):
+        return Response(json.dumps({'status': 'error', 'reason': 'Invalid uuid'}), mimetype='application/json'), 400
+
+    data = Import_helper.check_import_status(UUID)
+    if data:
+        return Response(json.dumps(data[0]), mimetype='application/json'), data[1]
+
+    return Response(json.dumps({'status': 'error', 'reason': 'Invalid response'}), mimetype='application/json'), 400
 
 # ========= REGISTRATION =========
 app.register_blueprint(restApi, url_prefix=baseUrl)