Merge branch 'master' into crawler_manager

bin/Update.py

@@ -221,7 +221,7 @@ def get_git_upper_tags_remote(current_tag, is_fork):
 
                     # add tag with last commit
                     if float(tag_val) >= float(current_tag_val):
-                        dict_tags_commit[tag_val] = commit
+                        dict_tags_commit[tag] = commit
                 list_upper_tags = [('v{}'.format(key), dict_tags_commit[key]) for key in dict_tags_commit]
                 # force update order
                 list_upper_tags.sort()

bin/lib/Tracker.py

@@ -41,7 +41,7 @@ def get_email_subject(tracker_uuid):
         return 'AIL framework: {}'.format(tracker_description)
 
 def get_tracker_last_updated_by_type(tracker_type):
-    epoch_update = r_serv_tracker.get('tracker:refresh:{}'.format(term_type))
+    epoch_update = r_serv_tracker.get('tracker:refresh:{}'.format(tracker_type))
     if not epoch_update:
         epoch_update = 0
     return float(epoch_update)
@@ -123,6 +123,20 @@ def save_yara_rule(yara_rule_type, yara_rule, tracker_uuid=None):
     if yara_rule_type == 'yara_default':
         filename = os.path.join('ail-yara-rules', 'rules', yara_rule)
     return filename
+
+def get_yara_rule_content(yara_rule):
+    yara_dir = get_yara_rules_dir()
+    filename = os.path.join(yara_dir, yara_rule)
+    filename = os.path.realpath(filename)
+
+    # incorrect filename
+    if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
+        return '' # # TODO: throw exception
+
+    with open(filename, 'r') as f:
+        rule_content = f.read()
+    return rule_content
+
 ##-- YARA --##
 
 

bin/trackers/Tracker_Yara.py

@@ -77,8 +77,6 @@ if __name__ == "__main__":
             if yara_match:
                 print(f'{item_id}: {yara_match}')
 
-            time.sleep(30)
-
         else:
             time.sleep(5)
 

installing_deps.sh

@@ -118,6 +118,7 @@ $AIL_HOME/doc/generate_modules_data_flow_graph.sh
 pushd ${AIL_HOME}
 # shallow clone
 git fetch --depth=500 --tags --prune
+git fetch --unshallow
 git describe --abbrev=0 --tags | tr -d '\n' > ${AIL_HOME}/update/current_version
 echo "AIL current version:"
 git describe --abbrev=0 --tags

other_installers/docker/Dockerfile

@@ -1,4 +1,7 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
+ARG tz_buildtime=Europe/Rome
+ENV TZ=$tz_buildtime
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
 # Make sure that all updates are in place
 RUN apt-get clean && apt-get update -y && apt-get upgrade -y \
@@ -19,12 +22,6 @@ WORKDIR /opt/AIL
 RUN ./installing_deps.sh 
 WORKDIR /opt/AIL
 
-# Installing Web dependencies,
-# remove all the parts below if you dont need the Web UI
-WORKDIR /opt/AIL/var/www
-RUN ./update_thirdparty.sh
-WORKDIR /opt/AIL
-
 # Default to UTF-8 file.encoding
 ENV LANG C.UTF-8
 ENV AIL_HOME /opt/AIL
@@ -38,7 +35,6 @@ ENV PATH ${AIL_VENV}/bin:${AIL_HOME}:${AIL_REDIS}:${AIL_ARDB}:${AIL_BIN}:${AIL_F
 
 RUN ./pystemon/install.sh
 RUN pip install -r /opt/pystemon/requirements.txt
-RUN pip install -r /opt/AIL/crawler_requirements.txt
 
 COPY docker_start.sh /docker_start.sh
 ENTRYPOINT ["/bin/bash", "docker_start.sh"]

other_installers/docker/README.md

@@ -1,4 +1,4 @@
-Docker Quick Start (Ubuntu 16.04 LTS)
+Docker Quick Start (Ubuntu 18.04 LTS)
 ------------
 
 :warning:
@@ -19,7 +19,10 @@ curl https://get.docker.com | /bin/bash
 ```bash
 git clone https://github.com/CIRCL/AIL-framework.git
 cd AIL-framework
-docker build -t ail-framework .
+cp -r ./other_installers/docker/Dockerfile ./other_installers/docker/docker_start.sh ./other_installers/docker/pystemon ./
+cp ./configs/update.cfg.sample ./configs/update.cfg
+vim/nano ./configs/update.cfg (set auto_update to False)
+docker build --build-arg tz_buildtime=YOUR_GEO_AREA/YOUR_CITY -t ail-framework .
 ```
 3. To start AIL on port 7000, type the following command below:
 ```

other_installers/docker/docker_start.sh

@@ -13,6 +13,7 @@ signalListener() {
 source ./AILENV/bin/activate
 cd bin
 ./LAUNCH.sh -l
+./LAUNCH.sh -c
 ./LAUNCH.sh -f
 
 signalListener tail -f /dev/null $@

update/v3.1.1/Update.sh

@@ -23,6 +23,10 @@ bash ${AIL_BIN}/LAUNCH.sh -ldbv &
 wait
 echo ""
 
+# SUBMODULES #
+git submodule init
+git submodule update
+
 echo -e $GREEN"Installing YARA ..."$DEFAULT
 pip3 install yara-python
 bash ${AIL_BIN}/LAUNCH.sh -t

var/www/modules/hunter/Flask_hunter.py

@@ -153,6 +153,11 @@ def show_tracker():
 
     tracker_metadata = Term.get_term_metedata(term_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sparkline=True)
 
+    if tracker_metadata['type'] == 'yara':
+        yara_rule_content = Tracker.get_yara_rule_content(tracker_metadata['term'])
+    else:
+        yara_rule_content = None
+
     if date_from:
         res = Term.parse_get_tracker_term_item({'uuid': term_uuid, 'date_from': date_from, 'date_to': date_to}, user_id)
         if res[1] !=200:
@@ -165,7 +170,9 @@ def show_tracker():
         tracker_metadata['date_from'] = ''
         tracker_metadata['date_to'] = ''
 
-    return render_template("showTracker.html", tracker_metadata=tracker_metadata, bootstrap_label=bootstrap_label)
+    return render_template("showTracker.html", tracker_metadata=tracker_metadata,
+                                    yara_rule_content=yara_rule_content,
+                                    bootstrap_label=bootstrap_label)
 
 @hunter.route("/tracker/update_tracker_description", methods=['POST'])
 @login_required

var/www/modules/hunter/templates/showTracker.html

@@ -175,6 +175,11 @@
 								<button class='btn btn-danger'><i class="fas fa-trash-alt"></i>
 								</button>
 							</a>
+
+							{%if yara_rule_content%}
+								<p class="my-0"></br></br><pre class="border bg-light">{{ yara_rule_content }}</pre></p>
+							{%endif%}
+
 						</div>
 					</div>
 

var/www/update_thirdparty.sh

@@ -93,7 +93,7 @@ wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb61
 wget -q http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js
 wget -q http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip
 unzip -qq temp/canvasjs.zip -d temp/
-mv temp/canvasjs-2.3.2/Chart\ 2.3.2\ GA\ -\ Stable/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js
+mv temp/Chart\ 2.3.2\ GA\ -\ Stable/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js
 
 wget -q https://jqueryui.com/resources/download/jquery-ui-1.12.1.zip -O temp/jquery-ui.zip
 unzip -qq temp/jquery-ui.zip -d temp/