Merge branch 'master' into crawler_manager

This commit is contained in:
Terrtia 2020-08-20 09:18:55 +02:00
commit 2c0ee71845
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
11 changed files with 45 additions and 16 deletions

View file

@ -221,7 +221,7 @@ def get_git_upper_tags_remote(current_tag, is_fork):
# add tag with last commit
if float(tag_val) >= float(current_tag_val):
dict_tags_commit[tag_val] = commit
dict_tags_commit[tag] = commit
list_upper_tags = [('v{}'.format(key), dict_tags_commit[key]) for key in dict_tags_commit]
# force update order
list_upper_tags.sort()

View file

@ -41,7 +41,7 @@ def get_email_subject(tracker_uuid):
return 'AIL framework: {}'.format(tracker_description)
def get_tracker_last_updated_by_type(tracker_type):
epoch_update = r_serv_tracker.get('tracker:refresh:{}'.format(term_type))
epoch_update = r_serv_tracker.get('tracker:refresh:{}'.format(tracker_type))
if not epoch_update:
epoch_update = 0
return float(epoch_update)
@ -123,6 +123,20 @@ def save_yara_rule(yara_rule_type, yara_rule, tracker_uuid=None):
if yara_rule_type == 'yara_default':
filename = os.path.join('ail-yara-rules', 'rules', yara_rule)
return filename
def get_yara_rule_content(yara_rule):
yara_dir = get_yara_rules_dir()
filename = os.path.join(yara_dir, yara_rule)
filename = os.path.realpath(filename)
# incorrect filename
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
return '' # # TODO: throw exception
with open(filename, 'r') as f:
rule_content = f.read()
return rule_content
##-- YARA --##

View file

@ -77,8 +77,6 @@ if __name__ == "__main__":
if yara_match:
print(f'{item_id}: {yara_match}')
time.sleep(30)
else:
time.sleep(5)

View file

@ -118,6 +118,7 @@ $AIL_HOME/doc/generate_modules_data_flow_graph.sh
pushd ${AIL_HOME}
# shallow clone
git fetch --depth=500 --tags --prune
git fetch --unshallow
git describe --abbrev=0 --tags | tr -d '\n' > ${AIL_HOME}/update/current_version
echo "AIL current version:"
git describe --abbrev=0 --tags

View file

@ -1,4 +1,7 @@
FROM ubuntu:16.04
FROM ubuntu:18.04
ARG tz_buildtime=Europe/Rome
ENV TZ=$tz_buildtime
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
# Make sure that all updates are in place
RUN apt-get clean && apt-get update -y && apt-get upgrade -y \
@ -19,12 +22,6 @@ WORKDIR /opt/AIL
RUN ./installing_deps.sh
WORKDIR /opt/AIL
# Installing Web dependencies,
# remove all the parts below if you dont need the Web UI
WORKDIR /opt/AIL/var/www
RUN ./update_thirdparty.sh
WORKDIR /opt/AIL
# Default to UTF-8 file.encoding
ENV LANG C.UTF-8
ENV AIL_HOME /opt/AIL
@ -38,7 +35,6 @@ ENV PATH ${AIL_VENV}/bin:${AIL_HOME}:${AIL_REDIS}:${AIL_ARDB}:${AIL_BIN}:${AIL_F
RUN ./pystemon/install.sh
RUN pip install -r /opt/pystemon/requirements.txt
RUN pip install -r /opt/AIL/crawler_requirements.txt
COPY docker_start.sh /docker_start.sh
ENTRYPOINT ["/bin/bash", "docker_start.sh"]

View file

@ -1,4 +1,4 @@
Docker Quick Start (Ubuntu 16.04 LTS)
Docker Quick Start (Ubuntu 18.04 LTS)
------------
:warning:
@ -19,7 +19,10 @@ curl https://get.docker.com | /bin/bash
```bash
git clone https://github.com/CIRCL/AIL-framework.git
cd AIL-framework
docker build -t ail-framework .
cp -r ./other_installers/docker/Dockerfile ./other_installers/docker/docker_start.sh ./other_installers/docker/pystemon ./
cp ./configs/update.cfg.sample ./configs/update.cfg
vim/nano ./configs/update.cfg (set auto_update to False)
docker build --build-arg tz_buildtime=YOUR_GEO_AREA/YOUR_CITY -t ail-framework .
```
3. To start AIL on port 7000, type the following command below:
```

View file

@ -13,6 +13,7 @@ signalListener() {
source ./AILENV/bin/activate
cd bin
./LAUNCH.sh -l
./LAUNCH.sh -c
./LAUNCH.sh -f
signalListener tail -f /dev/null $@

View file

@ -23,6 +23,10 @@ bash ${AIL_BIN}/LAUNCH.sh -ldbv &
wait
echo ""
# SUBMODULES #
git submodule init
git submodule update
echo -e $GREEN"Installing YARA ..."$DEFAULT
pip3 install yara-python
bash ${AIL_BIN}/LAUNCH.sh -t

View file

@ -153,6 +153,11 @@ def show_tracker():
tracker_metadata = Term.get_term_metedata(term_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sparkline=True)
if tracker_metadata['type'] == 'yara':
yara_rule_content = Tracker.get_yara_rule_content(tracker_metadata['term'])
else:
yara_rule_content = None
if date_from:
res = Term.parse_get_tracker_term_item({'uuid': term_uuid, 'date_from': date_from, 'date_to': date_to}, user_id)
if res[1] !=200:
@ -165,7 +170,9 @@ def show_tracker():
tracker_metadata['date_from'] = ''
tracker_metadata['date_to'] = ''
return render_template("showTracker.html", tracker_metadata=tracker_metadata, bootstrap_label=bootstrap_label)
return render_template("showTracker.html", tracker_metadata=tracker_metadata,
yara_rule_content=yara_rule_content,
bootstrap_label=bootstrap_label)
@hunter.route("/tracker/update_tracker_description", methods=['POST'])
@login_required

View file

@ -175,6 +175,11 @@
<button class='btn btn-danger'><i class="fas fa-trash-alt"></i>
</button>
</a>
{%if yara_rule_content%}
<p class="my-0"></br></br><pre class="border bg-light">{{ yara_rule_content }}</pre></p>
{%endif%}
</div>
</div>

View file

@ -93,7 +93,7 @@ wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb61
wget -q http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js
wget -q http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip
unzip -qq temp/canvasjs.zip -d temp/
mv temp/canvasjs-2.3.2/Chart\ 2.3.2\ GA\ -\ Stable/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js
mv temp/Chart\ 2.3.2\ GA\ -\ Stable/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js
wget -q https://jqueryui.com/resources/download/jquery-ui-1.12.1.zip -O temp/jquery-ui.zip
unzip -qq temp/jquery-ui.zip -d temp/