Merge branch 'master' into dev

This commit is contained in:
Terrtia 2021-06-10 17:10:02 +02:00
commit 89316e1eb5
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
5 changed files with 82 additions and 6 deletions

View file

@ -0,0 +1,47 @@
# This is a basic workflow to help you get started with Actions
name: CI
# Controls when the action will run.
on:
# Triggers the workflow on push or pull request events but only for the master branch
push:
branches: [ master, dev ]
pull_request:
branches: [ master, dev ]
workflow_dispatch:
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
# This workflow contains a single job called "build"
ail_test:
# The type of runner that the job will run on
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [3.6, 3.7, 3.8, 3.9]
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2
with:
submodules: 'recursive'
# Runs a single command using the runners shell
- name: Install AIL
run: bash installing_deps.sh
# Runs a set of commands using the runners shell
- name: Launch AIL
run: |
pushd bin
bash LAUNCH.sh -l
# Runs a set of commands using the runners shell
- name: Run tests
run: bash LAUNCH.sh -t

View file

@ -14,6 +14,8 @@ import uuid
import redis import redis
import time import time
import json import json
import binascii
import gzip
from pubsublogger import publisher from pubsublogger import publisher
from Helper import Process from Helper import Process
@ -56,7 +58,8 @@ import thehive4py.exceptions
from thehive4py.models import Alert, AlertArtifact from thehive4py.models import Alert, AlertArtifact
from thehive4py.models import Case, CaseTask, CustomFieldHelper from thehive4py.models import Case, CaseTask, CustomFieldHelper
def is_gzip_file(magic_nuber):
return binascii.hexlify(magic_nuber) == b'1f8b'
def create_the_hive_alert(source, item_id, tag): def create_the_hive_alert(source, item_id, tag):
# # TODO: check items status (processed by all modules) # # TODO: check items status (processed by all modules)
@ -64,9 +67,26 @@ def create_the_hive_alert(source, item_id, tag):
# # # TODO: description, add AIL link:show items ? # # # TODO: description, add AIL link:show items ?
tags = list( r_serv_metadata.smembers('tag:{}'.format(item_id)) ) tags = list( r_serv_metadata.smembers('tag:{}'.format(item_id)) )
path = item_basic.get_item_filepath(item_id)
paste_handle = open(path, 'rb')
paste_data = paste_handle.read()
tmp_path = None
if is_gzip_file(paste_data[0:2]): # if gzip, create a new file to supply to TheHive
paste_handle.close() # TheHive expects a file handle, that's why we create a new file
tmp_data = gzip.decompress(paste_data)
tmp_path = path + '.unzip'
with open(tmp_path, 'wb+') as f:
f.write(tmp_data)
paste_handle = open(tmp_path, 'rb')
if path.endswith(".gz"): # remove .gz from submitted path to TheHive beause we've decompressed it
path = path[:-3]
path = os.path.basename(os.path.normpath(path)) + ".txt" # get last part of path, add .txt so it's easier to open when downloaded from TheHive
artifacts = [ artifacts = [
AlertArtifact( dataType='uuid-ail', data=r_serv_db.get('ail:uuid') ), AlertArtifact( dataType='uuid-ail', data=r_serv_db.get('ail:uuid') ),
AlertArtifact( dataType='file', data=item_basic.get_item_filepath(item_id), tags=tags ) AlertArtifact( dataType='file', data=(paste_handle, path), tags=tags )
] ]
# Prepare the sample Alert # Prepare the sample Alert
@ -95,6 +115,10 @@ def create_the_hive_alert(source, item_id, tag):
except: except:
print('hive connection error') print('hive connection error')
paste_handle.close()
if tmp_path is not None: # this file has been send to TheHive, we won't ever need it again
os.remove(tmp_path)
def feeder(message, count=0): def feeder(message, count=0):
if flag_the_hive or flag_misp: if flag_the_hive or flag_misp:

View file

@ -260,9 +260,12 @@ def domains_search_languages_get():
page = int(page) page = int(page)
except: except:
page = 1 page = 1
domains_types = request.args.getlist('domain_types') domains_types = request.args.getlist('domain_types')
if domains_types: if domains_types:
domains_types = domains_types[0].split(',') domains_types = domains_types[0].split(',')
domains_types = Domain.sanitize_domain_types(domains_types)
languages = request.args.getlist('languages') languages = request.args.getlist('languages')
if languages: if languages:
languages = languages[0].split(',') languages = languages[0].split(',')
@ -281,9 +284,11 @@ def domains_search_name():
page = int(page) page = int(page)
except: except:
page = 1 page = 1
domains_types = request.args.getlist('domain_types') domains_types = request.args.getlist('domain_types')
if domains_types: if domains_types:
domains_types = domains_types[0].split(',') domains_types = domains_types[0].split(',')
domains_types = Domain.sanitize_domain_types(domains_types)
l_dict_domains = Domain.api_search_domains_by_name(name, domains_types, domains_metadata=True, page=page) l_dict_domains = Domain.api_search_domains_by_name(name, domains_types, domains_metadata=True, page=page)
return render_template("domains/domains_result_list.html", template_folder='../../', return render_template("domains/domains_result_list.html", template_folder='../../',

View file

@ -13,13 +13,13 @@
<div class="mb-3"> <div class="mb-3">
<div class="custom-control custom-switch"> <div class="custom-control custom-switch">
<input class="custom-control-input" type="checkbox" name="domain_onion_switch" value="" id="domain_onion_switch" {%if 'onion' in domains_types or not domains_types%}checked{%endif%}> <input class="custom-control-input" type="checkbox" name="domain_onion_switch" value="" id="domain_onion_switch" {%if not domains_types%}checked{%elif 'onion' in domains_types%}checked{%endif%}>
<label class="custom-control-label" for="domain_onion_switch"> <label class="custom-control-label" for="domain_onion_switch">
<span class="badge badge-danger"><i class="fas fa-user-secret"></i> Onion Domains</span> <span class="badge badge-danger"><i class="fas fa-user-secret"></i> Onion Domains</span>
</label> </label>
</div> </div>
<div class="custom-control custom-switch"> <div class="custom-control custom-switch">
<input class="custom-control-input" type="checkbox" name="domain_regular_switch" value="True" id="domain_regular_switch"{%if 'regular' in domains_types%}checked{%endif%}> <input class="custom-control-input" type="checkbox" name="domain_regular_switch" value="True" id="domain_regular_switch"{%if domains_types%}{%if 'regular' in domains_types%}checked{%endif%}{%endif%}>
<label class="custom-control-label" for="domain_regular_switch"> <label class="custom-control-label" for="domain_regular_switch">
<span class="badge badge-warning"><i class="fab fa-html5"></i> Web Domains</span> <span class="badge badge-warning"><i class="fab fa-html5"></i> Web Domains</span>
</label> </label>

View file

@ -17,13 +17,13 @@
<div class="mb-3"> <div class="mb-3">
<div class="custom-control custom-switch"> <div class="custom-control custom-switch">
<input class="custom-control-input" type="checkbox" name="domain_onion_switch" value="" id="domain_onion_switch" {%if 'onion' in domains_types%}checked{%endif%}> <input class="custom-control-input" type="checkbox" name="domain_onion_switch" value="" id="domain_onion_switch" {%if not domains_types%}checked{%elif 'onion' in domains_types%}checked{%endif%}>
<label class="custom-control-label" for="domain_onion_switch"> <label class="custom-control-label" for="domain_onion_switch">
<span class="badge badge-danger"><i class="fas fa-user-secret"></i> Onion Domains</span> <span class="badge badge-danger"><i class="fas fa-user-secret"></i> Onion Domains</span>
</label> </label>
</div> </div>
<div class="custom-control custom-switch"> <div class="custom-control custom-switch">
<input class="custom-control-input" type="checkbox" name="domain_regular_switch" value="True" id="domain_regular_switch"{%if 'regular' in domains_types%}checked{%endif%}> <input class="custom-control-input" type="checkbox" name="domain_regular_switch" value="True" id="domain_regular_switch" {%if not domains_types%}checked{%elif 'regular' in domains_types%}checked{%endif%}>
<label class="custom-control-label" for="domain_regular_switch"> <label class="custom-control-label" for="domain_regular_switch">
<span class="badge badge-warning"><i class="fab fa-html5"></i> Web Domains</span> <span class="badge badge-warning"><i class="fab fa-html5"></i> Web Domains</span>
</label> </label>