Skip to content

Commit

Permalink
Merge pull request #5 from s-tip/filter_only_stix1
Browse files Browse the repository at this point in the history 
Add a STIX 1.x filter and remove debug prints
  • Loading branch information
@ykoji8681
ykoji8681 authored Mar 9, 2020
2 parents f206d68 + a03e6ab commit 7f28391
Showing 1 changed file with 2 additions and 69 deletions.
71 changes: 2 additions & 69 deletions src/api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import tempfile
import io
import structlog
from pymongo import Connection
from opentaxii.persistence.api import OpenTAXIIPersistenceAPI
from opentaxii.taxii.entities import (
ServiceEntity,
Expand Down Expand Up @@ -77,7 +76,6 @@ def __init__(self,
self.black_account_list = black_account_list.split(',')
else:
self.black_account_list = []
# print '>>>black_account_list :' +str(self.black_account_list)

id = 0
for taxii_server in TaxiiServers.objects.all():
Expand Down Expand Up @@ -110,33 +108,23 @@ def __init__(self,
id += 1

def init_app(self, app):
# appは<class 'flask.app.Flask'>
pass

# discovery/taxii-data時にはcollection_id = Noneでコールされる
def get_services(self, collection_id=None):
# print '>>>>get_services enter'
# print '>>>>collection_id:' + str(collection_id)
ret = self.services
# print '>>>>get_services ret:' + str(ret)
# print '>>>>get_services exit'
return ret

# taxii-data時にコールされる
# push時にもコールされる
def get_collections(self, service_id):
# print '>>>>get_collections enter'
# print '>>>>service_id:' + str(service_id)

# opentaxii.taxii.entities.CollectionEntityのリストを返却
s = None
# sercice_idに該当するServiceEntityを検索する
for service in self.services:
if service.id == service_id:
s = service
break
# print '>>>>service:' + str(s)

ret = []
if s is not None:
# ServiceEntityと関連付けされているCollectionEntityを検索する
Expand All @@ -145,61 +133,38 @@ def get_collections(self, service_id):
for col in self.collections:
if col.id == ref[service_id]:
ret.append(col)
# print '>>>>ret:' + str(ret)
# print '>>>>get_collections exit'
return ret

# poll時にコールされる
def get_collection(self, collection_name, service_id):
# print '>>>>get_collection enter'
# print '>>>>collection_name:' + str(collection_name)
# print '>>>>service_id:' + str(service_id)
cols = self.get_collections(service_id)
col = None
for c in cols:
if c.name == collection_name:
col = c
break
# print '>>>>return:' + str(col)
# print '>>>>get_collection exit'

# collection entityを返却する
return col

# mongo の stix_files コレクションより開始日時と終了日時の間 を find し cursor を返却する
def get_stix_files_from_mongo(self, collection_name, start_time, end_time):
# print '>>>get_stix_files_from_mongo:start_time:' + str(start_time)
# print '>>>get_stix_files_from_mongo:end_time:' + str(end_time)
# print '>>>get_stix_files_from_mongo:collection_name:' + str(collection_name)

# TAXII Server 設定取得
ts = TaxiiServers.objects.get(collection_name=collection_name)

# 条件絞り込み
QQ = Q(information_source__in=ts.information_sources)
# for is_ in ts.information_sources:
# print '>>>get_stix_files_from_mongo:information_sources:' + str(is_.name)
# print '>>>get_stix_files_from_mongo:information_sources:' + str(is_.id)
QQ = QQ & Q (version__startswith='1.')
if start_time is not None and end_time is not None:
QQ = QQ & Q(produced__gt=start_time)
QQ = QQ & Q(produced__lt=end_time)
elif start_time is None and end_time is not None:
QQ = QQ & Q(produced__lt=end_time)
elif start_time is not None and end_time is None:
QQ = QQ & Q(produced__gt=start_time)
# print '>>>get_stix_files_from_mongo:count:' + str(StixFiles.objects(QQ).count())
return StixFiles.objects(QQ)

# poll時にコールされる
def get_content_blocks_count(self, collection_id=None, start_time=None, end_time=None, bindings=None):
# print '>>>>get_content_blocks_count enter'
# print '>>>>collection_id:' + str(collection_id)
# print '>>>>start_time:' + str(start_time)
# print '>>>>end_time:' + str(end_time)
# print '>>>>bindings:' + str(bindings)
# content block数を返却する
# print '>>>>get_content_blocks_count exit'
# print '>>>>>>>:' + str(len(self.get_content_blocks(collection_id=collection_id,start_time=start_time,end_time=end_time)))
return len(self.get_content_blocks(collection_id=collection_id, start_time=start_time, end_time=end_time))

# collection_id から collection_name を取得する
Expand All @@ -212,14 +177,6 @@ def get_collection_name_from_collection_id(self, collection_id):

# poll時にコールされる
def get_content_blocks(self, collection_id=None, start_time=None, end_time=None, bindings=None, offset=0, limit=None):
# print '>>>>get_content_blocks enter'
# print '>>>>collection_id:' + str(collection_id)
# print '>>>>start_time:' + str(start_time)
# print '>>>>end_time:' + str(end_time)
# print '>>>>bindings:' + str(bindings)
# print '>>>>offset:' + str(offset)
# print '>>>>limit:' + str(limit)

collection_name = self.get_collection_name_from_collection_id(collection_id)
timestamp_label = datetime.datetime.now(dateutil.tz.tzutc())
message = 'Message from S-TIP'
Expand All @@ -230,15 +187,12 @@ def get_content_blocks(self, collection_id=None, start_time=None, end_time=None,
for stix_file in self.get_stix_files_from_mongo(collection_name, start_time, end_time)[offset:limit]:
content = stix_file.content.read()
# ここでフィルタリングする
stix_like_file = io.StringIO(content)
stix_like_file = io.StringIO(content.decode('utf-8'))
stix_package = STIXPackage.from_xml(stix_like_file)
# username によるフィルタリング
username = self.get_stip_sns_username(stix_package)
if username in self.black_account_list:
# username が black list に含まれる場合はskip
# print '>>> %s is skipped by black list filter:' + str(stix_package.stix_header.title)
# import sys
# sys.stdout.flush()
continue
stix_like_file.close()
cb = ContentBlockEntity(
Expand Down Expand Up @@ -273,19 +227,11 @@ def get_stip_sns_username(self, stix_package):
# それに対する制御を行う場合は api.py の中で実装しなくてはならない

def create_inbox_message(self, inbox_message_entity):
# print '>>>>create_inbox_message enter'
# print '>>>>create_inbox_message:inbox_message_entity:' + str(inbox_message_entity)
# print '<<<<create_inbox_message exit'
return inbox_message_entity

# push時にコールされる
# 引数はContentBlockEntity
def create_content_block(self, content_block_entity, collection_ids, service_id):
# print '>>>>create_content_block enter'
# print '>>>>create_content_block:content_block_entity:' + str(content_block_entity)
# print '>>>>create_content_block:collection_ids:' + str(collection_ids)
# print '>>>>create_content_block:service_id:' + str(service_id)
# print '<<<<create_content_block exit'
# content_balock_entity.contentに中身が格納される
# with open('/home/terra/work/libtaxii/server/output/output_BabyTiger.xml','w', encoding='utf-8') as fp:
# fp.write(content_block_entity.content)
Expand All @@ -303,32 +249,19 @@ def create_content_block(self, content_block_entity, collection_ids, service_id)
return content_block_entity

def create_result_set(self, result_set_entity):
# print '>>>>create_result_set enter'
# print '>>>>create_result_set:result_set_entity' + str(result_set_entity)
return result_set_entity

def create_subsciption(self, subscription_entity):
# print '>>>>create_subscription enter'
# print '>>>>create_subscription:subscription_entity' + str(subscription_entity)
return

def update_subscription(self, subscription_entity):
# print '>>>>update_subscription enter'
# print '>>>>create_subscription:subscription_entity' + str(subscription_entity)
return

def attach_collection_to_services(collection_id, service_ids):
# print '>>>>attach_collection_to_services enter'
# print '>>>>attach_collection_to_services:collection_id' + str(collection_id)
# print '>>>>attach_collection_to_services:service_ids' + str(service_ids)
return

def get_subscription(self, subscription_id):
# print '>>>>get_subscription enter'
# print '>>>>get_subscription:subscription_id' + str(subscription_id)
return

def get_subscriptions(self, service_id):
# print '>>>>get_subscriptions enter'
# print '>>>>get_subscriptions:service_id' + str(service_id)
return

0 comments on commit 7f28391

Please sign in to comment.