From a03e6ab3c2aa5202ad81525c7e0107097beaf442 Mon Sep 17 00:00:00 2001 From: Toshitaka Satomi Date: Mon, 9 Mar 2020 11:59:46 +0900 Subject: [PATCH] Add a STIX 1.x filter and remove debug prints --- src/api.py | 71 ++---------------------------------------------------- 1 file changed, 2 insertions(+), 69 deletions(-) diff --git a/src/api.py b/src/api.py index 9111df8..0810b51 100644 --- a/src/api.py +++ b/src/api.py @@ -14,7 +14,6 @@ import tempfile import io import structlog -from pymongo import Connection from opentaxii.persistence.api import OpenTAXIIPersistenceAPI from opentaxii.taxii.entities import ( ServiceEntity, @@ -77,7 +76,6 @@ def __init__(self, self.black_account_list = black_account_list.split(',') else: self.black_account_list = [] - # print '>>>black_account_list :' +str(self.black_account_list) id = 0 for taxii_server in TaxiiServers.objects.all(): @@ -110,24 +108,16 @@ def __init__(self, id += 1 def init_app(self, app): - # appは pass # discovery/taxii-data時にはcollection_id = Noneでコールされる def get_services(self, collection_id=None): - # print '>>>>get_services enter' - # print '>>>>collection_id:' + str(collection_id) ret = self.services - # print '>>>>get_services ret:' + str(ret) - # print '>>>>get_services exit' return ret # taxii-data時にコールされる # push時にもコールされる def get_collections(self, service_id): - # print '>>>>get_collections enter' - # print '>>>>service_id:' + str(service_id) - # opentaxii.taxii.entities.CollectionEntityのリストを返却 s = None # sercice_idに該当するServiceEntityを検索する @@ -135,8 +125,6 @@ def get_collections(self, service_id): if service.id == service_id: s = service break - # print '>>>>service:' + str(s) - ret = [] if s is not None: # ServiceEntityと関連付けされているCollectionEntityを検索する @@ -145,41 +133,27 @@ def get_collections(self, service_id): for col in self.collections: if col.id == ref[service_id]: ret.append(col) - # print '>>>>ret:' + str(ret) - # print '>>>>get_collections exit' return ret # poll時にコールされる def get_collection(self, collection_name, service_id): - # print '>>>>get_collection enter' - # print '>>>>collection_name:' + str(collection_name) - # print '>>>>service_id:' + str(service_id) cols = self.get_collections(service_id) col = None for c in cols: if c.name == collection_name: col = c break - # print '>>>>return:' + str(col) - # print '>>>>get_collection exit' - # collection entityを返却する return col # mongo の stix_files コレクションより開始日時と終了日時の間 を find し cursor を返却する def get_stix_files_from_mongo(self, collection_name, start_time, end_time): - # print '>>>get_stix_files_from_mongo:start_time:' + str(start_time) - # print '>>>get_stix_files_from_mongo:end_time:' + str(end_time) - # print '>>>get_stix_files_from_mongo:collection_name:' + str(collection_name) - # TAXII Server 設定取得 ts = TaxiiServers.objects.get(collection_name=collection_name) # 条件絞り込み QQ = Q(information_source__in=ts.information_sources) - # for is_ in ts.information_sources: - # print '>>>get_stix_files_from_mongo:information_sources:' + str(is_.name) - # print '>>>get_stix_files_from_mongo:information_sources:' + str(is_.id) + QQ = QQ & Q (version__startswith='1.') if start_time is not None and end_time is not None: QQ = QQ & Q(produced__gt=start_time) QQ = QQ & Q(produced__lt=end_time) @@ -187,19 +161,10 @@ def get_stix_files_from_mongo(self, collection_name, start_time, end_time): QQ = QQ & Q(produced__lt=end_time) elif start_time is not None and end_time is None: QQ = QQ & Q(produced__gt=start_time) - # print '>>>get_stix_files_from_mongo:count:' + str(StixFiles.objects(QQ).count()) return StixFiles.objects(QQ) # poll時にコールされる def get_content_blocks_count(self, collection_id=None, start_time=None, end_time=None, bindings=None): - # print '>>>>get_content_blocks_count enter' - # print '>>>>collection_id:' + str(collection_id) - # print '>>>>start_time:' + str(start_time) - # print '>>>>end_time:' + str(end_time) - # print '>>>>bindings:' + str(bindings) - # content block数を返却する - # print '>>>>get_content_blocks_count exit' - # print '>>>>>>>:' + str(len(self.get_content_blocks(collection_id=collection_id,start_time=start_time,end_time=end_time))) return len(self.get_content_blocks(collection_id=collection_id, start_time=start_time, end_time=end_time)) # collection_id から collection_name を取得する @@ -212,14 +177,6 @@ def get_collection_name_from_collection_id(self, collection_id): # poll時にコールされる def get_content_blocks(self, collection_id=None, start_time=None, end_time=None, bindings=None, offset=0, limit=None): - # print '>>>>get_content_blocks enter' - # print '>>>>collection_id:' + str(collection_id) - # print '>>>>start_time:' + str(start_time) - # print '>>>>end_time:' + str(end_time) - # print '>>>>bindings:' + str(bindings) - # print '>>>>offset:' + str(offset) - # print '>>>>limit:' + str(limit) - collection_name = self.get_collection_name_from_collection_id(collection_id) timestamp_label = datetime.datetime.now(dateutil.tz.tzutc()) message = 'Message from S-TIP' @@ -230,15 +187,12 @@ def get_content_blocks(self, collection_id=None, start_time=None, end_time=None, for stix_file in self.get_stix_files_from_mongo(collection_name, start_time, end_time)[offset:limit]: content = stix_file.content.read() # ここでフィルタリングする - stix_like_file = io.StringIO(content) + stix_like_file = io.StringIO(content.decode('utf-8')) stix_package = STIXPackage.from_xml(stix_like_file) # username によるフィルタリング username = self.get_stip_sns_username(stix_package) if username in self.black_account_list: # username が black list に含まれる場合はskip - # print '>>> %s is skipped by black list filter:' + str(stix_package.stix_header.title) - # import sys - # sys.stdout.flush() continue stix_like_file.close() cb = ContentBlockEntity( @@ -273,19 +227,11 @@ def get_stip_sns_username(self, stix_package): # それに対する制御を行う場合は api.py の中で実装しなくてはならない def create_inbox_message(self, inbox_message_entity): - # print '>>>>create_inbox_message enter' - # print '>>>>create_inbox_message:inbox_message_entity:' + str(inbox_message_entity) - # print '<<<>>>create_content_block enter' - # print '>>>>create_content_block:content_block_entity:' + str(content_block_entity) - # print '>>>>create_content_block:collection_ids:' + str(collection_ids) - # print '>>>>create_content_block:service_id:' + str(service_id) - # print '<<<>>>create_result_set enter' - # print '>>>>create_result_set:result_set_entity' + str(result_set_entity) return result_set_entity def create_subsciption(self, subscription_entity): - # print '>>>>create_subscription enter' - # print '>>>>create_subscription:subscription_entity' + str(subscription_entity) return def update_subscription(self, subscription_entity): - # print '>>>>update_subscription enter' - # print '>>>>create_subscription:subscription_entity' + str(subscription_entity) return def attach_collection_to_services(collection_id, service_ids): - # print '>>>>attach_collection_to_services enter' - # print '>>>>attach_collection_to_services:collection_id' + str(collection_id) - # print '>>>>attach_collection_to_services:service_ids' + str(service_ids) return def get_subscription(self, subscription_id): - # print '>>>>get_subscription enter' - # print '>>>>get_subscription:subscription_id' + str(subscription_id) return def get_subscriptions(self, service_id): - # print '>>>>get_subscriptions enter' - # print '>>>>get_subscriptions:service_id' + str(service_id) return