Merge pull request #480 from InQuest/release/2023.04.17

Release/2023.04.17 -> rc

.gitignore

@@ -70,3 +70,5 @@ dump.rdb
 # STS (Spring Tool Suite)
 .springBeans
 .vagrant
+
+version

app/__init__.py

@@ -213,7 +213,6 @@ def teardown_request(exception):
     db.session.remove()
 
 
-@app.before_first_request
 def setup_logging():
     app.logger.addHandler(logging.StreamHandler())
     app.logger.setLevel(app.config["LOGGING_LEVEL"])
@@ -314,7 +313,6 @@ def teardown_request(exception):
             db.session.rollback()
         db.session.remove()
 
-    @app.before_first_request
     def setup_logging():
         app.logger.addHandler(logging.StreamHandler())
         app.logger.setLevel(app.config["LOGGING_LEVEL"])
@@ -343,6 +341,9 @@ def load_user_from_request(request):
 
         return None
 
+    with app.app_context():
+        setup_logging()
+
 
 def run(debug=False, port=0, host=''):
     import os
@@ -352,3 +353,7 @@ def run(debug=False, port=0, host=''):
 
     generate_app()
     app.run(debug=debug, port=port, host=host)
+
+
+with app.app_context():
+    setup_logging()

app/models/users.py

@@ -1,4 +1,4 @@
-from itsdangerous import (JSONWebSignatureSerializer
+from itsdangerous import (URLSafeTimedSerializer
                           as Serializer, BadSignature)
 
 from app import db

app/routes/files.py

@@ -254,9 +254,7 @@ def get_file_for_entity(entity_type, entity_id, file_id):
     if not file_entity:
         abort(404)
 
-    full_path = os.path.join(cfg_settings.Cfg_settings.get_setting("FILE_STORE_PATH"),
-                             str(ENTITY_MAPPING[entity_type]) if entity_type != "0" else "",
-                             str(entity_id) if entity_id != 0 else "",
+    full_path = os.path.join(file_entity.full_path,
                              secure_filename(file_entity.filename))
     if not os.path.exists(full_path):
         abort(404)

app/routes/version.py

@@ -11,7 +11,10 @@ def get_version():
     Return: version number"""
     try:
         version = [v.strip() for v in open("version", "r").readlines()]
+        return Response(
+            json.dumps({"version": version[0], "version_email": version[1], "version_date": version[2][:-6]}),
+            mimetype='application/json')
     except:
-        version = ["unavailable", "unavailable", "unavailable"]
-    return Response(json.dumps({"version": version[0], "version_email": version[1], "version_date": version[2][:-6]}),
-                    mimetype='application/json')
+        return Response(
+            json.dumps({"version": "unavailable", "version_email": "unavailable", "version_date": "unavailable"}),
+            mimetype='application/json')

app/routes/yara_rules.py

@@ -599,7 +599,7 @@ def update_yara_rule(id):
     if tags_to_create:
         create_tags_mapping(entity.__tablename__, entity.id, tags_to_create)
 
-    return jsonify(entity.to_dict()), 200
+    return jsonify(entity.to_dict(include_yara_rule_string=True)), 200
 
 
 @app.route('/ThreatKB/yara_rules/batch/edit', methods=['PUT'])

app/static/js/yara_rule/yara_rule-controller.js

@@ -703,8 +703,8 @@ angular.module('ThreatKB')
             $scope.revisionIsOpen = false;
 
         }])
-    .controller('Yara_ruleSaveController', ['$scope', '$http', '$cookies', '$uibModalInstance', '$location', '$window', 'yara_rule', 'yara_rules', 'metadata', 'Cfg_states', 'Comments', 'Upload', 'Files', 'CfgCategoryRangeMapping', 'growl', 'Users', 'Tags', 'Yara_rule', 'Cfg_settings', 'Bookmarks', 'hotkeys',
-        function ($scope, $http, $cookies, $uibModalInstance, $location, $window, yara_rule, yara_rules, metadata, Cfg_states, Comments, Upload, Files, CfgCategoryRangeMapping, growl, Users, Tags, Yara_rule, Cfg_settings, Bookmarks, hotkeys) {
+    .controller('Yara_ruleSaveController', ['$scope', '$http', '$cookies', '$uibModal', '$uibModalInstance', '$location', '$window', 'yara_rule', 'yara_rules', 'metadata', 'Cfg_states', 'Comments', 'Upload', 'Files', 'CfgCategoryRangeMapping', 'growl', 'Users', 'Tags', 'Yara_rule', 'Cfg_settings', 'Bookmarks', 'hotkeys',
+        function ($scope, $http, $cookies, $uibModal, $uibModalInstance, $location, $window, yara_rule, yara_rules, metadata, Cfg_states, Comments, Upload, Files, CfgCategoryRangeMapping, growl, Users, Tags, Yara_rule, Cfg_settings, Bookmarks, hotkeys) {
 
             if (yara_rule.$promise !== null && yara_rule.$promise !== undefined) {
                 yara_rule.$promise.then(
@@ -713,7 +713,16 @@ angular.module('ThreatKB')
                     }
                 );
             }
-
+            let category_prefixes = Cfg_settings.get({key: "CATEGORY_PREFIX_MAPPING"});
+            if (category_prefixes.$promise !== null && category_prefixes.$promise !== undefined) {
+                category_prefixes.$promise.then(
+                    function (category_prefixes) {
+                        if (category_prefixes !== null && category_prefixes.value !== null) {
+                            $scope.category_prefixes = JSON.parse(category_prefixes.value);
+                        }
+                    }
+                );
+            }
 
             var mitre_techniques = Cfg_settings.get({key: "MITRE_TECHNIQUES"});
             if (mitre_techniques.$promise !== null && mitre_techniques.$promise !== undefined) {
@@ -784,6 +793,7 @@ angular.module('ThreatKB')
                             growl.error(error.data, {ttl: -1});
                         } else {
                             $scope.yara_rule.state = data.state;
+                            $scope.yara_rule.yara_rule_string = data.yara_rule_string;
                             growl.info("Successfully saved signature '" + $scope.yara_rule.name + "'.", {ttl: 2000});
                         }
                     },
@@ -912,7 +922,22 @@ angular.module('ThreatKB')
                 });
             };
 
-
+            $scope.closeAndViewRevision = function(id) {
+                $window.document.title = "ThreatKB";
+                $uibModalInstance.close($scope.yara_rule);
+                $window.location.href = $location.absUrl().replace(/\/[0-9]+$/, "");
+                $uibModal.open({
+                    templateUrl: 'yara_rule-revision.html',
+                    controller: 'Yara_ruleRevisionViewController',
+                    size: 'lg',
+                    backdrop: 'static',
+                    resolve: {
+                        yara_rule: function () {
+                            return $scope.yara_rule;
+                        }
+                    }
+                });
+            }
 
             $scope.$watch('files', function () {
                 $scope.upload($scope.files);
@@ -978,13 +1003,30 @@ angular.module('ThreatKB')
             };
 
             $scope.ok = function () {
-                // Check if outstanding comment
-                if ($scope.yara_rule.new_comment && $scope.yara_rule.new_comment.trim() && confirm('There is a unsaved comment, do you wish to save it as well?')) {
-                    $scope.add_comment($scope.yara_rule.id);
+                // Ensure that Rule name starts with Category Prefix
+                let category;
+                let validationPassed = true;
+                if ($scope.yara_rule.category && (typeof $scope.yara_rule.category === 'object')) {
+                    category = $scope.yara_rule.category.category;
+                } else {
+                    category = $scope.yara_rule.category;
+                }
+                if ($scope.category_prefixes.hasOwnProperty(category)) {
+                    let expectedPrefix = $scope.category_prefixes[category] + "_";
+                    if (!$scope.yara_rule.name.startsWith(expectedPrefix)) {
+                        alert("Rule Name " + $scope.yara_rule.name + " must start with " + expectedPrefix + " based on selected Category.");
+                        validationPassed = false;
+                    }
+                }
+                if (validationPassed) {
+                    // Check if outstanding comment
+                    if ($scope.yara_rule.new_comment && $scope.yara_rule.new_comment.trim() && confirm('There is a unsaved comment, do you wish to save it as well?')) {
+                        $scope.add_comment($scope.yara_rule.id);
+                    }
+                    // Close modal
+                    $window.document.title = "ThreatKB";
+                    $uibModalInstance.close($scope.yara_rule);
                 }
-                // Close modal
-                $window.document.title = "ThreatKB";
-                $uibModalInstance.close($scope.yara_rule);
             };
 
             $scope.cancel = function () {
@@ -1167,7 +1209,7 @@ angular.module('ThreatKB')
                 if (isNaN(parseInt(last_spot, 10))) {
                     $window.location.href = $location.absUrl() + "/" + id;
                     return;
-                } else if (!isNaN(parseInt(last_spot, 10)) && last_spot !== id) {
+                } else if (!isNaN(parseInt(last_spot, 10)) && parseInt(last_spot) !== id) {
                     $window.location.href = $location.absUrl().replace(/\/[0-9]+$/, "/" + id);
                     return;
                 }

app/static/views/c2dns/c2dns.html

@@ -80,13 +80,13 @@ <h4 class="modal-title" style="float: left; margin-right: 10px;">
             <button type="button" class="close"
                     ng-click="cancel()">&times;
             </button>
-            <button class="btn btn-sm  btn-primary" ng-if="c2dns.id"
+            <span class="btn btn-sm  btn-primary" ng-if="c2dns.id"
                     title="Copy DNS to clipboard" tooltip-placement="bottom"
                     uib-tooltip="Copied DNS!"
                     tooltip-trigger="'outsideClick'"
                     ngclipboard data-clipboard-text="{{ c2dns.domain_name }}">
                 Copy To Clipboard
-            </button>
+            </span>
             <a class="btn btn-sm  btn-primary" ng-if="c2dns.id"
                ng-click="edit(c2dns.id)">Edit
             </a>
@@ -280,13 +280,13 @@ <h4 class="modal-title" id="myC2dnsLabel" style="float: left;">
                 </h4>
 
                 <div class="pull-right" ng-if="c2dns.id" style="margin: 0 35px 0 0;">
-                    <button class="btn btn-sm  btn-primary" ng-if="c2dns.id"
+                    <span class="btn btn-sm  btn-primary" ng-if="c2dns.id"
                             title="Copy DNS to clipboard" tooltip-placement="bottom"
                             uib-tooltip="Copied DNS!"
                             tooltip-trigger="'outsideClick'"
                             ngclipboard data-clipboard-text="{{ c2dns.domain_name }}">
                         Copy To Clipboard
-                    </button>
+                    </span>
 
                     <a class="btn btn-sm btn-primary modal-fullscreen-button"
                        ng-click="toggleFullscreen()">Toggle full screen</a>

app/static/views/c2ip/c2ips.html

@@ -97,13 +97,13 @@ <h4 class="modal-title" style="float: left; margin-right: 10px;">
             <button type="button" class="close"
                     ng-click="cancel()">&times;
             </button>
-            <button class="btn btn-sm  btn-primary" ng-if="c2ip.id"
+            <span class="btn btn-sm  btn-primary" ng-if="c2ip.id"
                     title="Copy IP to clipboard" tooltip-placement="bottom"
                     uib-tooltip="Copied IP!"
                     tooltip-trigger="'outsideClick'"
                     ngclipboard data-clipboard-text="{{ c2ip.ip }}">
                 Copy To Clipboard
-            </button>
+            </span>
             <a class="btn btn-sm  btn-primary" ng-if="c2ip.id"
                ng-click="edit(c2ip.id)">Edit
             </a>
@@ -301,13 +301,13 @@ <h4 class="modal-title" id="myC2ipLabel" style="float: left;">
                 </h4>
 
                 <div class="pull-right" ng-if="c2ip.id" style="margin: 0 35px 0 0;">
-                    <button class="btn btn-sm  btn-primary" ng-if="c2ip.id"
+                    <span class="btn btn-sm  btn-primary" ng-if="c2ip.id"
                             title="Copy IP to clipboard" tooltip-placement="bottom"
                             uib-tooltip="Copied IP!"
                             tooltip-trigger="'outsideClick'"
                             ngclipboard data-clipboard-text="{{ c2ip.ip }}">
                         Copy To Clipboard
-                    </button>
+                    </span>
 
                     <a class="btn btn-sm btn-primary modal-fullscreen-button"
                        ng-click="toggleFullscreen()">Toggle full screen</a>

app/static/views/yara_rule/yara_rules.html

@@ -79,13 +79,13 @@ <h4 class="modal-title" style="float: left; margin-right: 10px;">
             <button type="button" class="close"
                     ng-click="cancel()">&times;
             </button>
-            <button class="btn btn-sm  btn-primary" ng-if="yara_rule.id"
+            <span class="btn btn-sm  btn-primary" ng-if="yara_rule.id"
                     title="Copy Yara Rule to clipboard" tooltip-placement="bottom"
                     uib-tooltip="Copied Yara Rule!"
                     tooltip-trigger="'outsideClick'"
                     ngclipboard data-clipboard-text="{{ yara_rule.yara_rule_string }}">
                 Copy To Clipboard
-            </button>
+            </span>
             <a class="btn btn-sm  btn-primary" ng-if="yara_rule.id"
                ng-click="edit(yara_rule.id)">Edit
             </a>
@@ -285,13 +285,13 @@ <h4 class="modal-title" id="myC2ipLabel" style="float: left">
 
                 <div class="pull-right" ng-if="yara_rule.id"
                      style="margin: 0 35px 0 0;">
-                    <button class="btn btn-sm  btn-primary" ng-if="yara_rule.id"
+                    <span class="btn btn-sm  btn-primary" ng-if="yara_rule.id"
                             title="Copy Yara Rule to clipboard" tooltip-placement="bottom"
                             uib-tooltip="Copied Yara Rule!"
                             tooltip-trigger="'outsideClick'"
                             ngclipboard data-clipboard-text="{{ yara_rule.yara_rule_string }}">
                         Copy To Clipboard
-                    </button>
+                    </span>
                     <button ng-if="negTestDir.value" type="button" class="btn btn-sm btn-primary"
                             ng-click="negTestSignature(yara_rule.id)">{{ testButtonTextNeg }}
                     </button>
@@ -684,6 +684,10 @@ <h4 class="modal-title" id="myC2ipLabel" style="float: left">
                         </uib-accordion>
                     </div>
                 </div>
+                <div ng-if="yara_rule.id">
+                    <button type="button" class="btn btn-primary" ng-click="closeAndViewRevision(yara_rule.id)">View Revisions
+                    </button>
+                </div>
             </div>
 
             <div class="modal-footer">
@@ -704,7 +708,6 @@ <h4 class="modal-title" id="myC2ipLabel" style="float: left">
                     </ui-select>
                 </div>
 
-
                 <div class="pull-right">
                     <button type="button" class="btn btn-default" ng-click="cancel()">Cancel
                     </button>

migrations/versions/65e3f200abd3_category_prefix_mapping.py

@@ -0,0 +1,47 @@
+"""Category Prefix Mapping
+
+Revision ID: 65e3f200abd3
+Revises: fc0cab9d77dc
+Create Date: 2023-05-31 21:05:32.214274
+
+"""
+import datetime
+
+from alembic import op
+from app.models import cfg_settings
+
+# revision identifiers, used by Alembic.
+revision = '65e3f200abd3'
+down_revision = 'fc0cab9d77dc'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    date_created = datetime.datetime.now().isoformat()
+    date_modified = datetime.datetime.now().isoformat()
+
+    op.bulk_insert(
+        cfg_settings.Cfg_settings.__table__,
+        [
+            {
+                "key": "CATEGORY_PREFIX_MAPPING",
+                "value": "{ \"Hunt Pivots\": \"HP\", \"Evasion Characteristics\": \"EC\", \"File Characteristics\": \"FC\", \"FileID\": \"FID\", \"Suspicious Characteristics\": \"SC\", \"Malicious Characteristics\": \"MC\", \"Header Analytics\": \"HA\", \"Light House\" : \"LH\", \"Data Loss\": \"DL\", \"Template\": \"UD\", \"Retention\": \"RET\" }",
+                "public": True,
+                "date_created": date_created,
+                "date_modified": date_modified,
+                "description": "The mapping between categories and their rule prefixes."
+            }
+        ]
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    keys = ["CATEGORY_PREFIX_MAPPING"]
+    for key in keys:
+        op.execute("""DELETE from cfg_settings where `key`='%s';""" % (key))
+
+    # ### end Alembic commands ###