diff --git a/LICENSE b/LICENSE index 5dffdb791b08a5..7fc997b4f04dc7 100644 --- a/LICENSE +++ b/LICENSE @@ -1035,6 +1035,109 @@ The externally maintained libraries used by Node.js are: OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ +- inspector_protocol, located at tools/inspector_protocol, is licensed as follows: + """ + // Copyright 2016 The Chromium Authors. All rights reserved. + // + // Redistribution and use in source and binary forms, with or without + // modification, are permitted provided that the following conditions are + // met: + // + // * Redistributions of source code must retain the above copyright + // notice, this list of conditions and the following disclaimer. + // * Redistributions in binary form must reproduce the above + // copyright notice, this list of conditions and the following disclaimer + // in the documentation and/or other materials provided with the + // distribution. + // * Neither the name of Google Inc. nor the names of its + // contributors may be used to endorse or promote products derived from + // this software without specific prior written permission. + // + // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + """ + +- jinja2, located at tools/jinja2, is licensed as follows: + """ + Copyright (c) 2009 by the Jinja Team, see AUTHORS for more details. + + Some rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * The names of the contributors may not be used to endorse or + promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + """ + +- markupsafe, located at tools/markupsafe, is licensed as follows: + """ + Copyright (c) 2010 by Armin Ronacher and contributors. See AUTHORS + for more details. + + Some rights reserved. + + Redistribution and use in source and binary forms of the software as well + as documentation, with or without modification, are permitted provided + that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * The names of the contributors may not be used to endorse or + promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND + CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT + NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE AND DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + DAMAGE. + """ + - cpplint.py, located at tools/cpplint.py, is licensed as follows: """ Copyright (c) 2009 Google Inc. All rights reserved. diff --git a/node.gyp b/node.gyp index e1e2e7cd54f636..125afa8c268d49 100644 --- a/node.gyp +++ b/node.gyp @@ -1044,7 +1044,7 @@ }], # end aix section [ 'v8_enable_inspector==1', { 'variables': { - 'protocol_path': 'deps/v8/third_party/inspector_protocol', + 'protocol_path': 'tools/inspector_protocol', 'node_inspector_path': 'src/inspector', 'node_inspector_generated_sources': [ '<(SHARED_INTERMEDIATE_DIR)/src/node/inspector/protocol/Forward.h', @@ -1104,7 +1104,7 @@ ], 'action': [ 'python', - 'deps/v8/third_party/inspector_protocol/ConvertProtocolToJSON.py', + 'tools/inspector_protocol/ConvertProtocolToJSON.py', '<@(_inputs)', '<@(_outputs)', ], @@ -1159,7 +1159,7 @@ ], 'action': [ 'python', - 'deps/v8/third_party/inspector_protocol/ConvertProtocolToJSON.py', + 'tools/inspector_protocol/ConvertProtocolToJSON.py', '<@(_inputs)', '<@(_outputs)', ], @@ -1175,7 +1175,7 @@ ], 'action': [ 'python', - 'deps/v8/third_party/inspector_protocol/ConcatenateProtocols.py', + 'tools/inspector_protocol/ConcatenateProtocols.py', '<@(_inputs)', '<@(_outputs)', ], diff --git a/tools/inspector_protocol/CheckProtocolCompatibility.py b/tools/inspector_protocol/CheckProtocolCompatibility.py new file mode 100755 index 00000000000000..c70162a2a44ef0 --- /dev/null +++ b/tools/inspector_protocol/CheckProtocolCompatibility.py @@ -0,0 +1,479 @@ +#!/usr/bin/env python +# Copyright (c) 2011 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Inspector protocol validator. +# +# Tests that subsequent protocol changes are not breaking backwards compatibility. +# Following violations are reported: +# +# - Domain has been removed +# - Command has been removed +# - Required command parameter was added or changed from optional +# - Required response parameter was removed or changed to optional +# - Event has been removed +# - Required event parameter was removed or changed to optional +# - Parameter type has changed. +# +# For the parameters with composite types the above checks are also applied +# recursively to every property of the type. +# +# Adding --show_changes to the command line prints out a list of valid public API changes. + +import copy +import os.path +import optparse +import sys + +try: + import json +except ImportError: + import simplejson as json + + +def list_to_map(items, key): + result = {} + for item in items: + if "experimental" not in item and "hidden" not in item: + result[item[key]] = item + return result + + +def named_list_to_map(container, name, key): + if name in container: + return list_to_map(container[name], key) + return {} + + +def removed(reverse): + if reverse: + return "added" + return "removed" + + +def required(reverse): + if reverse: + return "optional" + return "required" + + +def compare_schemas(d_1, d_2, reverse): + errors = [] + domains_1 = copy.deepcopy(d_1) + domains_2 = copy.deepcopy(d_2) + types_1 = normalize_types_in_schema(domains_1) + types_2 = normalize_types_in_schema(domains_2) + + domains_by_name_1 = list_to_map(domains_1, "domain") + domains_by_name_2 = list_to_map(domains_2, "domain") + + for name in domains_by_name_1: + domain_1 = domains_by_name_1[name] + if name not in domains_by_name_2: + errors.append("%s: domain has been %s" % (name, removed(reverse))) + continue + compare_domains(domain_1, domains_by_name_2[name], types_1, types_2, errors, reverse) + return errors + + +def compare_domains(domain_1, domain_2, types_map_1, types_map_2, errors, reverse): + domain_name = domain_1["domain"] + commands_1 = named_list_to_map(domain_1, "commands", "name") + commands_2 = named_list_to_map(domain_2, "commands", "name") + for name in commands_1: + command_1 = commands_1[name] + if name not in commands_2: + errors.append("%s.%s: command has been %s" % (domain_1["domain"], name, removed(reverse))) + continue + compare_commands(domain_name, command_1, commands_2[name], types_map_1, types_map_2, errors, reverse) + + events_1 = named_list_to_map(domain_1, "events", "name") + events_2 = named_list_to_map(domain_2, "events", "name") + for name in events_1: + event_1 = events_1[name] + if name not in events_2: + errors.append("%s.%s: event has been %s" % (domain_1["domain"], name, removed(reverse))) + continue + compare_events(domain_name, event_1, events_2[name], types_map_1, types_map_2, errors, reverse) + + +def compare_commands(domain_name, command_1, command_2, types_map_1, types_map_2, errors, reverse): + context = domain_name + "." + command_1["name"] + + params_1 = named_list_to_map(command_1, "parameters", "name") + params_2 = named_list_to_map(command_2, "parameters", "name") + # Note the reversed order: we allow removing but forbid adding parameters. + compare_params_list(context, "parameter", params_2, params_1, types_map_2, types_map_1, 0, errors, not reverse) + + returns_1 = named_list_to_map(command_1, "returns", "name") + returns_2 = named_list_to_map(command_2, "returns", "name") + compare_params_list(context, "response parameter", returns_1, returns_2, types_map_1, types_map_2, 0, errors, reverse) + + +def compare_events(domain_name, event_1, event_2, types_map_1, types_map_2, errors, reverse): + context = domain_name + "." + event_1["name"] + params_1 = named_list_to_map(event_1, "parameters", "name") + params_2 = named_list_to_map(event_2, "parameters", "name") + compare_params_list(context, "parameter", params_1, params_2, types_map_1, types_map_2, 0, errors, reverse) + + +def compare_params_list(context, kind, params_1, params_2, types_map_1, types_map_2, depth, errors, reverse): + for name in params_1: + param_1 = params_1[name] + if name not in params_2: + if "optional" not in param_1: + errors.append("%s.%s: required %s has been %s" % (context, name, kind, removed(reverse))) + continue + + param_2 = params_2[name] + if param_2 and "optional" in param_2 and "optional" not in param_1: + errors.append("%s.%s: %s %s is now %s" % (context, name, required(reverse), kind, required(not reverse))) + continue + type_1 = extract_type(param_1, types_map_1, errors) + type_2 = extract_type(param_2, types_map_2, errors) + compare_types(context + "." + name, kind, type_1, type_2, types_map_1, types_map_2, depth, errors, reverse) + + +def compare_types(context, kind, type_1, type_2, types_map_1, types_map_2, depth, errors, reverse): + if depth > 5: + return + + base_type_1 = type_1["type"] + base_type_2 = type_2["type"] + + if base_type_1 != base_type_2: + errors.append("%s: %s base type mismatch, '%s' vs '%s'" % (context, kind, base_type_1, base_type_2)) + elif base_type_1 == "object": + params_1 = named_list_to_map(type_1, "properties", "name") + params_2 = named_list_to_map(type_2, "properties", "name") + # If both parameters have the same named type use it in the context. + if "id" in type_1 and "id" in type_2 and type_1["id"] == type_2["id"]: + type_name = type_1["id"] + else: + type_name = "" + context += " %s->%s" % (kind, type_name) + compare_params_list(context, "property", params_1, params_2, types_map_1, types_map_2, depth + 1, errors, reverse) + elif base_type_1 == "array": + item_type_1 = extract_type(type_1["items"], types_map_1, errors) + item_type_2 = extract_type(type_2["items"], types_map_2, errors) + compare_types(context, kind, item_type_1, item_type_2, types_map_1, types_map_2, depth + 1, errors, reverse) + + +def extract_type(typed_object, types_map, errors): + if "type" in typed_object: + result = {"id": "", "type": typed_object["type"]} + if typed_object["type"] == "object": + result["properties"] = [] + elif typed_object["type"] == "array": + result["items"] = typed_object["items"] + return result + elif "$ref" in typed_object: + ref = typed_object["$ref"] + if ref not in types_map: + errors.append("Can not resolve type: %s" % ref) + types_map[ref] = {"id": "", "type": "object"} + return types_map[ref] + + +def normalize_types_in_schema(domains): + types = {} + for domain in domains: + domain_name = domain["domain"] + normalize_types(domain, domain_name, types) + return types + + +def normalize_types(obj, domain_name, types): + if isinstance(obj, list): + for item in obj: + normalize_types(item, domain_name, types) + elif isinstance(obj, dict): + for key, value in obj.items(): + if key == "$ref" and value.find(".") == -1: + obj[key] = "%s.%s" % (domain_name, value) + elif key == "id": + obj[key] = "%s.%s" % (domain_name, value) + types[obj[key]] = obj + else: + normalize_types(value, domain_name, types) + + +def load_schema(file_name, domains): + # pylint: disable=W0613 + if not os.path.isfile(file_name): + return + input_file = open(file_name, "r") + json_string = input_file.read() + parsed_json = json.loads(json_string) + domains += parsed_json["domains"] + return parsed_json["version"] + + +def self_test(): + def create_test_schema_1(): + return [ + { + "domain": "Network", + "types": [ + { + "id": "LoaderId", + "type": "string" + }, + { + "id": "Headers", + "type": "object" + }, + { + "id": "Request", + "type": "object", + "properties": [ + {"name": "url", "type": "string"}, + {"name": "method", "type": "string"}, + {"name": "headers", "$ref": "Headers"}, + {"name": "becameOptionalField", "type": "string"}, + {"name": "removedField", "type": "string"}, + ] + } + ], + "commands": [ + { + "name": "removedCommand", + }, + { + "name": "setExtraHTTPHeaders", + "parameters": [ + {"name": "headers", "$ref": "Headers"}, + {"name": "mismatched", "type": "string"}, + {"name": "becameOptional", "$ref": "Headers"}, + {"name": "removedRequired", "$ref": "Headers"}, + {"name": "becameRequired", "$ref": "Headers", "optional": True}, + {"name": "removedOptional", "$ref": "Headers", "optional": True}, + ], + "returns": [ + {"name": "mimeType", "type": "string"}, + {"name": "becameOptional", "type": "string"}, + {"name": "removedRequired", "type": "string"}, + {"name": "becameRequired", "type": "string", "optional": True}, + {"name": "removedOptional", "type": "string", "optional": True}, + ] + } + ], + "events": [ + { + "name": "requestWillBeSent", + "parameters": [ + {"name": "frameId", "type": "string", "experimental": True}, + {"name": "request", "$ref": "Request"}, + {"name": "becameOptional", "type": "string"}, + {"name": "removedRequired", "type": "string"}, + {"name": "becameRequired", "type": "string", "optional": True}, + {"name": "removedOptional", "type": "string", "optional": True}, + ] + }, + { + "name": "removedEvent", + "parameters": [ + {"name": "errorText", "type": "string"}, + {"name": "canceled", "type": "boolean", "optional": True} + ] + } + ] + }, + { + "domain": "removedDomain" + } + ] + + def create_test_schema_2(): + return [ + { + "domain": "Network", + "types": [ + { + "id": "LoaderId", + "type": "string" + }, + { + "id": "Request", + "type": "object", + "properties": [ + {"name": "url", "type": "string"}, + {"name": "method", "type": "string"}, + {"name": "headers", "type": "object"}, + {"name": "becameOptionalField", "type": "string", "optional": True}, + ] + } + ], + "commands": [ + { + "name": "addedCommand", + }, + { + "name": "setExtraHTTPHeaders", + "parameters": [ + {"name": "headers", "type": "object"}, + {"name": "mismatched", "type": "object"}, + {"name": "becameOptional", "type": "object", "optional": True}, + {"name": "addedRequired", "type": "object"}, + {"name": "becameRequired", "type": "object"}, + {"name": "addedOptional", "type": "object", "optional": True}, + ], + "returns": [ + {"name": "mimeType", "type": "string"}, + {"name": "becameOptional", "type": "string", "optional": True}, + {"name": "addedRequired", "type": "string"}, + {"name": "becameRequired", "type": "string"}, + {"name": "addedOptional", "type": "string", "optional": True}, + ] + } + ], + "events": [ + { + "name": "requestWillBeSent", + "parameters": [ + {"name": "request", "$ref": "Request"}, + {"name": "becameOptional", "type": "string", "optional": True}, + {"name": "addedRequired", "type": "string"}, + {"name": "becameRequired", "type": "string"}, + {"name": "addedOptional", "type": "string", "optional": True}, + ] + }, + { + "name": "addedEvent" + } + ] + }, + { + "domain": "addedDomain" + } + ] + + expected_errors = [ + "removedDomain: domain has been removed", + "Network.removedCommand: command has been removed", + "Network.removedEvent: event has been removed", + "Network.setExtraHTTPHeaders.mismatched: parameter base type mismatch, 'object' vs 'string'", + "Network.setExtraHTTPHeaders.addedRequired: required parameter has been added", + "Network.setExtraHTTPHeaders.becameRequired: optional parameter is now required", + "Network.setExtraHTTPHeaders.removedRequired: required response parameter has been removed", + "Network.setExtraHTTPHeaders.becameOptional: required response parameter is now optional", + "Network.requestWillBeSent.removedRequired: required parameter has been removed", + "Network.requestWillBeSent.becameOptional: required parameter is now optional", + "Network.requestWillBeSent.request parameter->Network.Request.removedField: required property has been removed", + "Network.requestWillBeSent.request parameter->Network.Request.becameOptionalField: required property is now optional", + ] + + expected_errors_reverse = [ + "addedDomain: domain has been added", + "Network.addedEvent: event has been added", + "Network.addedCommand: command has been added", + "Network.setExtraHTTPHeaders.mismatched: parameter base type mismatch, 'string' vs 'object'", + "Network.setExtraHTTPHeaders.removedRequired: required parameter has been removed", + "Network.setExtraHTTPHeaders.becameOptional: required parameter is now optional", + "Network.setExtraHTTPHeaders.addedRequired: required response parameter has been added", + "Network.setExtraHTTPHeaders.becameRequired: optional response parameter is now required", + "Network.requestWillBeSent.becameRequired: optional parameter is now required", + "Network.requestWillBeSent.addedRequired: required parameter has been added", + ] + + def is_subset(subset, superset, message): + for i in range(len(subset)): + if subset[i] not in superset: + sys.stderr.write("%s error: %s\n" % (message, subset[i])) + return False + return True + + def errors_match(expected, actual): + return (is_subset(actual, expected, "Unexpected") and + is_subset(expected, actual, "Missing")) + + return (errors_match(expected_errors, + compare_schemas(create_test_schema_1(), create_test_schema_2(), False)) and + errors_match(expected_errors_reverse, + compare_schemas(create_test_schema_2(), create_test_schema_1(), True))) + + +def load_domains_and_baselines(file_name, domains, baseline_domains): + version = load_schema(os.path.normpath(file_name), domains) + suffix = "-%s.%s.json" % (version["major"], version["minor"]) + baseline_file = file_name.replace(".json", suffix) + load_schema(os.path.normpath(baseline_file), baseline_domains) + return version + + +def main(): + if not self_test(): + sys.stderr.write("Self-test failed") + return 1 + + cmdline_parser = optparse.OptionParser() + cmdline_parser.add_option("--show_changes") + cmdline_parser.add_option("--expected_errors") + cmdline_parser.add_option("--stamp") + arg_options, arg_values = cmdline_parser.parse_args() + + if len(arg_values) < 1: + sys.stderr.write("Usage: %s [--show_changes] [, ...]\n" % sys.argv[0]) + return 1 + + domains = [] + baseline_domains = [] + version = load_domains_and_baselines(arg_values[0], domains, baseline_domains) + for dependency in arg_values[1:]: + load_domains_and_baselines(dependency, domains, baseline_domains) + + expected_errors = [] + if arg_options.expected_errors: + expected_errors_file = open(arg_options.expected_errors, "r") + expected_errors = json.loads(expected_errors_file.read())["errors"] + expected_errors_file.close() + + errors = compare_schemas(baseline_domains, domains, False) + unexpected_errors = [] + for i in range(len(errors)): + if errors[i] not in expected_errors: + unexpected_errors.append(errors[i]) + if len(unexpected_errors) > 0: + sys.stderr.write(" Compatibility checks FAILED\n") + for error in unexpected_errors: + sys.stderr.write(" %s\n" % error) + return 1 + + if arg_options.show_changes: + changes = compare_schemas(domains, baseline_domains, True) + if len(changes) > 0: + print " Public changes since %s:" % version + for change in changes: + print " %s" % change + + if arg_options.stamp: + with open(arg_options.stamp, 'a') as _: + pass + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/inspector_protocol/CodeGenerator.py b/tools/inspector_protocol/CodeGenerator.py new file mode 100644 index 00000000000000..e630b02985710f --- /dev/null +++ b/tools/inspector_protocol/CodeGenerator.py @@ -0,0 +1,654 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os.path +import sys +import optparse +import collections +import functools +import re +import copy +try: + import json +except ImportError: + import simplejson as json + +# Path handling for libraries and templates +# Paths have to be normalized because Jinja uses the exact template path to +# determine the hash used in the cache filename, and we need a pre-caching step +# to be concurrency-safe. Use absolute path because __file__ is absolute if +# module is imported, and relative if executed directly. +# If paths differ between pre-caching and individual file compilation, the cache +# is regenerated, which causes a race condition and breaks concurrent build, +# since some compile processes will try to read the partially written cache. +module_path, module_filename = os.path.split(os.path.realpath(__file__)) + +def read_config(): + # pylint: disable=W0703 + def json_to_object(data, output_base, config_base): + def json_object_hook(object_dict): + items = [(k, os.path.join(config_base, v) if k == "path" else v) for (k, v) in object_dict.items()] + items = [(k, os.path.join(output_base, v) if k == "output" else v) for (k, v) in items] + keys, values = zip(*items) + return collections.namedtuple('X', keys)(*values) + return json.loads(data, object_hook=json_object_hook) + + def init_defaults(config_tuple, path, defaults): + keys = list(config_tuple._fields) # pylint: disable=E1101 + values = [getattr(config_tuple, k) for k in keys] + for i in xrange(len(keys)): + if hasattr(values[i], "_fields"): + values[i] = init_defaults(values[i], path + "." + keys[i], defaults) + for optional in defaults: + if optional.find(path + ".") != 0: + continue + optional_key = optional[len(path) + 1:] + if optional_key.find(".") == -1 and optional_key not in keys: + keys.append(optional_key) + values.append(defaults[optional]) + return collections.namedtuple('X', keys)(*values) + + try: + cmdline_parser = optparse.OptionParser() + cmdline_parser.add_option("--output_base") + cmdline_parser.add_option("--jinja_dir") + cmdline_parser.add_option("--config") + cmdline_parser.add_option("--config_value", action="append", type="string") + arg_options, _ = cmdline_parser.parse_args() + jinja_dir = arg_options.jinja_dir + if not jinja_dir: + raise Exception("jinja directory must be specified") + jinja_dir = jinja_dir.decode('utf8') + output_base = arg_options.output_base + if not output_base: + raise Exception("Base output directory must be specified") + output_base = output_base.decode('utf8') + config_file = arg_options.config + if not config_file: + raise Exception("Config file name must be specified") + config_file = config_file.decode('utf8') + config_base = os.path.dirname(config_file) + config_values = arg_options.config_value + if not config_values: + config_values = [] + except Exception: + # Work with python 2 and 3 http://docs.python.org/py3k/howto/pyporting.html + exc = sys.exc_info()[1] + sys.stderr.write("Failed to parse command-line arguments: %s\n\n" % exc) + exit(1) + + try: + config_json_file = open(config_file, "r") + config_json_string = config_json_file.read() + config_partial = json_to_object(config_json_string, output_base, config_base) + config_json_file.close() + defaults = { + ".use_snake_file_names": False, + ".use_title_case_methods": False, + ".imported": False, + ".imported.export_macro": "", + ".imported.export_header": False, + ".imported.header": False, + ".imported.package": False, + ".imported.options": False, + ".protocol.export_macro": "", + ".protocol.export_header": False, + ".protocol.options": False, + ".exported": False, + ".exported.export_macro": "", + ".exported.export_header": False, + ".lib": False, + ".lib.export_macro": "", + ".lib.export_header": False, + } + for key_value in config_values: + parts = key_value.split("=") + if len(parts) == 2: + defaults["." + parts[0]] = parts[1] + return (jinja_dir, config_file, init_defaults(config_partial, "", defaults)) + except Exception: + # Work with python 2 and 3 http://docs.python.org/py3k/howto/pyporting.html + exc = sys.exc_info()[1] + sys.stderr.write("Failed to parse config file: %s\n\n" % exc) + exit(1) + + +# ---- Begin of utilities exposed to generator ---- + + +def to_title_case(name): + return name[:1].upper() + name[1:] + + +def dash_to_camelcase(word): + prefix = "" + if word[0] == "-": + prefix = "Negative" + word = word[1:] + return prefix + "".join(to_title_case(x) or "-" for x in word.split("-")) + + +def to_snake_case(name): + return re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", name, sys.maxint).lower() + + +def to_method_case(config, name): + if config.use_title_case_methods: + return to_title_case(name) + return name + + +def join_arrays(dict, keys): + result = [] + for key in keys: + if key in dict: + result += dict[key] + return result + + +def format_include(config, header, file_name=None): + if file_name is not None: + header = header + "/" + file_name + ".h" + header = "\"" + header + "\"" if header[0] not in "<\"" else header + if config.use_snake_file_names: + header = to_snake_case(header) + return header + + +def to_file_name(config, file_name): + if config.use_snake_file_names: + return to_snake_case(file_name).replace(".cpp", ".cc") + return file_name + + +# ---- End of utilities exposed to generator ---- + + +def initialize_jinja_env(jinja_dir, cache_dir, config): + # pylint: disable=F0401 + sys.path.insert(1, os.path.abspath(jinja_dir)) + import jinja2 + + jinja_env = jinja2.Environment( + loader=jinja2.FileSystemLoader(module_path), + # Bytecode cache is not concurrency-safe unless pre-cached: + # if pre-cached this is read-only, but writing creates a race condition. + bytecode_cache=jinja2.FileSystemBytecodeCache(cache_dir), + keep_trailing_newline=True, # newline-terminate generated files + lstrip_blocks=True, # so can indent control flow tags + trim_blocks=True) + jinja_env.filters.update({"to_title_case": to_title_case, "dash_to_camelcase": dash_to_camelcase, "to_method_case": functools.partial(to_method_case, config)}) + jinja_env.add_extension("jinja2.ext.loopcontrols") + return jinja_env + + +def create_imported_type_definition(domain_name, type, imported_namespace): + # pylint: disable=W0622 + return { + "return_type": "std::unique_ptr<%s::%s::API::%s>" % (imported_namespace, domain_name, type["id"]), + "pass_type": "std::unique_ptr<%s::%s::API::%s>" % (imported_namespace, domain_name, type["id"]), + "to_raw_type": "%s.get()", + "to_pass_type": "std::move(%s)", + "to_rvalue": "std::move(%s)", + "type": "std::unique_ptr<%s::%s::API::%s>" % (imported_namespace, domain_name, type["id"]), + "raw_type": "%s::%s::API::%s" % (imported_namespace, domain_name, type["id"]), + "raw_pass_type": "%s::%s::API::%s*" % (imported_namespace, domain_name, type["id"]), + "raw_return_type": "%s::%s::API::%s*" % (imported_namespace, domain_name, type["id"]), + } + + +def create_user_type_definition(domain_name, type): + # pylint: disable=W0622 + return { + "return_type": "std::unique_ptr" % (domain_name, type["id"]), + "pass_type": "std::unique_ptr" % (domain_name, type["id"]), + "to_raw_type": "%s.get()", + "to_pass_type": "std::move(%s)", + "to_rvalue": "std::move(%s)", + "type": "std::unique_ptr" % (domain_name, type["id"]), + "raw_type": "protocol::%s::%s" % (domain_name, type["id"]), + "raw_pass_type": "protocol::%s::%s*" % (domain_name, type["id"]), + "raw_return_type": "protocol::%s::%s*" % (domain_name, type["id"]), + } + + +def create_object_type_definition(): + # pylint: disable=W0622 + return { + "return_type": "std::unique_ptr", + "pass_type": "std::unique_ptr", + "to_raw_type": "%s.get()", + "to_pass_type": "std::move(%s)", + "to_rvalue": "std::move(%s)", + "type": "std::unique_ptr", + "raw_type": "protocol::DictionaryValue", + "raw_pass_type": "protocol::DictionaryValue*", + "raw_return_type": "protocol::DictionaryValue*", + } + + +def create_any_type_definition(): + # pylint: disable=W0622 + return { + "return_type": "std::unique_ptr", + "pass_type": "std::unique_ptr", + "to_raw_type": "%s.get()", + "to_pass_type": "std::move(%s)", + "to_rvalue": "std::move(%s)", + "type": "std::unique_ptr", + "raw_type": "protocol::Value", + "raw_pass_type": "protocol::Value*", + "raw_return_type": "protocol::Value*", + } + + +def create_string_type_definition(): + # pylint: disable=W0622 + return { + "return_type": "String", + "pass_type": "const String&", + "to_pass_type": "%s", + "to_raw_type": "%s", + "to_rvalue": "%s", + "type": "String", + "raw_type": "String", + "raw_pass_type": "const String&", + "raw_return_type": "String", + } + + +def create_primitive_type_definition(type): + # pylint: disable=W0622 + typedefs = { + "number": "double", + "integer": "int", + "boolean": "bool" + } + defaults = { + "number": "0", + "integer": "0", + "boolean": "false" + } + jsontypes = { + "number": "TypeDouble", + "integer": "TypeInteger", + "boolean": "TypeBoolean", + } + return { + "return_type": typedefs[type], + "pass_type": typedefs[type], + "to_pass_type": "%s", + "to_raw_type": "%s", + "to_rvalue": "%s", + "type": typedefs[type], + "raw_type": typedefs[type], + "raw_pass_type": typedefs[type], + "raw_return_type": typedefs[type], + "default_value": defaults[type] + } + + +def wrap_array_definition(type): + # pylint: disable=W0622 + return { + "return_type": "std::unique_ptr>" % type["raw_type"], + "pass_type": "std::unique_ptr>" % type["raw_type"], + "to_raw_type": "%s.get()", + "to_pass_type": "std::move(%s)", + "to_rvalue": "std::move(%s)", + "type": "std::unique_ptr>" % type["raw_type"], + "raw_type": "protocol::Array<%s>" % type["raw_type"], + "raw_pass_type": "protocol::Array<%s>*" % type["raw_type"], + "raw_return_type": "protocol::Array<%s>*" % type["raw_type"], + "out_type": "protocol::Array<%s>&" % type["raw_type"], + } + + +class Protocol(object): + def __init__(self, config): + self.config = config + self.json_api = {"domains": []} + self.imported_domains = [] + self.exported_domains = [] + self.generate_domains = self.read_protocol_file(config.protocol.path) + + if config.protocol.options: + self.generate_domains = [rule.domain for rule in config.protocol.options] + self.exported_domains = [rule.domain for rule in config.protocol.options if hasattr(rule, "exported")] + + if config.imported: + self.imported_domains = self.read_protocol_file(config.imported.path) + if config.imported.options: + self.imported_domains = [rule.domain for rule in config.imported.options] + + self.patch_full_qualified_refs() + self.create_notification_types() + self.create_type_definitions() + self.generate_used_types() + + + def read_protocol_file(self, file_name): + input_file = open(file_name, "r") + json_string = input_file.read() + input_file.close() + parsed_json = json.loads(json_string) + version = parsed_json["version"]["major"] + "." + parsed_json["version"]["minor"] + domains = [] + for domain in parsed_json["domains"]: + domains.append(domain["domain"]) + domain["version"] = version + self.json_api["domains"] += parsed_json["domains"] + return domains + + + def patch_full_qualified_refs(self): + def patch_full_qualified_refs_in_domain(json, domain_name): + if isinstance(json, list): + for item in json: + patch_full_qualified_refs_in_domain(item, domain_name) + if not isinstance(json, dict): + return + for key in json: + if key == "type" and json[key] == "string": + json[key] = domain_name + ".string" + if key != "$ref": + patch_full_qualified_refs_in_domain(json[key], domain_name) + continue + if json["$ref"].find(".") == -1: + json["$ref"] = domain_name + "." + json["$ref"] + return + + for domain in self.json_api["domains"]: + patch_full_qualified_refs_in_domain(domain, domain["domain"]) + + + def all_references(self, json): + refs = set() + if isinstance(json, list): + for item in json: + refs |= self.all_references(item) + if not isinstance(json, dict): + return refs + for key in json: + if key != "$ref": + refs |= self.all_references(json[key]) + else: + refs.add(json["$ref"]) + return refs + + def generate_used_types(self): + all_refs = set() + for domain in self.json_api["domains"]: + domain_name = domain["domain"] + if "commands" in domain: + for command in domain["commands"]: + if self.generate_command(domain_name, command["name"]): + all_refs |= self.all_references(command) + if "events" in domain: + for event in domain["events"]: + if self.generate_event(domain_name, event["name"]): + all_refs |= self.all_references(event) + all_refs.add(domain_name + "." + to_title_case(event["name"]) + "Notification") + + dependencies = self.generate_type_dependencies() + queue = set(all_refs) + while len(queue): + ref = queue.pop() + if ref in dependencies: + queue |= dependencies[ref] - all_refs + all_refs |= dependencies[ref] + self.used_types = all_refs + + + def generate_type_dependencies(self): + dependencies = dict() + domains_with_types = (x for x in self.json_api["domains"] if "types" in x) + for domain in domains_with_types: + domain_name = domain["domain"] + for type in domain["types"]: + related_types = self.all_references(type) + if len(related_types): + dependencies[domain_name + "." + type["id"]] = related_types + return dependencies + + + def create_notification_types(self): + for domain in self.json_api["domains"]: + if "events" in domain: + for event in domain["events"]: + event_type = dict() + event_type["description"] = "Wrapper for notification params" + event_type["type"] = "object" + event_type["id"] = to_title_case(event["name"]) + "Notification" + if "parameters" in event: + event_type["properties"] = copy.deepcopy(event["parameters"]) + if "types" not in domain: + domain["types"] = list() + domain["types"].append(event_type) + + + def create_type_definitions(self): + imported_namespace = "::".join(self.config.imported.namespace) if self.config.imported else "" + self.type_definitions = {} + self.type_definitions["number"] = create_primitive_type_definition("number") + self.type_definitions["integer"] = create_primitive_type_definition("integer") + self.type_definitions["boolean"] = create_primitive_type_definition("boolean") + self.type_definitions["object"] = create_object_type_definition() + self.type_definitions["any"] = create_any_type_definition() + for domain in self.json_api["domains"]: + self.type_definitions[domain["domain"] + ".string"] = create_string_type_definition() + if not ("types" in domain): + continue + for type in domain["types"]: + type_name = domain["domain"] + "." + type["id"] + if type["type"] == "object" and domain["domain"] in self.imported_domains: + self.type_definitions[type_name] = create_imported_type_definition(domain["domain"], type, imported_namespace) + elif type["type"] == "object": + self.type_definitions[type_name] = create_user_type_definition(domain["domain"], type) + elif type["type"] == "array": + items_type = type["items"]["type"] + self.type_definitions[type_name] = wrap_array_definition(self.type_definitions[items_type]) + elif type["type"] == domain["domain"] + ".string": + self.type_definitions[type_name] = create_string_type_definition() + else: + self.type_definitions[type_name] = create_primitive_type_definition(type["type"]) + + + def check_options(self, options, domain, name, include_attr, exclude_attr, default): + for rule in options: + if rule.domain != domain: + continue + if include_attr and hasattr(rule, include_attr): + return name in getattr(rule, include_attr) + if exclude_attr and hasattr(rule, exclude_attr): + return name not in getattr(rule, exclude_attr) + return default + return False + + + # ---- Begin of methods exposed to generator + + + def type_definition(self, name): + return self.type_definitions[name] + + + def resolve_type(self, prop): + if "$ref" in prop: + return self.type_definitions[prop["$ref"]] + if prop["type"] == "array": + return wrap_array_definition(self.resolve_type(prop["items"])) + return self.type_definitions[prop["type"]] + + + def generate_command(self, domain, command): + if not self.config.protocol.options: + return domain in self.generate_domains + return self.check_options(self.config.protocol.options, domain, command, "include", "exclude", True) + + + def generate_event(self, domain, event): + if not self.config.protocol.options: + return domain in self.generate_domains + return self.check_options(self.config.protocol.options, domain, event, "include_events", "exclude_events", True) + + + def generate_type(self, domain, typename): + return domain + "." + typename in self.used_types + + + def is_async_command(self, domain, command): + if not self.config.protocol.options: + return False + return self.check_options(self.config.protocol.options, domain, command, "async", None, False) + + + def is_exported(self, domain, name): + if not self.config.protocol.options: + return False + return self.check_options(self.config.protocol.options, domain, name, "exported", None, False) + + + def is_imported(self, domain, name): + if not self.config.imported: + return False + if not self.config.imported.options: + return domain in self.imported_domains + return self.check_options(self.config.imported.options, domain, name, "imported", None, False) + + + def is_exported_domain(self, domain): + return domain in self.exported_domains + + + def generate_disable(self, domain): + if "commands" not in domain: + return True + for command in domain["commands"]: + if command["name"] == "disable" and self.generate_command(domain["domain"], "disable"): + return False + return True + + + def is_imported_dependency(self, domain): + return domain in self.generate_domains or domain in self.imported_domains + + +def main(): + jinja_dir, config_file, config = read_config() + + protocol = Protocol(config) + + if not config.exported and len(protocol.exported_domains): + sys.stderr.write("Domains [%s] are exported, but config is missing export entry\n\n" % ", ".join(protocol.exported_domains)) + exit(1) + + if not os.path.exists(config.protocol.output): + os.mkdir(config.protocol.output) + if len(protocol.exported_domains) and not os.path.exists(config.exported.output): + os.mkdir(config.exported.output) + jinja_env = initialize_jinja_env(jinja_dir, config.protocol.output, config) + + inputs = [] + inputs.append(__file__) + inputs.append(config_file) + inputs.append(config.protocol.path) + if config.imported: + inputs.append(config.imported.path) + templates_dir = os.path.join(module_path, "templates") + inputs.append(os.path.join(templates_dir, "TypeBuilder_h.template")) + inputs.append(os.path.join(templates_dir, "TypeBuilder_cpp.template")) + inputs.append(os.path.join(templates_dir, "Exported_h.template")) + inputs.append(os.path.join(templates_dir, "Imported_h.template")) + + h_template = jinja_env.get_template("templates/TypeBuilder_h.template") + cpp_template = jinja_env.get_template("templates/TypeBuilder_cpp.template") + exported_template = jinja_env.get_template("templates/Exported_h.template") + imported_template = jinja_env.get_template("templates/Imported_h.template") + + outputs = dict() + + for domain in protocol.json_api["domains"]: + class_name = domain["domain"] + template_context = { + "protocol": protocol, + "config": config, + "domain": domain, + "join_arrays": join_arrays, + "format_include": functools.partial(format_include, config), + } + + if domain["domain"] in protocol.generate_domains: + outputs[os.path.join(config.protocol.output, to_file_name(config, class_name + ".h"))] = h_template.render(template_context) + outputs[os.path.join(config.protocol.output, to_file_name(config, class_name + ".cpp"))] = cpp_template.render(template_context) + if domain["domain"] in protocol.exported_domains: + outputs[os.path.join(config.exported.output, to_file_name(config, class_name + ".h"))] = exported_template.render(template_context) + if domain["domain"] in protocol.imported_domains: + outputs[os.path.join(config.protocol.output, to_file_name(config, class_name + ".h"))] = imported_template.render(template_context) + + if config.lib: + template_context = { + "config": config, + "format_include": functools.partial(format_include, config), + } + + lib_templates_dir = os.path.join(module_path, "lib") + # Note these should be sorted in the right order. + # TODO(dgozman): sort them programmatically based on commented includes. + lib_h_templates = [ + "Collections_h.template", + "ErrorSupport_h.template", + "Values_h.template", + "Object_h.template", + "ValueConversions_h.template", + "Maybe_h.template", + "Array_h.template", + "DispatcherBase_h.template", + "Parser_h.template", + ] + + lib_cpp_templates = [ + "Protocol_cpp.template", + "ErrorSupport_cpp.template", + "Values_cpp.template", + "Object_cpp.template", + "DispatcherBase_cpp.template", + "Parser_cpp.template", + ] + + forward_h_templates = [ + "Forward_h.template", + "Allocator_h.template", + "FrontendChannel_h.template", + ] + + def generate_lib_file(file_name, template_files): + parts = [] + for template_file in template_files: + inputs.append(os.path.join(lib_templates_dir, template_file)) + template = jinja_env.get_template("lib/" + template_file) + parts.append(template.render(template_context)) + outputs[file_name] = "\n\n".join(parts) + + generate_lib_file(os.path.join(config.lib.output, to_file_name(config, "Forward.h")), forward_h_templates) + generate_lib_file(os.path.join(config.lib.output, to_file_name(config, "Protocol.h")), lib_h_templates) + generate_lib_file(os.path.join(config.lib.output, to_file_name(config, "Protocol.cpp")), lib_cpp_templates) + + # Make gyp / make generatos happy, otherwise make rebuilds world. + inputs_ts = max(map(os.path.getmtime, inputs)) + up_to_date = True + for output_file in outputs.iterkeys(): + if not os.path.exists(output_file) or os.path.getmtime(output_file) < inputs_ts: + up_to_date = False + break + if up_to_date: + sys.exit() + + for file_name, content in outputs.iteritems(): + out_file = open(file_name, "w") + out_file.write(content) + out_file.close() + + +main() diff --git a/tools/inspector_protocol/ConcatenateProtocols.py b/tools/inspector_protocol/ConcatenateProtocols.py new file mode 100755 index 00000000000000..a7cbc992c76e40 --- /dev/null +++ b/tools/inspector_protocol/ConcatenateProtocols.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os.path +import sys + +try: + import json +except ImportError: + import simplejson as json + + +def main(argv): + if len(argv) < 1: + sys.stderr.write("Usage: %s [ [, ...]] \n" % sys.argv[0]) + return 1 + + domains = [] + version = None + for protocol in argv[:-1]: + file_name = os.path.normpath(protocol) + if not os.path.isfile(file_name): + sys.stderr.write("Cannot find %s\n" % file_name) + return 1 + input_file = open(file_name, "r") + json_string = input_file.read() + parsed_json = json.loads(json_string) + domains += parsed_json["domains"] + version = parsed_json["version"] + + output_file = open(argv[-1], "w") + json.dump({"version": version, "domains": domains}, output_file, indent=4, sort_keys=False, separators=(',', ': ')) + output_file.close() + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/tools/inspector_protocol/ConvertProtocolToJSON.py b/tools/inspector_protocol/ConvertProtocolToJSON.py new file mode 100644 index 00000000000000..56fc09d78cb18f --- /dev/null +++ b/tools/inspector_protocol/ConvertProtocolToJSON.py @@ -0,0 +1,183 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import collections +import json +import os.path +import re +import sys + +file_name = None +description = '' + +primitiveTypes = ['integer', 'number', 'boolean', 'string', 'object', 'any', 'array'] + + +def assignType(item, type, isArray=False): + if isArray: + item['type'] = 'array' + item['items'] = collections.OrderedDict() + assignType(item['items'], type) + return + + if type == 'enum': + type = 'string' + if type in primitiveTypes: + item['type'] = type + else: + item['$ref'] = type + + +def createItem(d, experimental, deprecated, name=None): + result = collections.OrderedDict(d) + if name: + result['name'] = name + global description + if description: + result['description'] = description.strip() + if experimental: + result['experimental'] = True + if deprecated: + result['deprecated'] = True + return result + + +def parse(data): + protocol = collections.OrderedDict() + protocol['version'] = collections.OrderedDict() + protocol['domains'] = [] + domain = None + item = None + subitems = None + nukeDescription = False + global description + lines = data.split('\n') + for i in range(0, len(lines)): + if nukeDescription: + description = '' + nukeDescription = False + line = lines[i] + trimLine = line.strip() + + if trimLine.startswith('#'): + if len(description): + description += '\n' + description += trimLine[2:] + continue + else: + nukeDescription = True + + if len(trimLine) == 0: + continue + + match = re.compile('^(experimental )?(deprecated )?domain (.*)').match(line) + if match: + domain = createItem({'domain' : match.group(3)}, match.group(1), match.group(2)) + protocol['domains'].append(domain) + continue + + match = re.compile('^ depends on ([^\s]+)').match(line) + if match: + if 'dependencies' not in domain: + domain['dependencies'] = [] + domain['dependencies'].append(match.group(1)) + continue + + match = re.compile('^ (experimental )?(deprecated )?type (.*) extends (array of )?([^\s]+)').match(line) + if match: + if 'types' not in domain: + domain['types'] = [] + item = createItem({'id': match.group(3)}, match.group(1), match.group(2)) + assignType(item, match.group(5), match.group(4)) + domain['types'].append(item) + continue + + match = re.compile('^ (experimental )?(deprecated )?(command|event) (.*)').match(line) + if match: + list = [] + if match.group(3) == 'command': + if 'commands' in domain: + list = domain['commands'] + else: + list = domain['commands'] = [] + else: + if 'events' in domain: + list = domain['events'] + else: + list = domain['events'] = [] + + item = createItem({}, match.group(1), match.group(2), match.group(4)) + list.append(item) + continue + + match = re.compile('^ (experimental )?(deprecated )?(optional )?(array of )?([^\s]+) ([^\s]+)').match(line) + if match: + param = createItem({}, match.group(1), match.group(2), match.group(6)) + if match.group(3): + param['optional'] = True + assignType(param, match.group(5), match.group(4)) + if match.group(5) == 'enum': + enumliterals = param['enum'] = [] + subitems.append(param) + continue + + match = re.compile('^ (parameters|returns|properties)').match(line) + if match: + subitems = item[match.group(1)] = [] + continue + + match = re.compile('^ enum').match(line) + if match: + enumliterals = item['enum'] = [] + continue + + match = re.compile('^version').match(line) + if match: + continue + + match = re.compile('^ major (\d+)').match(line) + if match: + protocol['version']['major'] = match.group(1) + continue + + match = re.compile('^ minor (\d+)').match(line) + if match: + protocol['version']['minor'] = match.group(1) + continue + + match = re.compile('^ redirect ([^\s]+)').match(line) + if match: + item['redirect'] = match.group(1) + continue + + match = re.compile('^ ( )?[^\s]+$').match(line) + if match: + # enum literal + enumliterals.append(trimLine) + continue + + print 'Error in %s:%s, illegal token: \t%s' % (file_name, i, line) + sys.exit(1) + return protocol + +def main(argv): + if len(argv) < 2: + sys.stderr.write("Usage: %s \n" % sys.argv[0]) + return 1 + global file_name + file_name = os.path.normpath(argv[0]) + input_file = open(file_name, "r") + pdl_string = input_file.read() + protocol = parse(pdl_string) + output_file = open(argv[0].replace('.pdl', '.json'), 'wb') + json.dump(protocol, output_file, indent=4, separators=(',', ': ')) + output_file.close() + + output_file = open(os.path.normpath(argv[1]), 'wb') + json.dump(protocol, output_file, indent=4, separators=(',', ': ')) + output_file.close() + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/tools/inspector_protocol/LICENSE b/tools/inspector_protocol/LICENSE new file mode 100644 index 00000000000000..800468e5763479 --- /dev/null +++ b/tools/inspector_protocol/LICENSE @@ -0,0 +1,27 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tools/inspector_protocol/OWNERS b/tools/inspector_protocol/OWNERS new file mode 100644 index 00000000000000..8d0b6d90cb233b --- /dev/null +++ b/tools/inspector_protocol/OWNERS @@ -0,0 +1,8 @@ +set noparent + +alph@chromium.org +caseq@chromium.org +dgozman@chromium.org +kozyatinskiy@chromium.org +pfeldman@chromium.org +yangguo@chromium.org diff --git a/tools/inspector_protocol/README.v8 b/tools/inspector_protocol/README.v8 new file mode 100644 index 00000000000000..8a82f2a9c9d691 --- /dev/null +++ b/tools/inspector_protocol/README.v8 @@ -0,0 +1,16 @@ +Name: inspector protocol +Short Name: inspector_protocol +URL: https://chromium.googlesource.com/deps/inspector_protocol/ +Version: 0 +Revision: 752d4abd13119010cf30e454e8ef9b5fb7ef43a3 +License: BSD +License File: LICENSE +Security Critical: no + +Description: +src/inspector uses these scripts to generate handlers from protocol +description. + +Local modifications: +- This only includes the lib/ and templates/ directories, scripts, build + and the LICENSE files. diff --git a/tools/inspector_protocol/inspector_protocol.gni b/tools/inspector_protocol/inspector_protocol.gni new file mode 100644 index 00000000000000..5dcc1f522d8634 --- /dev/null +++ b/tools/inspector_protocol/inspector_protocol.gni @@ -0,0 +1,89 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# This template will generate inspector protocol source code. The code will +# not be compiled, use get_target_outputs() to compile them. +# +# Inputs +# +# config_file (required) +# Path to json file specifying inspector protocol configuration. +# +# out_dir (required) +# Path to put the generated files in. It must be inside output or +# generated file directory. +# +# outputs (required) +# Files generated. Relative to out_dir. +# +# inputs (optional) +# Extra inputs specified by the config file. +template("inspector_protocol_generate") { + assert(defined(invoker.config_file)) + assert(defined(invoker.out_dir)) + assert(defined(invoker.outputs)) + assert(defined(invoker.inspector_protocol_dir)) + inspector_protocol_dir = invoker.inspector_protocol_dir + + action(target_name) { + script = "$inspector_protocol_dir/CodeGenerator.py" + + inputs = [ + invoker.config_file, + "$inspector_protocol_dir/lib/Allocator_h.template", + "$inspector_protocol_dir/lib/Array_h.template", + "$inspector_protocol_dir/lib/Collections_h.template", + "$inspector_protocol_dir/lib/DispatcherBase_cpp.template", + "$inspector_protocol_dir/lib/DispatcherBase_h.template", + "$inspector_protocol_dir/lib/ErrorSupport_cpp.template", + "$inspector_protocol_dir/lib/ErrorSupport_h.template", + "$inspector_protocol_dir/lib/Forward_h.template", + "$inspector_protocol_dir/lib/FrontendChannel_h.template", + "$inspector_protocol_dir/lib/Maybe_h.template", + "$inspector_protocol_dir/lib/Object_cpp.template", + "$inspector_protocol_dir/lib/Object_h.template", + "$inspector_protocol_dir/lib/Parser_cpp.template", + "$inspector_protocol_dir/lib/Parser_h.template", + "$inspector_protocol_dir/lib/Protocol_cpp.template", + "$inspector_protocol_dir/lib/ValueConversions_h.template", + "$inspector_protocol_dir/lib/Values_cpp.template", + "$inspector_protocol_dir/lib/Values_h.template", + "$inspector_protocol_dir/templates/Exported_h.template", + "$inspector_protocol_dir/templates/Imported_h.template", + "$inspector_protocol_dir/templates/TypeBuilder_cpp.template", + "$inspector_protocol_dir/templates/TypeBuilder_h.template", + ] + if (defined(invoker.inputs)) { + inputs += invoker.inputs + } + + args = [ + "--jinja_dir", + rebase_path("//third_party/", root_build_dir), # jinja is in chromium's third_party + "--output_base", + rebase_path(invoker.out_dir, root_build_dir), + "--config", + rebase_path(invoker.config_file, root_build_dir), + ] + + if (defined(invoker.config_values)) { + foreach(value, invoker.config_values) { + args += [ + "--config_value", + value, + ] + } + } + + outputs = get_path_info(rebase_path(invoker.outputs, ".", invoker.out_dir), + "abspath") + + forward_variables_from(invoker, + [ + "visibility", + "deps", + "public_deps", + ]) + } +} diff --git a/tools/inspector_protocol/inspector_protocol.gypi b/tools/inspector_protocol/inspector_protocol.gypi new file mode 100644 index 00000000000000..1fb7119b5fa567 --- /dev/null +++ b/tools/inspector_protocol/inspector_protocol.gypi @@ -0,0 +1,33 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +{ + 'variables': { + 'inspector_protocol_files': [ + 'lib/Allocator_h.template', + 'lib/Array_h.template', + 'lib/Collections_h.template', + 'lib/DispatcherBase_cpp.template', + 'lib/DispatcherBase_h.template', + 'lib/ErrorSupport_cpp.template', + 'lib/ErrorSupport_h.template', + 'lib/Forward_h.template', + 'lib/FrontendChannel_h.template', + 'lib/Maybe_h.template', + 'lib/Object_cpp.template', + 'lib/Object_h.template', + 'lib/Parser_cpp.template', + 'lib/Parser_h.template', + 'lib/Protocol_cpp.template', + 'lib/ValueConversions_h.template', + 'lib/Values_cpp.template', + 'lib/Values_h.template', + 'templates/Exported_h.template', + 'templates/Imported_h.template', + 'templates/TypeBuilder_cpp.template', + 'templates/TypeBuilder_h.template', + 'CodeGenerator.py', + ] + } +} diff --git a/tools/inspector_protocol/lib/Allocator_h.template b/tools/inspector_protocol/lib/Allocator_h.template new file mode 100644 index 00000000000000..8f8109d695c597 --- /dev/null +++ b/tools/inspector_protocol/lib/Allocator_h.template @@ -0,0 +1,30 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_Allocator_h +#define {{"_".join(config.protocol.namespace)}}_Allocator_h + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +enum NotNullTagEnum { NotNullLiteral }; + +#define PROTOCOL_DISALLOW_NEW() \ + private: \ + void* operator new(size_t) = delete; \ + void* operator new(size_t, NotNullTagEnum, void*) = delete; \ + void* operator new(size_t, void*) = delete; \ + public: + +#define PROTOCOL_DISALLOW_COPY(ClassName) \ + private: \ + ClassName(const ClassName&) = delete; \ + ClassName& operator=(const ClassName&) = delete + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_Allocator_h) diff --git a/tools/inspector_protocol/lib/Array_h.template b/tools/inspector_protocol/lib/Array_h.template new file mode 100644 index 00000000000000..3854f6e5cd102e --- /dev/null +++ b/tools/inspector_protocol/lib/Array_h.template @@ -0,0 +1,136 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_Array_h +#define {{"_".join(config.protocol.namespace)}}_Array_h + +//#include "ErrorSupport.h" +//#include "Forward.h" +//#include "ValueConversions.h" +//#include "Values.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +template +class Array { +public: + static std::unique_ptr> create() + { + return std::unique_ptr>(new Array()); + } + + static std::unique_ptr> fromValue(protocol::Value* value, ErrorSupport* errors) + { + protocol::ListValue* array = ListValue::cast(value); + if (!array) { + errors->addError("array expected"); + return nullptr; + } + std::unique_ptr> result(new Array()); + errors->push(); + for (size_t i = 0; i < array->size(); ++i) { + errors->setName(StringUtil::fromInteger(i)); + std::unique_ptr item = ValueConversions::fromValue(array->at(i), errors); + result->m_vector.push_back(std::move(item)); + } + errors->pop(); + if (errors->hasErrors()) + return nullptr; + return result; + } + + void addItem(std::unique_ptr value) + { + m_vector.push_back(std::move(value)); + } + + size_t length() + { + return m_vector.size(); + } + + T* get(size_t index) + { + return m_vector[index].get(); + } + + std::unique_ptr toValue() + { + std::unique_ptr result = ListValue::create(); + for (auto& item : m_vector) + result->pushValue(ValueConversions::toValue(item)); + return result; + } + +private: + std::vector> m_vector; +}; + +template +class ArrayBase { +public: + static std::unique_ptr> create() + { + return std::unique_ptr>(new Array()); + } + + static std::unique_ptr> fromValue(protocol::Value* value, ErrorSupport* errors) + { + protocol::ListValue* array = ListValue::cast(value); + if (!array) { + errors->addError("array expected"); + return nullptr; + } + errors->push(); + std::unique_ptr> result(new Array()); + for (size_t i = 0; i < array->size(); ++i) { + errors->setName(StringUtil::fromInteger(i)); + T item = ValueConversions::fromValue(array->at(i), errors); + result->m_vector.push_back(item); + } + errors->pop(); + if (errors->hasErrors()) + return nullptr; + return result; + } + + void addItem(const T& value) + { + m_vector.push_back(value); + } + + size_t length() + { + return m_vector.size(); + } + + T get(size_t index) + { + return m_vector[index]; + } + + std::unique_ptr toValue() + { + std::unique_ptr result = ListValue::create(); + for (auto& item : m_vector) + result->pushValue(ValueConversions::toValue(item)); + return result; + } + +private: + std::vector m_vector; +}; + +template<> class Array : public ArrayBase {}; +template<> class Array : public ArrayBase {}; +template<> class Array : public ArrayBase {}; +template<> class Array : public ArrayBase {}; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_Array_h) diff --git a/tools/inspector_protocol/lib/Collections_h.template b/tools/inspector_protocol/lib/Collections_h.template new file mode 100644 index 00000000000000..7505a17bfa6e68 --- /dev/null +++ b/tools/inspector_protocol/lib/Collections_h.template @@ -0,0 +1,43 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_Collections_h +#define {{"_".join(config.protocol.namespace)}}_Collections_h + +#include {{format_include(config.protocol.package, "Forward")}} +#include + +#if defined(__APPLE__) && !defined(_LIBCPP_VERSION) +#include +#include + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +template using HashMap = std::map; +template using HashSet = std::set; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#else +#include +#include + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +template using HashMap = std::unordered_map; +template using HashSet = std::unordered_set; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // defined(__APPLE__) && !defined(_LIBCPP_VERSION) + +#endif // !defined({{"_".join(config.protocol.namespace)}}_Collections_h) diff --git a/tools/inspector_protocol/lib/DispatcherBase_cpp.template b/tools/inspector_protocol/lib/DispatcherBase_cpp.template new file mode 100644 index 00000000000000..cecef743bffcc6 --- /dev/null +++ b/tools/inspector_protocol/lib/DispatcherBase_cpp.template @@ -0,0 +1,354 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +//#include "DispatcherBase.h" +//#include "Parser.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +// static +DispatchResponse DispatchResponse::OK() +{ + DispatchResponse result; + result.m_status = kSuccess; + result.m_errorCode = kParseError; + return result; +} + +// static +DispatchResponse DispatchResponse::Error(const String& error) +{ + DispatchResponse result; + result.m_status = kError; + result.m_errorCode = kServerError; + result.m_errorMessage = error; + return result; +} + +// static +DispatchResponse DispatchResponse::InternalError() +{ + DispatchResponse result; + result.m_status = kError; + result.m_errorCode = kInternalError; + result.m_errorMessage = "Internal error"; + return result; +} + +// static +DispatchResponse DispatchResponse::InvalidParams(const String& error) +{ + DispatchResponse result; + result.m_status = kError; + result.m_errorCode = kInvalidParams; + result.m_errorMessage = error; + return result; +} + +// static +DispatchResponse DispatchResponse::FallThrough() +{ + DispatchResponse result; + result.m_status = kFallThrough; + result.m_errorCode = kParseError; + return result; +} + +// static +const char DispatcherBase::kInvalidParamsString[] = "Invalid parameters"; + +DispatcherBase::WeakPtr::WeakPtr(DispatcherBase* dispatcher) : m_dispatcher(dispatcher) { } + +DispatcherBase::WeakPtr::~WeakPtr() +{ + if (m_dispatcher) + m_dispatcher->m_weakPtrs.erase(this); +} + +DispatcherBase::Callback::Callback(std::unique_ptr backendImpl, int callId, int callbackId) + : m_backendImpl(std::move(backendImpl)) + , m_callId(callId) + , m_callbackId(callbackId) { } + +DispatcherBase::Callback::~Callback() = default; + +void DispatcherBase::Callback::dispose() +{ + m_backendImpl = nullptr; +} + +void DispatcherBase::Callback::sendIfActive(std::unique_ptr partialMessage, const DispatchResponse& response) +{ + if (!m_backendImpl || !m_backendImpl->get()) + return; + m_backendImpl->get()->sendResponse(m_callId, response, std::move(partialMessage)); + m_backendImpl = nullptr; +} + +void DispatcherBase::Callback::fallThroughIfActive() +{ + if (!m_backendImpl || !m_backendImpl->get()) + return; + m_backendImpl->get()->markFallThrough(m_callbackId); + m_backendImpl = nullptr; +} + +DispatcherBase::DispatcherBase(FrontendChannel* frontendChannel) + : m_frontendChannel(frontendChannel) + , m_lastCallbackId(0) + , m_lastCallbackFallThrough(false) { } + +DispatcherBase::~DispatcherBase() +{ + clearFrontend(); +} + +int DispatcherBase::nextCallbackId() +{ + m_lastCallbackFallThrough = false; + return ++m_lastCallbackId; +} + +void DispatcherBase::markFallThrough(int callbackId) +{ + DCHECK(callbackId == m_lastCallbackId); + m_lastCallbackFallThrough = true; +} + +void DispatcherBase::sendResponse(int callId, const DispatchResponse& response, std::unique_ptr result) +{ + if (!m_frontendChannel) + return; + if (response.status() == DispatchResponse::kError) { + reportProtocolError(callId, response.errorCode(), response.errorMessage(), nullptr); + return; + } + m_frontendChannel->sendProtocolResponse(callId, InternalResponse::createResponse(callId, std::move(result))); +} + +void DispatcherBase::sendResponse(int callId, const DispatchResponse& response) +{ + sendResponse(callId, response, DictionaryValue::create()); +} + +namespace { + +class ProtocolError : public Serializable { +public: + static std::unique_ptr createErrorResponse(int callId, DispatchResponse::ErrorCode code, const String& errorMessage, ErrorSupport* errors) + { + std::unique_ptr protocolError(new ProtocolError(code, errorMessage)); + protocolError->m_callId = callId; + protocolError->m_hasCallId = true; + if (errors && errors->hasErrors()) + protocolError->m_data = errors->errors(); + return protocolError; + } + + static std::unique_ptr createErrorNotification(DispatchResponse::ErrorCode code, const String& errorMessage) + { + return std::unique_ptr(new ProtocolError(code, errorMessage)); + } + + String serialize() override + { + std::unique_ptr error = DictionaryValue::create(); + error->setInteger("code", m_code); + error->setString("message", m_errorMessage); + if (m_data.length()) + error->setString("data", m_data); + std::unique_ptr message = DictionaryValue::create(); + message->setObject("error", std::move(error)); + if (m_hasCallId) + message->setInteger("id", m_callId); + return message->serialize(); + } + + ~ProtocolError() override {} + +private: + ProtocolError(DispatchResponse::ErrorCode code, const String& errorMessage) + : m_code(code) + , m_errorMessage(errorMessage) + { + } + + DispatchResponse::ErrorCode m_code; + String m_errorMessage; + String m_data; + int m_callId = 0; + bool m_hasCallId = false; +}; + +} // namespace + +static void reportProtocolErrorTo(FrontendChannel* frontendChannel, int callId, DispatchResponse::ErrorCode code, const String& errorMessage, ErrorSupport* errors) +{ + if (frontendChannel) + frontendChannel->sendProtocolResponse(callId, ProtocolError::createErrorResponse(callId, code, errorMessage, errors)); +} + +static void reportProtocolErrorTo(FrontendChannel* frontendChannel, DispatchResponse::ErrorCode code, const String& errorMessage) +{ + if (frontendChannel) + frontendChannel->sendProtocolNotification(ProtocolError::createErrorNotification(code, errorMessage)); +} + +void DispatcherBase::reportProtocolError(int callId, DispatchResponse::ErrorCode code, const String& errorMessage, ErrorSupport* errors) +{ + reportProtocolErrorTo(m_frontendChannel, callId, code, errorMessage, errors); +} + +void DispatcherBase::clearFrontend() +{ + m_frontendChannel = nullptr; + for (auto& weak : m_weakPtrs) + weak->dispose(); + m_weakPtrs.clear(); +} + +std::unique_ptr DispatcherBase::weakPtr() +{ + std::unique_ptr weak(new DispatcherBase::WeakPtr(this)); + m_weakPtrs.insert(weak.get()); + return weak; +} + +UberDispatcher::UberDispatcher(FrontendChannel* frontendChannel) + : m_frontendChannel(frontendChannel) + , m_fallThroughForNotFound(false) { } + +void UberDispatcher::setFallThroughForNotFound(bool fallThroughForNotFound) +{ + m_fallThroughForNotFound = fallThroughForNotFound; +} + +void UberDispatcher::registerBackend(const String& name, std::unique_ptr dispatcher) +{ + m_dispatchers[name] = std::move(dispatcher); +} + +void UberDispatcher::setupRedirects(const HashMap& redirects) +{ + for (const auto& pair : redirects) + m_redirects[pair.first] = pair.second; +} + +DispatchResponse::Status UberDispatcher::dispatch(std::unique_ptr parsedMessage, int* outCallId, String* outMethod) +{ + if (!parsedMessage) { + reportProtocolErrorTo(m_frontendChannel, DispatchResponse::kParseError, "Message must be a valid JSON"); + return DispatchResponse::kError; + } + std::unique_ptr messageObject = DictionaryValue::cast(std::move(parsedMessage)); + if (!messageObject) { + reportProtocolErrorTo(m_frontendChannel, DispatchResponse::kInvalidRequest, "Message must be an object"); + return DispatchResponse::kError; + } + + int callId = 0; + protocol::Value* callIdValue = messageObject->get("id"); + bool success = callIdValue && callIdValue->asInteger(&callId); + if (outCallId) + *outCallId = callId; + if (!success) { + reportProtocolErrorTo(m_frontendChannel, DispatchResponse::kInvalidRequest, "Message must have integer 'id' property"); + return DispatchResponse::kError; + } + + protocol::Value* methodValue = messageObject->get("method"); + String method; + success = methodValue && methodValue->asString(&method); + if (outMethod) + *outMethod = method; + if (!success) { + reportProtocolErrorTo(m_frontendChannel, callId, DispatchResponse::kInvalidRequest, "Message must have string 'method' property", nullptr); + return DispatchResponse::kError; + } + + HashMap::iterator redirectIt = m_redirects.find(method); + if (redirectIt != m_redirects.end()) + method = redirectIt->second; + + size_t dotIndex = StringUtil::find(method, "."); + if (dotIndex == StringUtil::kNotFound) { + if (m_fallThroughForNotFound) + return DispatchResponse::kFallThrough; + reportProtocolErrorTo(m_frontendChannel, callId, DispatchResponse::kMethodNotFound, "'" + method + "' wasn't found", nullptr); + return DispatchResponse::kError; + } + String domain = StringUtil::substring(method, 0, dotIndex); + auto it = m_dispatchers.find(domain); + if (it == m_dispatchers.end()) { + if (m_fallThroughForNotFound) + return DispatchResponse::kFallThrough; + reportProtocolErrorTo(m_frontendChannel, callId, DispatchResponse::kMethodNotFound, "'" + method + "' wasn't found", nullptr); + return DispatchResponse::kError; + } + return it->second->dispatch(callId, method, std::move(messageObject)); +} + +bool UberDispatcher::getCommandName(const String& message, String* method, std::unique_ptr* parsedMessage) +{ + std::unique_ptr value = StringUtil::parseJSON(message); + if (!value) { + reportProtocolErrorTo(m_frontendChannel, DispatchResponse::kParseError, "Message must be a valid JSON"); + return false; + } + + protocol::DictionaryValue* object = DictionaryValue::cast(value.get()); + if (!object) { + reportProtocolErrorTo(m_frontendChannel, DispatchResponse::kInvalidRequest, "Message must be an object"); + return false; + } + + if (!object->getString("method", method)) { + reportProtocolErrorTo(m_frontendChannel, DispatchResponse::kInvalidRequest, "Message must have string 'method' property"); + return false; + } + + parsedMessage->reset(DictionaryValue::cast(value.release())); + return true; +} + +UberDispatcher::~UberDispatcher() = default; + +// static +std::unique_ptr InternalResponse::createResponse(int callId, std::unique_ptr params) +{ + return std::unique_ptr(new InternalResponse(callId, String(), std::move(params))); +} + +// static +std::unique_ptr InternalResponse::createNotification(const String& notification, std::unique_ptr params) +{ + return std::unique_ptr(new InternalResponse(0, notification, std::move(params))); +} + +String InternalResponse::serialize() +{ + std::unique_ptr result = DictionaryValue::create(); + std::unique_ptr params(m_params ? std::move(m_params) : DictionaryValue::create()); + if (m_notification.length()) { + result->setString("method", m_notification); + result->setValue("params", SerializedValue::create(params->serialize())); + } else { + result->setInteger("id", m_callId); + result->setValue("result", SerializedValue::create(params->serialize())); + } + return result->serialize(); +} + +InternalResponse::InternalResponse(int callId, const String& notification, std::unique_ptr params) + : m_callId(callId) + , m_notification(notification) + , m_params(params ? std::move(params) : nullptr) +{ +} + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} diff --git a/tools/inspector_protocol/lib/DispatcherBase_h.template b/tools/inspector_protocol/lib/DispatcherBase_h.template new file mode 100644 index 00000000000000..d70a4afe71de2c --- /dev/null +++ b/tools/inspector_protocol/lib/DispatcherBase_h.template @@ -0,0 +1,173 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_DispatcherBase_h +#define {{"_".join(config.protocol.namespace)}}_DispatcherBase_h + +//#include "Collections.h" +//#include "ErrorSupport.h" +//#include "Forward.h" +//#include "Values.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +class WeakPtr; + +class {{config.lib.export_macro}} DispatchResponse { +public: + enum Status { + kSuccess = 0, + kError = 1, + kFallThrough = 2, + kAsync = 3 + }; + + enum ErrorCode { + kParseError = -32700, + kInvalidRequest = -32600, + kMethodNotFound = -32601, + kInvalidParams = -32602, + kInternalError = -32603, + kServerError = -32000, + }; + + Status status() const { return m_status; } + const String& errorMessage() const { return m_errorMessage; } + ErrorCode errorCode() const { return m_errorCode; } + bool isSuccess() const { return m_status == kSuccess; } + + static DispatchResponse OK(); + static DispatchResponse Error(const String&); + static DispatchResponse InternalError(); + static DispatchResponse InvalidParams(const String&); + static DispatchResponse FallThrough(); + +private: + Status m_status; + String m_errorMessage; + ErrorCode m_errorCode; +}; + +class {{config.lib.export_macro}} DispatcherBase { + PROTOCOL_DISALLOW_COPY(DispatcherBase); +public: + static const char kInvalidParamsString[]; + class {{config.lib.export_macro}} WeakPtr { + public: + explicit WeakPtr(DispatcherBase*); + ~WeakPtr(); + DispatcherBase* get() { return m_dispatcher; } + void dispose() { m_dispatcher = nullptr; } + + private: + DispatcherBase* m_dispatcher; + }; + + class {{config.lib.export_macro}} Callback { + public: + Callback(std::unique_ptr backendImpl, int callId, int callbackId); + virtual ~Callback(); + void dispose(); + + protected: + void sendIfActive(std::unique_ptr partialMessage, const DispatchResponse& response); + void fallThroughIfActive(); + + private: + std::unique_ptr m_backendImpl; + int m_callId; + int m_callbackId; + }; + + explicit DispatcherBase(FrontendChannel*); + virtual ~DispatcherBase(); + + virtual DispatchResponse::Status dispatch(int callId, const String& method, std::unique_ptr messageObject) = 0; + + void sendResponse(int callId, const DispatchResponse&, std::unique_ptr result); + void sendResponse(int callId, const DispatchResponse&); + + void reportProtocolError(int callId, DispatchResponse::ErrorCode, const String& errorMessage, ErrorSupport* errors); + void clearFrontend(); + + std::unique_ptr weakPtr(); + + int nextCallbackId(); + void markFallThrough(int callbackId); + bool lastCallbackFallThrough() { return m_lastCallbackFallThrough; } + +private: + FrontendChannel* m_frontendChannel; + protocol::HashSet m_weakPtrs; + int m_lastCallbackId; + bool m_lastCallbackFallThrough; +}; + +class {{config.lib.export_macro}} UberDispatcher { + PROTOCOL_DISALLOW_COPY(UberDispatcher); +public: + explicit UberDispatcher(FrontendChannel*); + void registerBackend(const String& name, std::unique_ptr); + void setupRedirects(const HashMap&); + DispatchResponse::Status dispatch(std::unique_ptr message, int* callId = nullptr, String* method = nullptr); + FrontendChannel* channel() { return m_frontendChannel; } + bool fallThroughForNotFound() { return m_fallThroughForNotFound; } + void setFallThroughForNotFound(bool); + bool getCommandName(const String& message, String* method, std::unique_ptr* parsedMessage); + virtual ~UberDispatcher(); + +private: + FrontendChannel* m_frontendChannel; + bool m_fallThroughForNotFound; + HashMap m_redirects; + protocol::HashMap> m_dispatchers; +}; + +class InternalResponse : public Serializable { + PROTOCOL_DISALLOW_COPY(InternalResponse); +public: + static std::unique_ptr createResponse(int callId, std::unique_ptr params); + static std::unique_ptr createNotification(const String& notification, std::unique_ptr params = nullptr); + + String serialize() override; + + ~InternalResponse() override {} + +private: + InternalResponse(int callId, const String& notification, std::unique_ptr params); + + int m_callId; + String m_notification; + std::unique_ptr m_params; +}; + +class InternalRawNotification : public Serializable { +public: + static std::unique_ptr create(const String& notification) + { + return std::unique_ptr(new InternalRawNotification(notification)); + } + ~InternalRawNotification() override {} + + String serialize() override + { + return m_notification; + } + +private: + explicit InternalRawNotification(const String& notification) + : m_notification(notification) + { + } + + String m_notification; +}; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_DispatcherBase_h) diff --git a/tools/inspector_protocol/lib/ErrorSupport_cpp.template b/tools/inspector_protocol/lib/ErrorSupport_cpp.template new file mode 100644 index 00000000000000..7b858b8dc48f37 --- /dev/null +++ b/tools/inspector_protocol/lib/ErrorSupport_cpp.template @@ -0,0 +1,71 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +//#include "ErrorSupport.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +ErrorSupport::ErrorSupport() { } +ErrorSupport::~ErrorSupport() { } + +void ErrorSupport::setName(const char* name) +{ + setName(String(name)); +} + +void ErrorSupport::setName(const String& name) +{ + DCHECK(m_path.size()); + m_path[m_path.size() - 1] = name; +} + +void ErrorSupport::push() +{ + m_path.push_back(String()); +} + +void ErrorSupport::pop() +{ + m_path.pop_back(); +} + +void ErrorSupport::addError(const char* error) +{ + addError(String(error)); +} + +void ErrorSupport::addError(const String& error) +{ + StringBuilder builder; + for (size_t i = 0; i < m_path.size(); ++i) { + if (i) + StringUtil::builderAppend(builder, '.'); + StringUtil::builderAppend(builder, m_path[i]); + } + StringUtil::builderAppend(builder, ": "); + StringUtil::builderAppend(builder, error); + m_errors.push_back(StringUtil::builderToString(builder)); +} + +bool ErrorSupport::hasErrors() +{ + return !!m_errors.size(); +} + +String ErrorSupport::errors() +{ + StringBuilder builder; + for (size_t i = 0; i < m_errors.size(); ++i) { + if (i) + StringUtil::builderAppend(builder, "; "); + StringUtil::builderAppend(builder, m_errors[i]); + } + return StringUtil::builderToString(builder); +} + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} diff --git a/tools/inspector_protocol/lib/ErrorSupport_h.template b/tools/inspector_protocol/lib/ErrorSupport_h.template new file mode 100644 index 00000000000000..083f2a5eb0d4d3 --- /dev/null +++ b/tools/inspector_protocol/lib/ErrorSupport_h.template @@ -0,0 +1,37 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_ErrorSupport_h +#define {{"_".join(config.protocol.namespace)}}_ErrorSupport_h + +//#include "Forward.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +class {{config.lib.export_macro}} ErrorSupport { +public: + ErrorSupport(); + ~ErrorSupport(); + + void push(); + void setName(const char*); + void setName(const String&); + void pop(); + void addError(const char*); + void addError(const String&); + bool hasErrors(); + String errors(); + +private: + std::vector m_path; + std::vector m_errors; +}; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_ErrorSupport_h) diff --git a/tools/inspector_protocol/lib/Forward_h.template b/tools/inspector_protocol/lib/Forward_h.template new file mode 100644 index 00000000000000..34d1c0d3e946cd --- /dev/null +++ b/tools/inspector_protocol/lib/Forward_h.template @@ -0,0 +1,37 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_Forward_h +#define {{"_".join(config.protocol.namespace)}}_Forward_h + +{% if config.lib.export_header %} +#include {{format_include(config.lib.export_header)}} +{% endif %} +#include {{format_include(config.lib.string_header)}} + +#include + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +template class Array; +class DictionaryValue; +class DispatchResponse; +class ErrorSupport; +class FundamentalValue; +class ListValue; +template class Maybe; +class Object; +using Response = DispatchResponse; +class SerializedValue; +class StringValue; +class UberDispatcher; +class Value; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_Forward_h) diff --git a/tools/inspector_protocol/lib/FrontendChannel_h.template b/tools/inspector_protocol/lib/FrontendChannel_h.template new file mode 100644 index 00000000000000..0454978b0c8e88 --- /dev/null +++ b/tools/inspector_protocol/lib/FrontendChannel_h.template @@ -0,0 +1,30 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_FrontendChannel_h +#define {{"_".join(config.protocol.namespace)}}_FrontendChannel_h + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +class {{config.lib.export_macro}} Serializable { +public: + virtual String serialize() = 0; + virtual ~Serializable() = default; +}; + +class {{config.lib.export_macro}} FrontendChannel { +public: + virtual ~FrontendChannel() { } + virtual void sendProtocolResponse(int callId, std::unique_ptr message) = 0; + virtual void sendProtocolNotification(std::unique_ptr message) = 0; + virtual void flushProtocolNotifications() = 0; +}; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_FrontendChannel_h) diff --git a/tools/inspector_protocol/lib/Maybe_h.template b/tools/inspector_protocol/lib/Maybe_h.template new file mode 100644 index 00000000000000..71593acd0e553d --- /dev/null +++ b/tools/inspector_protocol/lib/Maybe_h.template @@ -0,0 +1,86 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_Maybe_h +#define {{"_".join(config.protocol.namespace)}}_Maybe_h + +//#include "Forward.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +template +class Maybe { +public: + Maybe() : m_value() { } + Maybe(std::unique_ptr value) : m_value(std::move(value)) { } + Maybe(Maybe&& other) : m_value(std::move(other.m_value)) { } + void operator=(std::unique_ptr value) { m_value = std::move(value); } + T* fromJust() const { DCHECK(m_value); return m_value.get(); } + T* fromMaybe(T* defaultValue) const { return m_value ? m_value.get() : defaultValue; } + bool isJust() const { return !!m_value; } + std::unique_ptr takeJust() { DCHECK(m_value); return std::move(m_value); } +private: + std::unique_ptr m_value; +}; + +template +class MaybeBase { +public: + MaybeBase() : m_isJust(false) { } + MaybeBase(T value) : m_isJust(true), m_value(value) { } + MaybeBase(MaybeBase&& other) : m_isJust(other.m_isJust), m_value(std::move(other.m_value)) { } + void operator=(T value) { m_value = value; m_isJust = true; } + T fromJust() const { DCHECK(m_isJust); return m_value; } + T fromMaybe(const T& defaultValue) const { return m_isJust ? m_value : defaultValue; } + bool isJust() const { return m_isJust; } + T takeJust() { DCHECK(m_isJust); return m_value; } + +protected: + bool m_isJust; + T m_value; +}; + +template<> +class Maybe : public MaybeBase { +public: + Maybe() { } + Maybe(bool value) : MaybeBase(value) { } + Maybe(Maybe&& other) : MaybeBase(std::move(other)) { } + using MaybeBase::operator=; +}; + +template<> +class Maybe : public MaybeBase { +public: + Maybe() { } + Maybe(int value) : MaybeBase(value) { } + Maybe(Maybe&& other) : MaybeBase(std::move(other)) { } + using MaybeBase::operator=; +}; + +template<> +class Maybe : public MaybeBase { +public: + Maybe() { } + Maybe(double value) : MaybeBase(value) { } + Maybe(Maybe&& other) : MaybeBase(std::move(other)) { } + using MaybeBase::operator=; +}; + +template<> +class Maybe : public MaybeBase { +public: + Maybe() { } + Maybe(const String& value) : MaybeBase(value) { } + Maybe(Maybe&& other) : MaybeBase(std::move(other)) { } + using MaybeBase::operator=; +}; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_Maybe_h) diff --git a/tools/inspector_protocol/lib/Object_cpp.template b/tools/inspector_protocol/lib/Object_cpp.template new file mode 100644 index 00000000000000..91723a71e29ce4 --- /dev/null +++ b/tools/inspector_protocol/lib/Object_cpp.template @@ -0,0 +1,38 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +//#include "Object.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +std::unique_ptr Object::fromValue(protocol::Value* value, ErrorSupport* errors) +{ + protocol::DictionaryValue* dictionary = DictionaryValue::cast(value); + if (!dictionary) { + errors->addError("object expected"); + return nullptr; + } + dictionary = static_cast(dictionary->clone().release()); + return std::unique_ptr(new Object(std::unique_ptr(dictionary))); +} + +std::unique_ptr Object::toValue() const +{ + return DictionaryValue::cast(m_object->clone()); +} + +std::unique_ptr Object::clone() const +{ + return std::unique_ptr(new Object(DictionaryValue::cast(m_object->clone()))); +} + +Object::Object(std::unique_ptr object) : m_object(std::move(object)) { } + +Object::~Object() { } + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} diff --git a/tools/inspector_protocol/lib/Object_h.template b/tools/inspector_protocol/lib/Object_h.template new file mode 100644 index 00000000000000..f6ffc57659e148 --- /dev/null +++ b/tools/inspector_protocol/lib/Object_h.template @@ -0,0 +1,32 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_Object_h +#define {{"_".join(config.protocol.namespace)}}_Object_h + +//#include "ErrorSupport.h" +//#include "Forward.h" +//#include "Values.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +class {{config.lib.export_macro}} Object { +public: + static std::unique_ptr fromValue(protocol::Value*, ErrorSupport*); + ~Object(); + + std::unique_ptr toValue() const; + std::unique_ptr clone() const; +private: + explicit Object(std::unique_ptr); + std::unique_ptr m_object; +}; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_Object_h) diff --git a/tools/inspector_protocol/lib/Parser_cpp.template b/tools/inspector_protocol/lib/Parser_cpp.template new file mode 100644 index 00000000000000..f3dde5ac218e6f --- /dev/null +++ b/tools/inspector_protocol/lib/Parser_cpp.template @@ -0,0 +1,547 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +namespace { + +const int stackLimit = 1000; + +enum Token { + ObjectBegin, + ObjectEnd, + ArrayBegin, + ArrayEnd, + StringLiteral, + Number, + BoolTrue, + BoolFalse, + NullToken, + ListSeparator, + ObjectPairSeparator, + InvalidToken, +}; + +const char* const nullString = "null"; +const char* const trueString = "true"; +const char* const falseString = "false"; + +bool isASCII(uint16_t c) +{ + return !(c & ~0x7F); +} + +bool isSpaceOrNewLine(uint16_t c) +{ + return isASCII(c) && c <= ' ' && (c == ' ' || (c <= 0xD && c >= 0x9)); +} + +double charactersToDouble(const uint16_t* characters, size_t length, bool* ok) +{ + std::vector buffer; + buffer.reserve(length + 1); + for (size_t i = 0; i < length; ++i) { + if (!isASCII(characters[i])) { + *ok = false; + return 0; + } + buffer.push_back(static_cast(characters[i])); + } + buffer.push_back('\0'); + return StringUtil::toDouble(buffer.data(), length, ok); +} + +double charactersToDouble(const uint8_t* characters, size_t length, bool* ok) +{ + std::string buffer(reinterpret_cast(characters), length); + return StringUtil::toDouble(buffer.data(), length, ok); +} + +template +bool parseConstToken(const Char* start, const Char* end, const Char** tokenEnd, const char* token) +{ + while (start < end && *token != '\0' && *start++ == *token++) { } + if (*token != '\0') + return false; + *tokenEnd = start; + return true; +} + +template +bool readInt(const Char* start, const Char* end, const Char** tokenEnd, bool canHaveLeadingZeros) +{ + if (start == end) + return false; + bool haveLeadingZero = '0' == *start; + int length = 0; + while (start < end && '0' <= *start && *start <= '9') { + ++start; + ++length; + } + if (!length) + return false; + if (!canHaveLeadingZeros && length > 1 && haveLeadingZero) + return false; + *tokenEnd = start; + return true; +} + +template +bool parseNumberToken(const Char* start, const Char* end, const Char** tokenEnd) +{ + // We just grab the number here. We validate the size in DecodeNumber. + // According to RFC4627, a valid number is: [minus] int [frac] [exp] + if (start == end) + return false; + Char c = *start; + if ('-' == c) + ++start; + + if (!readInt(start, end, &start, false)) + return false; + if (start == end) { + *tokenEnd = start; + return true; + } + + // Optional fraction part + c = *start; + if ('.' == c) { + ++start; + if (!readInt(start, end, &start, true)) + return false; + if (start == end) { + *tokenEnd = start; + return true; + } + c = *start; + } + + // Optional exponent part + if ('e' == c || 'E' == c) { + ++start; + if (start == end) + return false; + c = *start; + if ('-' == c || '+' == c) { + ++start; + if (start == end) + return false; + } + if (!readInt(start, end, &start, true)) + return false; + } + + *tokenEnd = start; + return true; +} + +template +bool readHexDigits(const Char* start, const Char* end, const Char** tokenEnd, int digits) +{ + if (end - start < digits) + return false; + for (int i = 0; i < digits; ++i) { + Char c = *start++; + if (!(('0' <= c && c <= '9') || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F'))) + return false; + } + *tokenEnd = start; + return true; +} + +template +bool parseStringToken(const Char* start, const Char* end, const Char** tokenEnd) +{ + while (start < end) { + Char c = *start++; + if ('\\' == c) { + if (start == end) + return false; + c = *start++; + // Make sure the escaped char is valid. + switch (c) { + case 'x': + if (!readHexDigits(start, end, &start, 2)) + return false; + break; + case 'u': + if (!readHexDigits(start, end, &start, 4)) + return false; + break; + case '\\': + case '/': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + case 'v': + case '"': + break; + default: + return false; + } + } else if ('"' == c) { + *tokenEnd = start; + return true; + } + } + return false; +} + +template +bool skipComment(const Char* start, const Char* end, const Char** commentEnd) +{ + if (start == end) + return false; + + if (*start != '/' || start + 1 >= end) + return false; + ++start; + + if (*start == '/') { + // Single line comment, read to newline. + for (++start; start < end; ++start) { + if (*start == '\n' || *start == '\r') { + *commentEnd = start + 1; + return true; + } + } + *commentEnd = end; + // Comment reaches end-of-input, which is fine. + return true; + } + + if (*start == '*') { + Char previous = '\0'; + // Block comment, read until end marker. + for (++start; start < end; previous = *start++) { + if (previous == '*' && *start == '/') { + *commentEnd = start + 1; + return true; + } + } + // Block comment must close before end-of-input. + return false; + } + + return false; +} + +template +void skipWhitespaceAndComments(const Char* start, const Char* end, const Char** whitespaceEnd) +{ + while (start < end) { + if (isSpaceOrNewLine(*start)) { + ++start; + } else if (*start == '/') { + const Char* commentEnd; + if (!skipComment(start, end, &commentEnd)) + break; + start = commentEnd; + } else { + break; + } + } + *whitespaceEnd = start; +} + +template +Token parseToken(const Char* start, const Char* end, const Char** tokenStart, const Char** tokenEnd) +{ + skipWhitespaceAndComments(start, end, tokenStart); + start = *tokenStart; + + if (start == end) + return InvalidToken; + + switch (*start) { + case 'n': + if (parseConstToken(start, end, tokenEnd, nullString)) + return NullToken; + break; + case 't': + if (parseConstToken(start, end, tokenEnd, trueString)) + return BoolTrue; + break; + case 'f': + if (parseConstToken(start, end, tokenEnd, falseString)) + return BoolFalse; + break; + case '[': + *tokenEnd = start + 1; + return ArrayBegin; + case ']': + *tokenEnd = start + 1; + return ArrayEnd; + case ',': + *tokenEnd = start + 1; + return ListSeparator; + case '{': + *tokenEnd = start + 1; + return ObjectBegin; + case '}': + *tokenEnd = start + 1; + return ObjectEnd; + case ':': + *tokenEnd = start + 1; + return ObjectPairSeparator; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case '-': + if (parseNumberToken(start, end, tokenEnd)) + return Number; + break; + case '"': + if (parseStringToken(start + 1, end, tokenEnd)) + return StringLiteral; + break; + } + return InvalidToken; +} + +template +int hexToInt(Char c) +{ + if ('0' <= c && c <= '9') + return c - '0'; + if ('A' <= c && c <= 'F') + return c - 'A' + 10; + if ('a' <= c && c <= 'f') + return c - 'a' + 10; + DCHECK(false); + return 0; +} + +template +bool decodeString(const Char* start, const Char* end, StringBuilder* output) +{ + while (start < end) { + uint16_t c = *start++; + if ('\\' != c) { + StringUtil::builderAppend(*output, c); + continue; + } + if (start == end) + return false; + c = *start++; + + if (c == 'x') { + // \x is not supported. + return false; + } + + switch (c) { + case '"': + case '/': + case '\\': + break; + case 'b': + c = '\b'; + break; + case 'f': + c = '\f'; + break; + case 'n': + c = '\n'; + break; + case 'r': + c = '\r'; + break; + case 't': + c = '\t'; + break; + case 'v': + c = '\v'; + break; + case 'u': + c = (hexToInt(*start) << 12) + + (hexToInt(*(start + 1)) << 8) + + (hexToInt(*(start + 2)) << 4) + + hexToInt(*(start + 3)); + start += 4; + break; + default: + return false; + } + StringUtil::builderAppend(*output, c); + } + return true; +} + +template +bool decodeString(const Char* start, const Char* end, String* output) +{ + if (start == end) { + *output = ""; + return true; + } + if (start > end) + return false; + StringBuilder buffer; + StringUtil::builderReserve(buffer, end - start); + if (!decodeString(start, end, &buffer)) + return false; + *output = StringUtil::builderToString(buffer); + return true; +} + +template +std::unique_ptr buildValue(const Char* start, const Char* end, const Char** valueTokenEnd, int depth) +{ + if (depth > stackLimit) + return nullptr; + + std::unique_ptr result; + const Char* tokenStart; + const Char* tokenEnd; + Token token = parseToken(start, end, &tokenStart, &tokenEnd); + switch (token) { + case InvalidToken: + return nullptr; + case NullToken: + result = Value::null(); + break; + case BoolTrue: + result = FundamentalValue::create(true); + break; + case BoolFalse: + result = FundamentalValue::create(false); + break; + case Number: { + bool ok; + double value = charactersToDouble(tokenStart, tokenEnd - tokenStart, &ok); + if (!ok) + return nullptr; + int number = static_cast(value); + if (number == value) + result = FundamentalValue::create(number); + else + result = FundamentalValue::create(value); + break; + } + case StringLiteral: { + String value; + bool ok = decodeString(tokenStart + 1, tokenEnd - 1, &value); + if (!ok) + return nullptr; + result = StringValue::create(value); + break; + } + case ArrayBegin: { + std::unique_ptr array = ListValue::create(); + start = tokenEnd; + token = parseToken(start, end, &tokenStart, &tokenEnd); + while (token != ArrayEnd) { + std::unique_ptr arrayNode = buildValue(start, end, &tokenEnd, depth + 1); + if (!arrayNode) + return nullptr; + array->pushValue(std::move(arrayNode)); + + // After a list value, we expect a comma or the end of the list. + start = tokenEnd; + token = parseToken(start, end, &tokenStart, &tokenEnd); + if (token == ListSeparator) { + start = tokenEnd; + token = parseToken(start, end, &tokenStart, &tokenEnd); + if (token == ArrayEnd) + return nullptr; + } else if (token != ArrayEnd) { + // Unexpected value after list value. Bail out. + return nullptr; + } + } + if (token != ArrayEnd) + return nullptr; + result = std::move(array); + break; + } + case ObjectBegin: { + std::unique_ptr object = DictionaryValue::create(); + start = tokenEnd; + token = parseToken(start, end, &tokenStart, &tokenEnd); + while (token != ObjectEnd) { + if (token != StringLiteral) + return nullptr; + String key; + if (!decodeString(tokenStart + 1, tokenEnd - 1, &key)) + return nullptr; + start = tokenEnd; + + token = parseToken(start, end, &tokenStart, &tokenEnd); + if (token != ObjectPairSeparator) + return nullptr; + start = tokenEnd; + + std::unique_ptr value = buildValue(start, end, &tokenEnd, depth + 1); + if (!value) + return nullptr; + object->setValue(key, std::move(value)); + start = tokenEnd; + + // After a key/value pair, we expect a comma or the end of the + // object. + token = parseToken(start, end, &tokenStart, &tokenEnd); + if (token == ListSeparator) { + start = tokenEnd; + token = parseToken(start, end, &tokenStart, &tokenEnd); + if (token == ObjectEnd) + return nullptr; + } else if (token != ObjectEnd) { + // Unexpected value after last object value. Bail out. + return nullptr; + } + } + if (token != ObjectEnd) + return nullptr; + result = std::move(object); + break; + } + + default: + // We got a token that's not a value. + return nullptr; + } + + skipWhitespaceAndComments(tokenEnd, end, valueTokenEnd); + return result; +} + +template +std::unique_ptr parseJSONInternal(const Char* start, unsigned length) +{ + const Char* end = start + length; + const Char *tokenEnd; + std::unique_ptr value = buildValue(start, end, &tokenEnd, 0); + if (!value || tokenEnd != end) + return nullptr; + return value; +} + +} // anonymous namespace + +std::unique_ptr parseJSONCharacters(const uint16_t* characters, unsigned length) +{ + return parseJSONInternal(characters, length); +} + +std::unique_ptr parseJSONCharacters(const uint8_t* characters, unsigned length) +{ + return parseJSONInternal(characters, length); +} + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} diff --git a/tools/inspector_protocol/lib/Parser_h.template b/tools/inspector_protocol/lib/Parser_h.template new file mode 100644 index 00000000000000..8397d3f5d6911c --- /dev/null +++ b/tools/inspector_protocol/lib/Parser_h.template @@ -0,0 +1,22 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_Parser_h +#define {{"_".join(config.protocol.namespace)}}_Parser_h + +//#include "Forward.h" +//#include "Values.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +{{config.lib.export_macro}} std::unique_ptr parseJSONCharacters(const uint8_t*, unsigned); +{{config.lib.export_macro}} std::unique_ptr parseJSONCharacters(const uint16_t*, unsigned); + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_Parser_h) diff --git a/tools/inspector_protocol/lib/Protocol_cpp.template b/tools/inspector_protocol/lib/Protocol_cpp.template new file mode 100644 index 00000000000000..901656373a4f52 --- /dev/null +++ b/tools/inspector_protocol/lib/Protocol_cpp.template @@ -0,0 +1,12 @@ +// This file is generated. + +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include {{format_include(config.protocol.package, "Protocol")}} + +#include +#include + +#include diff --git a/tools/inspector_protocol/lib/ValueConversions_h.template b/tools/inspector_protocol/lib/ValueConversions_h.template new file mode 100644 index 00000000000000..4d64ec9091a6c2 --- /dev/null +++ b/tools/inspector_protocol/lib/ValueConversions_h.template @@ -0,0 +1,171 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_ValueConversions_h +#define {{"_".join(config.protocol.namespace)}}_ValueConversions_h + +//#include "ErrorSupport.h" +//#include "Forward.h" +//#include "Values.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +template +struct ValueConversions { + static std::unique_ptr fromValue(protocol::Value* value, ErrorSupport* errors) + { + return T::fromValue(value, errors); + } + + static std::unique_ptr toValue(T* value) + { + return value->toValue(); + } + + static std::unique_ptr toValue(const std::unique_ptr& value) + { + return value->toValue(); + } +}; + +template<> +struct ValueConversions { + static bool fromValue(protocol::Value* value, ErrorSupport* errors) + { + bool result = false; + bool success = value ? value->asBoolean(&result) : false; + if (!success) + errors->addError("boolean value expected"); + return result; + } + + static std::unique_ptr toValue(bool value) + { + return FundamentalValue::create(value); + } +}; + +template<> +struct ValueConversions { + static int fromValue(protocol::Value* value, ErrorSupport* errors) + { + int result = 0; + bool success = value ? value->asInteger(&result) : false; + if (!success) + errors->addError("integer value expected"); + return result; + } + + static std::unique_ptr toValue(int value) + { + return FundamentalValue::create(value); + } +}; + +template<> +struct ValueConversions { + static double fromValue(protocol::Value* value, ErrorSupport* errors) + { + double result = 0; + bool success = value ? value->asDouble(&result) : false; + if (!success) + errors->addError("double value expected"); + return result; + } + + static std::unique_ptr toValue(double value) + { + return FundamentalValue::create(value); + } +}; + +template<> +struct ValueConversions { + static String fromValue(protocol::Value* value, ErrorSupport* errors) + { + String result; + bool success = value ? value->asString(&result) : false; + if (!success) + errors->addError("string value expected"); + return result; + } + + static std::unique_ptr toValue(const String& value) + { + return StringValue::create(value); + } +}; + +template<> +struct ValueConversions { + static std::unique_ptr fromValue(protocol::Value* value, ErrorSupport* errors) + { + bool success = !!value; + if (!success) { + errors->addError("value expected"); + return nullptr; + } + return value->clone(); + } + + static std::unique_ptr toValue(Value* value) + { + return value->clone(); + } + + static std::unique_ptr toValue(const std::unique_ptr& value) + { + return value->clone(); + } +}; + +template<> +struct ValueConversions { + static std::unique_ptr fromValue(protocol::Value* value, ErrorSupport* errors) + { + bool success = value && value->type() == protocol::Value::TypeObject; + if (!success) + errors->addError("object expected"); + return DictionaryValue::cast(value->clone()); + } + + static std::unique_ptr toValue(DictionaryValue* value) + { + return value->clone(); + } + + static std::unique_ptr toValue(const std::unique_ptr& value) + { + return value->clone(); + } +}; + +template<> +struct ValueConversions { + static std::unique_ptr fromValue(protocol::Value* value, ErrorSupport* errors) + { + bool success = value && value->type() == protocol::Value::TypeArray; + if (!success) + errors->addError("list expected"); + return ListValue::cast(value->clone()); + } + + static std::unique_ptr toValue(ListValue* value) + { + return value->clone(); + } + + static std::unique_ptr toValue(const std::unique_ptr& value) + { + return value->clone(); + } +}; + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_ValueConversions_h) diff --git a/tools/inspector_protocol/lib/Values_cpp.template b/tools/inspector_protocol/lib/Values_cpp.template new file mode 100644 index 00000000000000..b9f061346bf66f --- /dev/null +++ b/tools/inspector_protocol/lib/Values_cpp.template @@ -0,0 +1,409 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +//#include "Values.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +namespace { + +const char* const nullValueString = "null"; +const char* const trueValueString = "true"; +const char* const falseValueString = "false"; + +inline bool escapeChar(uint16_t c, StringBuilder* dst) +{ + switch (c) { + case '\b': StringUtil::builderAppend(*dst, "\\b"); break; + case '\f': StringUtil::builderAppend(*dst, "\\f"); break; + case '\n': StringUtil::builderAppend(*dst, "\\n"); break; + case '\r': StringUtil::builderAppend(*dst, "\\r"); break; + case '\t': StringUtil::builderAppend(*dst, "\\t"); break; + case '\\': StringUtil::builderAppend(*dst, "\\\\"); break; + case '"': StringUtil::builderAppend(*dst, "\\\""); break; + default: + return false; + } + return true; +} + +const char hexDigits[17] = "0123456789ABCDEF"; + +void appendUnsignedAsHex(uint16_t number, StringBuilder* dst) +{ + StringUtil::builderAppend(*dst, "\\u"); + for (size_t i = 0; i < 4; ++i) { + uint16_t c = hexDigits[(number & 0xF000) >> 12]; + StringUtil::builderAppend(*dst, c); + number <<= 4; + } +} + +template +void escapeStringForJSONInternal(const Char* str, unsigned len, + StringBuilder* dst) +{ + for (unsigned i = 0; i < len; ++i) { + Char c = str[i]; + if (escapeChar(c, dst)) + continue; + if (c < 32 || c > 126) { + appendUnsignedAsHex(c, dst); + } else { + StringUtil::builderAppend(*dst, c); + } + } +} + +} // anonymous namespace + +bool Value::asBoolean(bool*) const +{ + return false; +} + +bool Value::asDouble(double*) const +{ + return false; +} + +bool Value::asInteger(int*) const +{ + return false; +} + +bool Value::asString(String*) const +{ + return false; +} + +bool Value::asSerialized(String*) const +{ + return false; +} + +void Value::writeJSON(StringBuilder* output) const +{ + DCHECK(m_type == TypeNull); + StringUtil::builderAppend(*output, nullValueString, 4); +} + +std::unique_ptr Value::clone() const +{ + return Value::null(); +} + +String Value::serialize() +{ + StringBuilder result; + StringUtil::builderReserve(result, 512); + writeJSON(&result); + return StringUtil::builderToString(result); +} + +bool FundamentalValue::asBoolean(bool* output) const +{ + if (type() != TypeBoolean) + return false; + *output = m_boolValue; + return true; +} + +bool FundamentalValue::asDouble(double* output) const +{ + if (type() == TypeDouble) { + *output = m_doubleValue; + return true; + } + if (type() == TypeInteger) { + *output = m_integerValue; + return true; + } + return false; +} + +bool FundamentalValue::asInteger(int* output) const +{ + if (type() != TypeInteger) + return false; + *output = m_integerValue; + return true; +} + +void FundamentalValue::writeJSON(StringBuilder* output) const +{ + DCHECK(type() == TypeBoolean || type() == TypeInteger || type() == TypeDouble); + if (type() == TypeBoolean) { + if (m_boolValue) + StringUtil::builderAppend(*output, trueValueString, 4); + else + StringUtil::builderAppend(*output, falseValueString, 5); + } else if (type() == TypeDouble) { + if (!std::isfinite(m_doubleValue)) { + StringUtil::builderAppend(*output, nullValueString, 4); + return; + } + StringUtil::builderAppend(*output, StringUtil::fromDouble(m_doubleValue)); + } else if (type() == TypeInteger) { + StringUtil::builderAppend(*output, StringUtil::fromInteger(m_integerValue)); + } +} + +std::unique_ptr FundamentalValue::clone() const +{ + switch (type()) { + case TypeDouble: return FundamentalValue::create(m_doubleValue); + case TypeInteger: return FundamentalValue::create(m_integerValue); + case TypeBoolean: return FundamentalValue::create(m_boolValue); + default: + DCHECK(false); + } + return nullptr; +} + +bool StringValue::asString(String* output) const +{ + *output = m_stringValue; + return true; +} + +void StringValue::writeJSON(StringBuilder* output) const +{ + DCHECK(type() == TypeString); + StringUtil::builderAppendQuotedString(*output, m_stringValue); +} + +std::unique_ptr StringValue::clone() const +{ + return StringValue::create(m_stringValue); +} + +bool SerializedValue::asSerialized(String* output) const +{ + *output = m_serializedValue; + return true; +} + +void SerializedValue::writeJSON(StringBuilder* output) const +{ + DCHECK(type() == TypeSerialized); + StringUtil::builderAppend(*output, m_serializedValue); +} + +std::unique_ptr SerializedValue::clone() const +{ + return SerializedValue::create(m_serializedValue); +} + +DictionaryValue::~DictionaryValue() +{ +} + +void DictionaryValue::setBoolean(const String& name, bool value) +{ + setValue(name, FundamentalValue::create(value)); +} + +void DictionaryValue::setInteger(const String& name, int value) +{ + setValue(name, FundamentalValue::create(value)); +} + +void DictionaryValue::setDouble(const String& name, double value) +{ + setValue(name, FundamentalValue::create(value)); +} + +void DictionaryValue::setString(const String& name, const String& value) +{ + setValue(name, StringValue::create(value)); +} + +void DictionaryValue::setValue(const String& name, std::unique_ptr value) +{ + set(name, value); +} + +void DictionaryValue::setObject(const String& name, std::unique_ptr value) +{ + set(name, value); +} + +void DictionaryValue::setArray(const String& name, std::unique_ptr value) +{ + set(name, value); +} + +bool DictionaryValue::getBoolean(const String& name, bool* output) const +{ + protocol::Value* value = get(name); + if (!value) + return false; + return value->asBoolean(output); +} + +bool DictionaryValue::getInteger(const String& name, int* output) const +{ + Value* value = get(name); + if (!value) + return false; + return value->asInteger(output); +} + +bool DictionaryValue::getDouble(const String& name, double* output) const +{ + Value* value = get(name); + if (!value) + return false; + return value->asDouble(output); +} + +bool DictionaryValue::getString(const String& name, String* output) const +{ + protocol::Value* value = get(name); + if (!value) + return false; + return value->asString(output); +} + +DictionaryValue* DictionaryValue::getObject(const String& name) const +{ + return DictionaryValue::cast(get(name)); +} + +protocol::ListValue* DictionaryValue::getArray(const String& name) const +{ + return ListValue::cast(get(name)); +} + +protocol::Value* DictionaryValue::get(const String& name) const +{ + Dictionary::const_iterator it = m_data.find(name); + if (it == m_data.end()) + return nullptr; + return it->second.get(); +} + +DictionaryValue::Entry DictionaryValue::at(size_t index) const +{ + const String key = m_order[index]; + return std::make_pair(key, m_data.find(key)->second.get()); +} + +bool DictionaryValue::booleanProperty(const String& name, bool defaultValue) const +{ + bool result = defaultValue; + getBoolean(name, &result); + return result; +} + +int DictionaryValue::integerProperty(const String& name, int defaultValue) const +{ + int result = defaultValue; + getInteger(name, &result); + return result; +} + +double DictionaryValue::doubleProperty(const String& name, double defaultValue) const +{ + double result = defaultValue; + getDouble(name, &result); + return result; +} + +void DictionaryValue::remove(const String& name) +{ + m_data.erase(name); + m_order.erase(std::remove(m_order.begin(), m_order.end(), name), m_order.end()); +} + +void DictionaryValue::writeJSON(StringBuilder* output) const +{ + StringUtil::builderAppend(*output, '{'); + for (size_t i = 0; i < m_order.size(); ++i) { + Dictionary::const_iterator it = m_data.find(m_order[i]); + CHECK(it != m_data.end()); + if (i) + StringUtil::builderAppend(*output, ','); + StringUtil::builderAppendQuotedString(*output, it->first); + StringUtil::builderAppend(*output, ':'); + it->second->writeJSON(output); + } + StringUtil::builderAppend(*output, '}'); +} + +std::unique_ptr DictionaryValue::clone() const +{ + std::unique_ptr result = DictionaryValue::create(); + for (size_t i = 0; i < m_order.size(); ++i) { + String key = m_order[i]; + Dictionary::const_iterator value = m_data.find(key); + DCHECK(value != m_data.cend() && value->second); + result->setValue(key, value->second->clone()); + } + return std::move(result); +} + +DictionaryValue::DictionaryValue() + : Value(TypeObject) +{ +} + +ListValue::~ListValue() +{ +} + +void ListValue::writeJSON(StringBuilder* output) const +{ + StringUtil::builderAppend(*output, '['); + bool first = true; + for (const std::unique_ptr& value : m_data) { + if (!first) + StringUtil::builderAppend(*output, ','); + value->writeJSON(output); + first = false; + } + StringUtil::builderAppend(*output, ']'); +} + +std::unique_ptr ListValue::clone() const +{ + std::unique_ptr result = ListValue::create(); + for (const std::unique_ptr& value : m_data) + result->pushValue(value->clone()); + return std::move(result); +} + +ListValue::ListValue() + : Value(TypeArray) +{ +} + +void ListValue::pushValue(std::unique_ptr value) +{ + DCHECK(value); + m_data.push_back(std::move(value)); +} + +protocol::Value* ListValue::at(size_t index) +{ + DCHECK_LT(index, m_data.size()); + return m_data[index].get(); +} + +void escapeLatinStringForJSON(const uint8_t* str, unsigned len, StringBuilder* dst) +{ + escapeStringForJSONInternal(str, len, dst); +} + +void escapeWideStringForJSON(const uint16_t* str, unsigned len, StringBuilder* dst) +{ + escapeStringForJSONInternal(str, len, dst); +} + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} diff --git a/tools/inspector_protocol/lib/Values_h.template b/tools/inspector_protocol/lib/Values_h.template new file mode 100644 index 00000000000000..3638b34b4e7718 --- /dev/null +++ b/tools/inspector_protocol/lib/Values_h.template @@ -0,0 +1,249 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_Values_h +#define {{"_".join(config.protocol.namespace)}}_Values_h + +//#include "Allocator.h" +//#include "Collections.h" +//#include "Forward.h" + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + +class ListValue; +class DictionaryValue; +class Value; + +class {{config.lib.export_macro}} Value : public Serializable { + PROTOCOL_DISALLOW_COPY(Value); +public: + virtual ~Value() override { } + + static std::unique_ptr null() + { + return std::unique_ptr(new Value()); + } + + enum ValueType { + TypeNull = 0, + TypeBoolean, + TypeInteger, + TypeDouble, + TypeString, + TypeObject, + TypeArray, + TypeSerialized + }; + + ValueType type() const { return m_type; } + + bool isNull() const { return m_type == TypeNull; } + + virtual bool asBoolean(bool* output) const; + virtual bool asDouble(double* output) const; + virtual bool asInteger(int* output) const; + virtual bool asString(String* output) const; + virtual bool asSerialized(String* output) const; + + virtual void writeJSON(StringBuilder* output) const; + virtual std::unique_ptr clone() const; + String serialize() override; + +protected: + Value() : m_type(TypeNull) { } + explicit Value(ValueType type) : m_type(type) { } + +private: + friend class DictionaryValue; + friend class ListValue; + + ValueType m_type; +}; + +class {{config.lib.export_macro}} FundamentalValue : public Value { +public: + static std::unique_ptr create(bool value) + { + return std::unique_ptr(new FundamentalValue(value)); + } + + static std::unique_ptr create(int value) + { + return std::unique_ptr(new FundamentalValue(value)); + } + + static std::unique_ptr create(double value) + { + return std::unique_ptr(new FundamentalValue(value)); + } + + bool asBoolean(bool* output) const override; + bool asDouble(double* output) const override; + bool asInteger(int* output) const override; + void writeJSON(StringBuilder* output) const override; + std::unique_ptr clone() const override; + +private: + explicit FundamentalValue(bool value) : Value(TypeBoolean), m_boolValue(value) { } + explicit FundamentalValue(int value) : Value(TypeInteger), m_integerValue(value) { } + explicit FundamentalValue(double value) : Value(TypeDouble), m_doubleValue(value) { } + + union { + bool m_boolValue; + double m_doubleValue; + int m_integerValue; + }; +}; + +class {{config.lib.export_macro}} StringValue : public Value { +public: + static std::unique_ptr create(const String& value) + { + return std::unique_ptr(new StringValue(value)); + } + + static std::unique_ptr create(const char* value) + { + return std::unique_ptr(new StringValue(value)); + } + + bool asString(String* output) const override; + void writeJSON(StringBuilder* output) const override; + std::unique_ptr clone() const override; + +private: + explicit StringValue(const String& value) : Value(TypeString), m_stringValue(value) { } + explicit StringValue(const char* value) : Value(TypeString), m_stringValue(value) { } + + String m_stringValue; +}; + +class {{config.lib.export_macro}} SerializedValue : public Value { +public: + static std::unique_ptr create(const String& value) + { + return std::unique_ptr(new SerializedValue(value)); + } + + bool asSerialized(String* output) const override; + void writeJSON(StringBuilder* output) const override; + std::unique_ptr clone() const override; + +private: + explicit SerializedValue(const String& value) : Value(TypeSerialized), m_serializedValue(value) { } + + String m_serializedValue; +}; + +class {{config.lib.export_macro}} DictionaryValue : public Value { +public: + using Entry = std::pair; + static std::unique_ptr create() + { + return std::unique_ptr(new DictionaryValue()); + } + + static DictionaryValue* cast(Value* value) + { + if (!value || value->type() != TypeObject) + return nullptr; + return static_cast(value); + } + + static std::unique_ptr cast(std::unique_ptr value) + { + return std::unique_ptr(DictionaryValue::cast(value.release())); + } + + void writeJSON(StringBuilder* output) const override; + std::unique_ptr clone() const override; + + size_t size() const { return m_data.size(); } + + void setBoolean(const String& name, bool); + void setInteger(const String& name, int); + void setDouble(const String& name, double); + void setString(const String& name, const String&); + void setValue(const String& name, std::unique_ptr); + void setObject(const String& name, std::unique_ptr); + void setArray(const String& name, std::unique_ptr); + + bool getBoolean(const String& name, bool* output) const; + bool getInteger(const String& name, int* output) const; + bool getDouble(const String& name, double* output) const; + bool getString(const String& name, String* output) const; + + DictionaryValue* getObject(const String& name) const; + ListValue* getArray(const String& name) const; + Value* get(const String& name) const; + Entry at(size_t index) const; + + bool booleanProperty(const String& name, bool defaultValue) const; + int integerProperty(const String& name, int defaultValue) const; + double doubleProperty(const String& name, double defaultValue) const; + void remove(const String& name); + + ~DictionaryValue() override; + +private: + DictionaryValue(); + template + void set(const String& key, std::unique_ptr& value) + { + DCHECK(value); + bool isNew = m_data.find(key) == m_data.end(); + m_data[key] = std::move(value); + if (isNew) + m_order.push_back(key); + } + + using Dictionary = protocol::HashMap>; + Dictionary m_data; + std::vector m_order; +}; + +class {{config.lib.export_macro}} ListValue : public Value { +public: + static std::unique_ptr create() + { + return std::unique_ptr(new ListValue()); + } + + static ListValue* cast(Value* value) + { + if (!value || value->type() != TypeArray) + return nullptr; + return static_cast(value); + } + + static std::unique_ptr cast(std::unique_ptr value) + { + return std::unique_ptr(ListValue::cast(value.release())); + } + + ~ListValue() override; + + void writeJSON(StringBuilder* output) const override; + std::unique_ptr clone() const override; + + void pushValue(std::unique_ptr); + + Value* at(size_t index); + size_t size() const { return m_data.size(); } + +private: + ListValue(); + std::vector> m_data; +}; + +void escapeLatinStringForJSON(const uint8_t* str, unsigned len, StringBuilder* dst); +void escapeWideStringForJSON(const uint16_t* str, unsigned len, StringBuilder* dst); + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // {{"_".join(config.protocol.namespace)}}_Values_h diff --git a/tools/inspector_protocol/templates/Exported_h.template b/tools/inspector_protocol/templates/Exported_h.template new file mode 100644 index 00000000000000..3d36ecffae3ca3 --- /dev/null +++ b/tools/inspector_protocol/templates/Exported_h.template @@ -0,0 +1,65 @@ +// This file is generated + +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_{{domain.domain}}_api_h +#define {{"_".join(config.protocol.namespace)}}_{{domain.domain}}_api_h + +{% if config.exported.export_header %} +#include {{format_include(config.exported.export_header)}} +{% endif %} +#include {{format_include(config.exported.string_header)}} + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} +namespace {{domain.domain}} { +namespace API { + +// ------------- Enums. + {% for type in domain.types %} + {% if ("enum" in type) and protocol.is_exported(domain.domain, type.id) %} + +namespace {{type.id}}Enum { + {% for literal in type.enum %} +{{config.exported.export_macro}} extern const char* {{ literal | dash_to_camelcase}}; + {% endfor %} +} // {{type.id}}Enum + {% endif %} + {% endfor %} + {% for command in join_arrays(domain, ["commands", "events"]) %} + {% for param in join_arrays(command, ["parameters", "returns"]) %} + {% if ("enum" in param) and protocol.is_exported(domain.domain, command.name + "." + param.name) %} + +namespace {{command.name | to_title_case}} { +namespace {{param.name | to_title_case}}Enum { + {% for literal in param.enum %} +{{config.exported.export_macro}} extern const char* {{ literal | dash_to_camelcase}}; + {% endfor %} +} // {{param.name | to_title_case}}Enum +} // {{command.name | to_title_case }} + {% endif %} + {% endfor %} + {% endfor %} + +// ------------- Types. + {% for type in domain.types %} + {% if not (type.type == "object") or not ("properties" in type) or not protocol.is_exported(domain.domain, type.id) %}{% continue %}{% endif %} + +class {{config.exported.export_macro}} {{type.id}} { +public: + virtual {{config.exported.string_out}} toJSONString() const = 0; + virtual ~{{type.id}}() { } + static std::unique_ptr fromJSONString(const {{config.exported.string_in}}& json); +}; + {% endfor %} + +} // namespace API +} // namespace {{domain.domain}} +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_{{domain.domain}}_api_h) diff --git a/tools/inspector_protocol/templates/Imported_h.template b/tools/inspector_protocol/templates/Imported_h.template new file mode 100644 index 00000000000000..4c9d24bd5fccf7 --- /dev/null +++ b/tools/inspector_protocol/templates/Imported_h.template @@ -0,0 +1,55 @@ +// This file is generated + +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_{{domain.domain}}_imported_h +#define {{"_".join(config.protocol.namespace)}}_{{domain.domain}}_imported_h + +#include {{format_include(config.protocol.package, "Protocol")}} +{% if config.imported.header %} +#include {{format_include(config.imported.header)}} +{% else %} +#include {{format_include(config.imported.package, domain.domain)}} +{% endif %} + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} + {% for type in domain.types %} + {% if not (type.type == "object") or not ("properties" in type) or not protocol.is_imported(domain.domain, type.id) %}{% continue %}{% endif %} + +template<> +struct ValueConversions<{{"::".join(config.imported.namespace)}}::{{domain.domain}}::API::{{type.id}}> { + static std::unique_ptr<{{"::".join(config.imported.namespace)}}::{{domain.domain}}::API::{{type.id}}> fromValue(protocol::Value* value, ErrorSupport* errors) + { + if (!value) { + errors->addError("value expected"); + return nullptr; + } + String json = value->serialize(); + auto result = {{"::".join(config.imported.namespace)}}::{{domain.domain}}::API::{{type.id}}::fromJSONString({{config.imported.to_imported_string % "json"}}); + if (!result) + errors->addError("cannot parse"); + return result; + } + + static std::unique_ptr toValue(const {{"::".join(config.imported.namespace)}}::{{domain.domain}}::API::{{type.id}}* value) + { + auto json = value->toJSONString(); + return SerializedValue::create({{config.imported.from_imported_string % "std::move(json)"}}); + } + + static std::unique_ptr toValue(const std::unique_ptr<{{"::".join(config.imported.namespace)}}::{{domain.domain}}::API::{{type.id}}>& value) + { + return toValue(value.get()); + } +}; + {% endfor %} + +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_{{domain.domain}}_imported_h) diff --git a/tools/inspector_protocol/templates/TypeBuilder_cpp.template b/tools/inspector_protocol/templates/TypeBuilder_cpp.template new file mode 100644 index 00000000000000..026c1cdb8da9e1 --- /dev/null +++ b/tools/inspector_protocol/templates/TypeBuilder_cpp.template @@ -0,0 +1,397 @@ +// This file is generated + +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include {{format_include(config.protocol.package, domain.domain)}} + +#include {{format_include(config.protocol.package, "Protocol")}} + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} +namespace {{domain.domain}} { + +// ------------- Enum values from types. + +const char Metainfo::domainName[] = "{{domain.domain}}"; +const char Metainfo::commandPrefix[] = "{{domain.domain}}."; +const char Metainfo::version[] = "{{domain.version}}"; + {% for type in domain.types %} + {% if not protocol.generate_type(domain.domain, type.id) %}{% continue %} {% endif %} + {% if "enum" in type %} + +namespace {{type.id}}Enum { + {% for literal in type.enum %} +const char* {{ literal | dash_to_camelcase}} = "{{literal}}"; + {% endfor %} +} // namespace {{type.id}}Enum + {% if protocol.is_exported(domain.domain, type.id) %} + +namespace API { +namespace {{type.id}}Enum { + {% for literal in type.enum %} +const char* {{ literal | dash_to_camelcase}} = "{{literal}}"; + {% endfor %} +} // namespace {{type.id}}Enum +} // namespace API + {% endif %} + {% endif %} + {% for property in type.properties %} + {% if "enum" in property %} + + {% for literal in property.enum %} +const char* {{type.id}}::{{property.name | to_title_case}}Enum::{{literal | dash_to_camelcase}} = "{{literal}}"; + {% endfor %} + {% endif %} + {% endfor %} + {% if not (type.type == "object") or not ("properties" in type) %}{% continue %}{% endif %} + +std::unique_ptr<{{type.id}}> {{type.id}}::fromValue(protocol::Value* value, ErrorSupport* errors) +{ + if (!value || value->type() != protocol::Value::TypeObject) { + errors->addError("object expected"); + return nullptr; + } + + std::unique_ptr<{{type.id}}> result(new {{type.id}}()); + protocol::DictionaryValue* object = DictionaryValue::cast(value); + errors->push(); + {% for property in type.properties %} + protocol::Value* {{property.name}}Value = object->get("{{property.name}}"); + {% if property.optional %} + if ({{property.name}}Value) { + errors->setName("{{property.name}}"); + result->m_{{property.name}} = ValueConversions<{{protocol.resolve_type(property).raw_type}}>::fromValue({{property.name}}Value, errors); + } + {% else %} + errors->setName("{{property.name}}"); + result->m_{{property.name}} = ValueConversions<{{protocol.resolve_type(property).raw_type}}>::fromValue({{property.name}}Value, errors); + {% endif %} + {% endfor %} + errors->pop(); + if (errors->hasErrors()) + return nullptr; + return result; +} + +std::unique_ptr {{type.id}}::toValue() const +{ + std::unique_ptr result = DictionaryValue::create(); + {% for property in type.properties %} + {% set property_type = protocol.resolve_type(property) %} + {% set property_field = "m_" + property.name %} + {% if property.optional %} + if ({{property_field}}.isJust()) + result->setValue("{{property.name}}", ValueConversions<{{property_type.raw_type}}>::toValue({{property_field}}.fromJust())); + {% else %} + result->setValue("{{property.name}}", ValueConversions<{{property_type.raw_type}}>::toValue({{property_type.to_raw_type % property_field}})); + {% endif %} + {% endfor %} + return result; +} + +std::unique_ptr<{{type.id}}> {{type.id}}::clone() const +{ + ErrorSupport errors; + return fromValue(toValue().get(), &errors); +} + {% if protocol.is_exported(domain.domain, type.id) %} + +{{config.exported.string_out}} {{type.id}}::toJSONString() const +{ + String json = toValue()->serialize(); + return {{config.exported.to_string_out % "json"}}; +} + +// static +std::unique_ptr API::{{type.id}}::fromJSONString(const {{config.exported.string_in}}& json) +{ + ErrorSupport errors; + std::unique_ptr value = StringUtil::parseJSON(json); + if (!value) + return nullptr; + return protocol::{{domain.domain}}::{{type.id}}::fromValue(value.get(), &errors); +} + {% endif %} + {% endfor %} + +// ------------- Enum values from params. + + {% for command in join_arrays(domain, ["commands", "events"]) %} + {% for param in join_arrays(command, ["parameters", "returns"]) %} + {% if "enum" in param %} + +namespace {{command.name | to_title_case}} { +namespace {{param.name | to_title_case}}Enum { + {% for literal in param.enum %} +const char* {{ literal | to_title_case}} = "{{literal}}"; + {% endfor %} +} // namespace {{param.name | to_title_case}}Enum +} // namespace {{command.name | to_title_case }} + {% if protocol.is_exported(domain.domain, command.name + "." + param.name) %} + +namespace API { +namespace {{command.name | to_title_case}} { +namespace {{param.name | to_title_case}}Enum { + {% for literal in param.enum %} +const char* {{ literal | to_title_case}} = "{{literal}}"; + {% endfor %} +} // namespace {{param.name | to_title_case}}Enum +} // namespace {{command.name | to_title_case }} +} // namespace API + {% endif %} + {% endif %} + {% endfor %} + {% endfor %} + +// ------------- Frontend notifications. + {% for event in domain.events %} + {% if not protocol.generate_event(domain.domain, event.name) %}{% continue %}{% endif %} + +void Frontend::{{event.name | to_method_case}}( + {%- for parameter in event.parameters %} + {% if "optional" in parameter -%} + Maybe<{{protocol.resolve_type(parameter).raw_type}}> + {%- else -%} + {{protocol.resolve_type(parameter).pass_type}} + {%- endif %} {{parameter.name}}{%- if not loop.last -%}, {% endif -%} + {% endfor -%}) +{ + if (!m_frontendChannel) + return; + {% if event.parameters %} + std::unique_ptr<{{event.name | to_title_case}}Notification> messageData = {{event.name | to_title_case}}Notification::{{"create" | to_method_case}}() + {% for parameter in event.parameters %} + {% if not "optional" in parameter %} + .{{"set" | to_method_case}}{{parameter.name | to_title_case}}({{protocol.resolve_type(parameter).to_pass_type % parameter.name}}) + {% endif %} + {% endfor %} + .{{ "build" | to_method_case }}(); + {% for parameter in event.parameters %} + {% if "optional" in parameter %} + if ({{parameter.name}}.isJust()) + messageData->{{"set" | to_method_case}}{{parameter.name | to_title_case}}(std::move({{parameter.name}}).takeJust()); + {% endif %} + {% endfor %} + m_frontendChannel->sendProtocolNotification(InternalResponse::createNotification("{{domain.domain}}.{{event.name}}", std::move(messageData))); + {% else %} + m_frontendChannel->sendProtocolNotification(InternalResponse::createNotification("{{domain.domain}}.{{event.name}}")); + {% endif %} +} + {% endfor %} + +void Frontend::flush() +{ + m_frontendChannel->flushProtocolNotifications(); +} + +void Frontend::sendRawNotification(const String& notification) +{ + m_frontendChannel->sendProtocolNotification(InternalRawNotification::create(notification)); +} + +// --------------------- Dispatcher. + +class DispatcherImpl : public protocol::DispatcherBase { +public: + DispatcherImpl(FrontendChannel* frontendChannel, Backend* backend, bool fallThroughForNotFound) + : DispatcherBase(frontendChannel) + , m_backend(backend) + , m_fallThroughForNotFound(fallThroughForNotFound) { + {% for command in domain.commands %} + {% if "redirect" in command %} + m_redirects["{{domain.domain}}.{{command.name}}"] = "{{command.redirect}}.{{command.name}}"; + {% continue %} + {% endif %} + {% if not protocol.generate_command(domain.domain, command.name) %}{% continue %}{% endif %} + m_dispatchMap["{{domain.domain}}.{{command.name}}"] = &DispatcherImpl::{{command.name}}; + {% endfor %} + } + ~DispatcherImpl() override { } + DispatchResponse::Status dispatch(int callId, const String& method, std::unique_ptr messageObject) override; + HashMap& redirects() { return m_redirects; } + +protected: + using CallHandler = DispatchResponse::Status (DispatcherImpl::*)(int callId, std::unique_ptr messageObject, ErrorSupport* errors); + using DispatchMap = protocol::HashMap; + DispatchMap m_dispatchMap; + HashMap m_redirects; + + {% for command in domain.commands %} + {% if "redirect" in command %}{% continue %}{% endif %} + {% if not protocol.generate_command(domain.domain, command.name) %}{% continue %}{% endif %} + DispatchResponse::Status {{command.name}}(int callId, std::unique_ptr requestMessageObject, ErrorSupport*); + {% endfor %} + + Backend* m_backend; + bool m_fallThroughForNotFound; +}; + +DispatchResponse::Status DispatcherImpl::dispatch(int callId, const String& method, std::unique_ptr messageObject) +{ + protocol::HashMap::iterator it = m_dispatchMap.find(method); + if (it == m_dispatchMap.end()) { + if (m_fallThroughForNotFound) + return DispatchResponse::kFallThrough; + reportProtocolError(callId, DispatchResponse::kMethodNotFound, "'" + method + "' wasn't found", nullptr); + return DispatchResponse::kError; + } + + protocol::ErrorSupport errors; + return (this->*(it->second))(callId, std::move(messageObject), &errors); +} + + {% for command in domain.commands %} + {% set command_name_title = command.name | to_title_case %} + {% if "redirect" in command %}{% continue %}{% endif %} + {% if not protocol.generate_command(domain.domain, command.name) %}{% continue %}{% endif %} + {% if protocol.is_async_command(domain.domain, command.name) %} + +class {{command_name_title}}CallbackImpl : public Backend::{{command_name_title}}Callback, public DispatcherBase::Callback { +public: + {{command_name_title}}CallbackImpl(std::unique_ptr backendImpl, int callId, int callbackId) + : DispatcherBase::Callback(std::move(backendImpl), callId, callbackId) { } + + void sendSuccess( + {%- for parameter in command.returns -%} + {%- if "optional" in parameter -%} + Maybe<{{protocol.resolve_type(parameter).raw_type}}> {{parameter.name}} + {%- else -%} + {{protocol.resolve_type(parameter).pass_type}} {{parameter.name}} + {%- endif -%} + {%- if not loop.last -%}, {% endif -%} + {%- endfor -%}) override + { + std::unique_ptr resultObject = DictionaryValue::create(); + {% for parameter in command.returns %} + {% if "optional" in parameter %} + if ({{parameter.name}}.isJust()) + resultObject->setValue("{{parameter.name}}", ValueConversions<{{protocol.resolve_type(parameter).raw_type}}>::toValue({{parameter.name}}.fromJust())); + {% else %} + resultObject->setValue("{{parameter.name}}", ValueConversions<{{protocol.resolve_type(parameter).raw_type}}>::toValue({{protocol.resolve_type(parameter).to_raw_type % parameter.name}})); + {% endif %} + {% endfor %} + sendIfActive(std::move(resultObject), DispatchResponse::OK()); + } + + void fallThrough() override + { + fallThroughIfActive(); + } + + void sendFailure(const DispatchResponse& response) override + { + DCHECK(response.status() == DispatchResponse::kError); + sendIfActive(nullptr, response); + } +}; + {% endif %} + +DispatchResponse::Status DispatcherImpl::{{command.name}}(int callId, std::unique_ptr requestMessageObject, ErrorSupport* errors) +{ + {% if "parameters" in command %} + // Prepare input parameters. + protocol::DictionaryValue* object = DictionaryValue::cast(requestMessageObject->get("params")); + errors->push(); + {% for parameter in command.parameters %} + {% set parameter_type = protocol.resolve_type(parameter) %} + protocol::Value* {{parameter.name}}Value = object ? object->get("{{parameter.name}}") : nullptr; + {% if parameter.optional %} + Maybe<{{parameter_type.raw_type}}> in_{{parameter.name}}; + if ({{parameter.name}}Value) { + errors->setName("{{parameter.name}}"); + in_{{parameter.name}} = ValueConversions<{{parameter_type.raw_type}}>::fromValue({{parameter.name}}Value, errors); + } + {% else %} + errors->setName("{{parameter.name}}"); + {{parameter_type.type}} in_{{parameter.name}} = ValueConversions<{{parameter_type.raw_type}}>::fromValue({{parameter.name}}Value, errors); + {% endif %} + {% endfor %} + errors->pop(); + if (errors->hasErrors()) { + reportProtocolError(callId, DispatchResponse::kInvalidParams, kInvalidParamsString, errors); + return DispatchResponse::kError; + } + {% endif %} + {% if "returns" in command and not protocol.is_async_command(domain.domain, command.name) %} + // Declare output parameters. + {% for parameter in command.returns %} + {% if "optional" in parameter %} + Maybe<{{protocol.resolve_type(parameter).raw_type}}> out_{{parameter.name}}; + {% else %} + {{protocol.resolve_type(parameter).type}} out_{{parameter.name}}; + {% endif %} + {% endfor %} + {% endif %} + + {% if not protocol.is_async_command(domain.domain, command.name) %} + std::unique_ptr weak = weakPtr(); + DispatchResponse response = m_backend->{{command.name | to_method_case}}( + {%- for parameter in command.parameters -%} + {%- if not loop.first -%}, {% endif -%} + {%- if "optional" in parameter -%} + std::move(in_{{parameter.name}}) + {%- else -%} + {{protocol.resolve_type(parameter).to_pass_type % ("in_" + parameter.name)}} + {%- endif -%} + {%- endfor %} + {%- if "returns" in command %} + {%- for parameter in command.returns -%} + {%- if not loop.first or command.parameters -%}, {% endif -%} + &out_{{parameter.name}} + {%- endfor %} + {% endif %}); + if (response.status() == DispatchResponse::kFallThrough) + return response.status(); + {% if "returns" in command %} + std::unique_ptr result = DictionaryValue::create(); + if (response.status() == DispatchResponse::kSuccess) { + {% for parameter in command.returns %} + {% if "optional" in parameter %} + if (out_{{parameter.name}}.isJust()) + result->setValue("{{parameter.name}}", ValueConversions<{{protocol.resolve_type(parameter).raw_type}}>::toValue(out_{{parameter.name}}.fromJust())); + {% else %} + result->setValue("{{parameter.name}}", ValueConversions<{{protocol.resolve_type(parameter).raw_type}}>::toValue({{protocol.resolve_type(parameter).to_raw_type % ("out_" + parameter.name)}})); + {% endif %} + {% endfor %} + } + if (weak->get()) + weak->get()->sendResponse(callId, response, std::move(result)); + {% else %} + if (weak->get()) + weak->get()->sendResponse(callId, response); + {% endif %} + return response.status(); + {% else %} + std::unique_ptr weak = weakPtr(); + std::unique_ptr<{{command_name_title}}CallbackImpl> callback(new {{command.name | to_title_case}}CallbackImpl(weakPtr(), callId, nextCallbackId())); + m_backend->{{command.name | to_method_case}}( + {%- for property in command.parameters -%} + {%- if not loop.first -%}, {% endif -%} + {%- if "optional" in property -%} + std::move(in_{{property.name}}) + {%- else -%} + {{protocol.resolve_type(property).to_pass_type % ("in_" + property.name)}} + {%- endif -%} + {%- endfor -%} + {%- if command.parameters -%}, {% endif -%} + std::move(callback)); + return (weak->get() && weak->get()->lastCallbackFallThrough()) ? DispatchResponse::kFallThrough : DispatchResponse::kAsync; + {% endif %} +} + {% endfor %} + +// static +void Dispatcher::wire(UberDispatcher* uber, Backend* backend) +{ + std::unique_ptr dispatcher(new DispatcherImpl(uber->channel(), backend, uber->fallThroughForNotFound())); + uber->setupRedirects(dispatcher->redirects()); + uber->registerBackend("{{domain.domain}}", std::move(dispatcher)); +} + +} // {{domain.domain}} +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} diff --git a/tools/inspector_protocol/templates/TypeBuilder_h.template b/tools/inspector_protocol/templates/TypeBuilder_h.template new file mode 100644 index 00000000000000..744d496026a279 --- /dev/null +++ b/tools/inspector_protocol/templates/TypeBuilder_h.template @@ -0,0 +1,301 @@ +// This file is generated + +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef {{"_".join(config.protocol.namespace)}}_{{domain.domain}}_h +#define {{"_".join(config.protocol.namespace)}}_{{domain.domain}}_h + +{% if config.protocol.export_header %} +#include {{format_include(config.protocol.export_header)}} +{% endif %} +#include {{format_include(config.protocol.package, "Protocol")}} +// For each imported domain we generate a ValueConversions struct instead of a full domain definition +// and include Domain::API version from there. +{% for name in domain.dependencies %} + {% if protocol.is_imported_dependency(name) %} +#include {{format_include(config.protocol.package, name)}} + {% endif %} +{% endfor %} +{% if protocol.is_exported_domain(domain.domain) %} +#include {{format_include(config.exported.package, domain.domain)}} +{% endif %} + +{% for namespace in config.protocol.namespace %} +namespace {{namespace}} { +{% endfor %} +namespace {{domain.domain}} { + +// ------------- Forward and enum declarations. + {% for type in domain.types %} + {% if not protocol.generate_type(domain.domain, type.id) %}{% continue %}{% endif %} + {% if type.type == "object" %} + {% if "properties" in type %} +class {{type.id}}; + {% else %} +using {{type.id}} = Object; + {% endif %} + {% elif type.type != "array" %} +using {{type.id}} = {{protocol.resolve_type(type).type}}; + {% endif %} + {% endfor %} + {% for type in domain.types %} + {% if not protocol.generate_type(domain.domain, type.id) %}{% continue %}{% endif %} + {% if "enum" in type %} + +namespace {{type.id}}Enum { + {% for literal in type.enum %} +{{config.protocol.export_macro}} extern const char* {{ literal | dash_to_camelcase}}; + {% endfor %} +} // namespace {{type.id}}Enum + {% endif %} + {% endfor %} + {% for command in join_arrays(domain, ["commands", "events"]) %} + {% for param in join_arrays(command, ["parameters", "returns"]) %} + {% if "enum" in param %} + +namespace {{command.name | to_title_case}} { +namespace {{param.name | to_title_case}}Enum { + {% for literal in param.enum %} +{{config.protocol.export_macro}} extern const char* {{literal | dash_to_camelcase}}; + {% endfor %} +} // {{param.name | to_title_case}}Enum +} // {{command.name | to_title_case }} + {% endif %} + {% endfor %} + {% endfor %} + +// ------------- Type and builder declarations. + {% for type in domain.types %} + {% if not protocol.generate_type(domain.domain, type.id) %}{% continue %}{% endif %} + {% if not (type.type == "object") or not ("properties" in type) %}{% continue %}{% endif %} + +class {{config.protocol.export_macro}} {{type.id}} : public Serializable{% if protocol.is_exported(domain.domain, type.id) %}, public API::{{type.id}}{% endif %}{ + PROTOCOL_DISALLOW_COPY({{type.id}}); +public: + static std::unique_ptr<{{type.id}}> fromValue(protocol::Value* value, ErrorSupport* errors); + + ~{{type.id}}() override { } + {% for property in type.properties %} + {% set property_type = protocol.resolve_type(property) %} + {% set property_name = property.name | to_title_case %} + {% set property_field = "m_" + property.name %} + {% if "enum" in property %} + + struct {{config.protocol.export_macro}} {{property_name}}Enum { + {% for literal in property.enum %} + static const char* {{literal | dash_to_camelcase}}; + {% endfor %} + }; // {{property_name}}Enum + {% endif %} + + {% if property.optional %} + bool {{"has" | to_method_case}}{{property_name}}() { return {{property_field}}.isJust(); } + {{property_type.raw_return_type}} {{"get" | to_method_case}}{{property_name}}({{property_type.raw_pass_type}} defaultValue) { return {{property_field}}.isJust() ? {{property_field}}.fromJust() : defaultValue; } + {% else %} + {{property_type.raw_return_type}} {{"get" | to_method_case}}{{property_name}}() { return {{property_type.to_raw_type % property_field}}; } + {% endif %} + void {{"set" | to_method_case}}{{property_name}}({{property_type.pass_type}} value) { {{property_field}} = {{property_type.to_rvalue % "value"}}; } + {% endfor %} + + std::unique_ptr toValue() const; + String serialize() override { return toValue()->serialize(); } + std::unique_ptr<{{type.id}}> clone() const; + {% if protocol.is_exported(domain.domain, type.id) %} + {{config.exported.string_out}} toJSONString() const override; + {% endif %} + + template + class {{type.id}}Builder { + public: + enum { + NoFieldsSet = 0, + {% for property in type.properties|rejectattr("optional") %} + {{property.name | to_title_case}}Set = 1 << {{loop.index}}, + {% endfor %} + AllFieldsSet = ( + {%- for property in type.properties %} + {% if not(property.optional) %}{{property.name | to_title_case}}Set | {%endif %} + {% endfor %}0)}; + + {% for property in type.properties %} + {% set property_type = protocol.resolve_type(property) %} + {% set property_name = property.name | to_title_case %} + + {% if property.optional %} + {{type.id}}Builder& {{"set" | to_method_case}}{{property_name}}({{property_type.pass_type}} value) + { + m_result->{{"set" | to_method_case}}{{property_name}}({{property_type.to_rvalue % "value"}}); + return *this; + } + {% else %} + {{type.id}}Builder& {{"set" | to_method_case}}{{property_name}}({{property_type.pass_type}} value) + { + static_assert(!(STATE & {{property_name}}Set), "property {{property.name}} should not be set yet"); + m_result->{{"set" | to_method_case}}{{property_name}}({{property_type.to_rvalue % "value"}}); + return castState<{{property_name}}Set>(); + } + {% endif %} + {% endfor %} + + std::unique_ptr<{{type.id}}> {{"build" | to_method_case}}() + { + static_assert(STATE == AllFieldsSet, "state should be AllFieldsSet"); + return std::move(m_result); + } + + private: + friend class {{type.id}}; + {{type.id}}Builder() : m_result(new {{type.id}}()) { } + + template {{type.id}}Builder& castState() + { + return *reinterpret_cast<{{type.id}}Builder*>(this); + } + + {{protocol.type_definition(domain.domain + "." + type.id).type}} m_result; + }; + + static {{type.id}}Builder<0> {{"create" | to_method_case}}() + { + return {{type.id}}Builder<0>(); + } + +private: + {{type.id}}() + { + {% for property in type.properties %} + {% if not(property.optional) and "default_value" in protocol.resolve_type(property) %} + m_{{property.name}} = {{protocol.resolve_type(property).default_value}}; + {%endif %} + {% endfor %} + } + + {% for property in type.properties %} + {% if property.optional %} + Maybe<{{protocol.resolve_type(property).raw_type}}> m_{{property.name}}; + {% else %} + {{protocol.resolve_type(property).type}} m_{{property.name}}; + {% endif %} + {% endfor %} +}; + + {% endfor %} + +// ------------- Backend interface. + +class {{config.protocol.export_macro}} Backend { +public: + virtual ~Backend() { } + + {% for command in domain.commands %} + {% if "redirect" in command %}{% continue %}{% endif %} + {% if not protocol.generate_command(domain.domain, command.name) %}{% continue %}{% endif %} + {% if protocol.is_async_command(domain.domain, command.name) %} + class {{config.protocol.export_macro}} {{command.name | to_title_case}}Callback { + public: + virtual void sendSuccess( + {%- for parameter in command.returns -%} + {%- if "optional" in parameter -%} + Maybe<{{protocol.resolve_type(parameter).raw_type}}> {{parameter.name}} + {%- else -%} + {{protocol.resolve_type(parameter).pass_type}} {{parameter.name}} + {%- endif -%} + {%- if not loop.last -%}, {% endif -%} + {%- endfor -%} + ) = 0; + virtual void sendFailure(const DispatchResponse&) = 0; + virtual void fallThrough() = 0; + virtual ~{{command.name | to_title_case}}Callback() { } + }; + {% endif %} + {%- if not protocol.is_async_command(domain.domain, command.name) %} + virtual DispatchResponse {{command.name | to_method_case}}( + {%- else %} + virtual void {{command.name | to_method_case}}( + {%- endif %} + {%- for parameter in command.parameters -%} + {%- if not loop.first -%}, {% endif -%} + {%- if "optional" in parameter -%} + Maybe<{{protocol.resolve_type(parameter).raw_type}}> in_{{parameter.name}} + {%- else -%} + {{protocol.resolve_type(parameter).pass_type}} in_{{parameter.name}} + {%- endif -%} + {%- endfor -%} + {%- if protocol.is_async_command(domain.domain, command.name) -%} + {%- if command.parameters -%}, {% endif -%} + std::unique_ptr<{{command.name | to_title_case}}Callback> callback + {%- else -%} + {%- for parameter in command.returns -%} + {%- if (not loop.first) or command.parameters -%}, {% endif -%} + {%- if "optional" in parameter -%} + Maybe<{{protocol.resolve_type(parameter).raw_type}}>* out_{{parameter.name}} + {%- else -%} + {{protocol.resolve_type(parameter).type}}* out_{{parameter.name}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + ) = 0; + {% endfor %} + + {% if protocol.generate_disable(domain) %} + virtual DispatchResponse {{"disable" | to_method_case}}() + { + return DispatchResponse::OK(); + } + {% endif %} +}; + +// ------------- Frontend interface. + +class {{config.protocol.export_macro}} Frontend { +public: + explicit Frontend(FrontendChannel* frontendChannel) : m_frontendChannel(frontendChannel) { } + {% for event in domain.events %} + {% if not protocol.generate_event(domain.domain, event.name) %}{% continue %}{% endif %} + void {{event.name | to_method_case}}( + {%- for parameter in event.parameters -%} + {%- if "optional" in parameter -%} + Maybe<{{protocol.resolve_type(parameter).raw_type}}> {{parameter.name}} = Maybe<{{protocol.resolve_type(parameter).raw_type}}>() + {%- else -%} + {{protocol.resolve_type(parameter).pass_type}} {{parameter.name}} + {%- endif -%}{%- if not loop.last -%}, {% endif -%} + {%- endfor -%} + ); + {% endfor %} + + void flush(); + void sendRawNotification(const String&); +private: + FrontendChannel* m_frontendChannel; +}; + +// ------------- Dispatcher. + +class {{config.protocol.export_macro}} Dispatcher { +public: + static void wire(UberDispatcher*, Backend*); + +private: + Dispatcher() { } +}; + +// ------------- Metainfo. + +class {{config.protocol.export_macro}} Metainfo { +public: + using BackendClass = Backend; + using FrontendClass = Frontend; + using DispatcherClass = Dispatcher; + static const char domainName[]; + static const char commandPrefix[]; + static const char version[]; +}; + +} // namespace {{domain.domain}} +{% for namespace in config.protocol.namespace %} +} // namespace {{namespace}} +{% endfor %} + +#endif // !defined({{"_".join(config.protocol.namespace)}}_{{domain.domain}}_h) diff --git a/tools/jinja2/AUTHORS b/tools/jinja2/AUTHORS new file mode 100644 index 00000000000000..fd6dbfcbfd8fe2 --- /dev/null +++ b/tools/jinja2/AUTHORS @@ -0,0 +1,34 @@ +Jinja is written and maintained by the Jinja Team and various +contributors: + +Lead Developer: + +- Armin Ronacher + +Developers: + +- Christoph Hack +- Georg Brandl + +Contributors: + +- Bryan McLemore +- Mickaël Guérin +- Cameron Knight +- Lawrence Journal-World. +- David Cramer +- Adrian Mönnich (ThiefMaster) + +Patches and suggestions: + +- Ronny Pfannschmidt +- Axel Böhm +- Alexey Melchakov +- Bryan McLemore +- Clovis Fabricio (nosklo) +- Cameron Knight +- Peter van Dijk (Habbie) +- Stefan Ebner +- Rene Leonhardt +- Thomas Waldmann +- Cory Benfield (Lukasa) diff --git a/tools/jinja2/Jinja2-2.10.tar.gz.md5 b/tools/jinja2/Jinja2-2.10.tar.gz.md5 new file mode 100644 index 00000000000000..9137ee129a0a6f --- /dev/null +++ b/tools/jinja2/Jinja2-2.10.tar.gz.md5 @@ -0,0 +1 @@ +61ef1117f945486472850819b8d1eb3d Jinja2-2.10.tar.gz diff --git a/tools/jinja2/Jinja2-2.10.tar.gz.sha512 b/tools/jinja2/Jinja2-2.10.tar.gz.sha512 new file mode 100644 index 00000000000000..087d24c18eb80d --- /dev/null +++ b/tools/jinja2/Jinja2-2.10.tar.gz.sha512 @@ -0,0 +1 @@ +0ea7371be67ffcf19e46dfd06523a45a0806e678a407d54f5f2f3e573982f0959cf82ec5d07b203670309928a62ef71109701ab16547a9bba2ebcdc178cb67f2 Jinja2-2.10.tar.gz diff --git a/tools/jinja2/LICENSE b/tools/jinja2/LICENSE new file mode 100644 index 00000000000000..31bf900e58e30f --- /dev/null +++ b/tools/jinja2/LICENSE @@ -0,0 +1,31 @@ +Copyright (c) 2009 by the Jinja Team, see AUTHORS for more details. + +Some rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * The names of the contributors may not be used to endorse or + promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tools/jinja2/OWNERS b/tools/jinja2/OWNERS new file mode 100644 index 00000000000000..ee2bec9ba3a3a2 --- /dev/null +++ b/tools/jinja2/OWNERS @@ -0,0 +1,6 @@ +timloh@chromium.org +haraken@chromium.org +nbarth@chromium.org + +# TEAM: platform-architecture-dev@chromium.org +# COMPONENT: Blink>Internals diff --git a/tools/jinja2/README.chromium b/tools/jinja2/README.chromium new file mode 100644 index 00000000000000..5246c2f84b6021 --- /dev/null +++ b/tools/jinja2/README.chromium @@ -0,0 +1,26 @@ +Name: Jinja2 Python Template Engine +Short Name: jinja2 +URL: http://jinja.pocoo.org/ +Version: 2.10 +License: BSD 3-Clause +License File: LICENSE +Security Critical: no + +Description: +Template engine for code generation in Blink. + +Source: https://pypi.python.org/packages/56/e6/332789f295cf22308386cf5bbd1f4e00ed11484299c5d7383378cf48ba47/Jinja2-2.10.tar.gz +MD5: 61ef1117f945486472850819b8d1eb3d +SHA-1: 34b69e5caab12ee37b9df69df9018776c008b7b8 + +Local Modifications: +This only includes the jinja2 directory from the tarball and the LICENSE and +AUTHORS files. Unit tests (testsuite directory) have been removed. +Additional chromium-specific files are: +* README.chromium (this file) +* OWNERS +* get_jinja2.sh (install script) +* jinja2.gni (generated by get_jinja2.sh) +* files of hashes (MD5 is also posted on website, SHA-512 computed locally). +Script checks hash then unpacks archive and installs desired files. +Retrieve or update by executing jinja2/get_jinja2.sh from third_party. diff --git a/tools/jinja2/__init__.py b/tools/jinja2/__init__.py new file mode 100644 index 00000000000000..42aa763d571e5b --- /dev/null +++ b/tools/jinja2/__init__.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +""" + jinja2 + ~~~~~~ + + Jinja2 is a template engine written in pure Python. It provides a + Django inspired non-XML syntax but supports inline expressions and + an optional sandboxed environment. + + Nutshell + -------- + + Here a small example of a Jinja2 template:: + + {% extends 'base.html' %} + {% block title %}Memberlist{% endblock %} + {% block content %} + + {% endblock %} + + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +__docformat__ = 'restructuredtext en' +__version__ = '2.10' + +# high level interface +from jinja2.environment import Environment, Template + +# loaders +from jinja2.loaders import BaseLoader, FileSystemLoader, PackageLoader, \ + DictLoader, FunctionLoader, PrefixLoader, ChoiceLoader, \ + ModuleLoader + +# bytecode caches +from jinja2.bccache import BytecodeCache, FileSystemBytecodeCache, \ + MemcachedBytecodeCache + +# undefined types +from jinja2.runtime import Undefined, DebugUndefined, StrictUndefined, \ + make_logging_undefined + +# exceptions +from jinja2.exceptions import TemplateError, UndefinedError, \ + TemplateNotFound, TemplatesNotFound, TemplateSyntaxError, \ + TemplateAssertionError, TemplateRuntimeError + +# decorators and public utilities +from jinja2.filters import environmentfilter, contextfilter, \ + evalcontextfilter +from jinja2.utils import Markup, escape, clear_caches, \ + environmentfunction, evalcontextfunction, contextfunction, \ + is_undefined, select_autoescape + +__all__ = [ + 'Environment', 'Template', 'BaseLoader', 'FileSystemLoader', + 'PackageLoader', 'DictLoader', 'FunctionLoader', 'PrefixLoader', + 'ChoiceLoader', 'BytecodeCache', 'FileSystemBytecodeCache', + 'MemcachedBytecodeCache', 'Undefined', 'DebugUndefined', + 'StrictUndefined', 'TemplateError', 'UndefinedError', 'TemplateNotFound', + 'TemplatesNotFound', 'TemplateSyntaxError', 'TemplateAssertionError', + 'TemplateRuntimeError', + 'ModuleLoader', 'environmentfilter', 'contextfilter', 'Markup', 'escape', + 'environmentfunction', 'contextfunction', 'clear_caches', 'is_undefined', + 'evalcontextfilter', 'evalcontextfunction', 'make_logging_undefined', + 'select_autoescape', +] + + +def _patch_async(): + from jinja2.utils import have_async_gen + if have_async_gen: + from jinja2.asyncsupport import patch_all + patch_all() + + +_patch_async() +del _patch_async diff --git a/tools/jinja2/_compat.py b/tools/jinja2/_compat.py new file mode 100644 index 00000000000000..61d85301a4a9ef --- /dev/null +++ b/tools/jinja2/_compat.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +""" + jinja2._compat + ~~~~~~~~~~~~~~ + + Some py2/py3 compatibility support based on a stripped down + version of six so we don't have to depend on a specific version + of it. + + :copyright: Copyright 2013 by the Jinja team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +import sys + +PY2 = sys.version_info[0] == 2 +PYPY = hasattr(sys, 'pypy_translation_info') +_identity = lambda x: x + + +if not PY2: + unichr = chr + range_type = range + text_type = str + string_types = (str,) + integer_types = (int,) + + iterkeys = lambda d: iter(d.keys()) + itervalues = lambda d: iter(d.values()) + iteritems = lambda d: iter(d.items()) + + import pickle + from io import BytesIO, StringIO + NativeStringIO = StringIO + + def reraise(tp, value, tb=None): + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + + ifilter = filter + imap = map + izip = zip + intern = sys.intern + + implements_iterator = _identity + implements_to_string = _identity + encode_filename = _identity + +else: + unichr = unichr + text_type = unicode + range_type = xrange + string_types = (str, unicode) + integer_types = (int, long) + + iterkeys = lambda d: d.iterkeys() + itervalues = lambda d: d.itervalues() + iteritems = lambda d: d.iteritems() + + import cPickle as pickle + from cStringIO import StringIO as BytesIO, StringIO + NativeStringIO = BytesIO + + exec('def reraise(tp, value, tb=None):\n raise tp, value, tb') + + from itertools import imap, izip, ifilter + intern = intern + + def implements_iterator(cls): + cls.next = cls.__next__ + del cls.__next__ + return cls + + def implements_to_string(cls): + cls.__unicode__ = cls.__str__ + cls.__str__ = lambda x: x.__unicode__().encode('utf-8') + return cls + + def encode_filename(filename): + if isinstance(filename, unicode): + return filename.encode('utf-8') + return filename + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + # This requires a bit of explanation: the basic idea is to make a + # dummy metaclass for one level of class instantiation that replaces + # itself with the actual metaclass. + class metaclass(type): + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + return type.__new__(metaclass, 'temporary_class', (), {}) + + +try: + from urllib.parse import quote_from_bytes as url_quote +except ImportError: + from urllib import quote as url_quote diff --git a/tools/jinja2/_identifier.py b/tools/jinja2/_identifier.py new file mode 100644 index 00000000000000..2eac35d5c35531 --- /dev/null +++ b/tools/jinja2/_identifier.py @@ -0,0 +1,2 @@ +# generated by scripts/generate_identifier_pattern.py +pattern = '·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛ࣔ-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఃా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഁ-ഃാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳸᳹᷀-᷵᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑅳𑄴𑆀-𑆂𑆳-𑇊𑇀-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯' diff --git a/tools/jinja2/asyncfilters.py b/tools/jinja2/asyncfilters.py new file mode 100644 index 00000000000000..5c1f46d7fa7d46 --- /dev/null +++ b/tools/jinja2/asyncfilters.py @@ -0,0 +1,146 @@ +from functools import wraps + +from jinja2.asyncsupport import auto_aiter +from jinja2 import filters + + +async def auto_to_seq(value): + seq = [] + if hasattr(value, '__aiter__'): + async for item in value: + seq.append(item) + else: + for item in value: + seq.append(item) + return seq + + +async def async_select_or_reject(args, kwargs, modfunc, lookup_attr): + seq, func = filters.prepare_select_or_reject( + args, kwargs, modfunc, lookup_attr) + if seq: + async for item in auto_aiter(seq): + if func(item): + yield item + + +def dualfilter(normal_filter, async_filter): + wrap_evalctx = False + if getattr(normal_filter, 'environmentfilter', False): + is_async = lambda args: args[0].is_async + wrap_evalctx = False + else: + if not getattr(normal_filter, 'evalcontextfilter', False) and \ + not getattr(normal_filter, 'contextfilter', False): + wrap_evalctx = True + is_async = lambda args: args[0].environment.is_async + + @wraps(normal_filter) + def wrapper(*args, **kwargs): + b = is_async(args) + if wrap_evalctx: + args = args[1:] + if b: + return async_filter(*args, **kwargs) + return normal_filter(*args, **kwargs) + + if wrap_evalctx: + wrapper.evalcontextfilter = True + + wrapper.asyncfiltervariant = True + + return wrapper + + +def asyncfiltervariant(original): + def decorator(f): + return dualfilter(original, f) + return decorator + + +@asyncfiltervariant(filters.do_first) +async def do_first(environment, seq): + try: + return await auto_aiter(seq).__anext__() + except StopAsyncIteration: + return environment.undefined('No first item, sequence was empty.') + + +@asyncfiltervariant(filters.do_groupby) +async def do_groupby(environment, value, attribute): + expr = filters.make_attrgetter(environment, attribute) + return [filters._GroupTuple(key, await auto_to_seq(values)) + for key, values in filters.groupby(sorted( + await auto_to_seq(value), key=expr), expr)] + + +@asyncfiltervariant(filters.do_join) +async def do_join(eval_ctx, value, d=u'', attribute=None): + return filters.do_join(eval_ctx, await auto_to_seq(value), d, attribute) + + +@asyncfiltervariant(filters.do_list) +async def do_list(value): + return await auto_to_seq(value) + + +@asyncfiltervariant(filters.do_reject) +async def do_reject(*args, **kwargs): + return async_select_or_reject(args, kwargs, lambda x: not x, False) + + +@asyncfiltervariant(filters.do_rejectattr) +async def do_rejectattr(*args, **kwargs): + return async_select_or_reject(args, kwargs, lambda x: not x, True) + + +@asyncfiltervariant(filters.do_select) +async def do_select(*args, **kwargs): + return async_select_or_reject(args, kwargs, lambda x: x, False) + + +@asyncfiltervariant(filters.do_selectattr) +async def do_selectattr(*args, **kwargs): + return async_select_or_reject(args, kwargs, lambda x: x, True) + + +@asyncfiltervariant(filters.do_map) +async def do_map(*args, **kwargs): + seq, func = filters.prepare_map(args, kwargs) + if seq: + async for item in auto_aiter(seq): + yield func(item) + + +@asyncfiltervariant(filters.do_sum) +async def do_sum(environment, iterable, attribute=None, start=0): + rv = start + if attribute is not None: + func = filters.make_attrgetter(environment, attribute) + else: + func = lambda x: x + async for item in auto_aiter(iterable): + rv += func(item) + return rv + + +@asyncfiltervariant(filters.do_slice) +async def do_slice(value, slices, fill_with=None): + return filters.do_slice(await auto_to_seq(value), slices, fill_with) + + +ASYNC_FILTERS = { + 'first': do_first, + 'groupby': do_groupby, + 'join': do_join, + 'list': do_list, + # we intentionally do not support do_last because that would be + # ridiculous + 'reject': do_reject, + 'rejectattr': do_rejectattr, + 'map': do_map, + 'select': do_select, + 'selectattr': do_selectattr, + 'sum': do_sum, + 'slice': do_slice, +} diff --git a/tools/jinja2/asyncsupport.py b/tools/jinja2/asyncsupport.py new file mode 100644 index 00000000000000..b1e7b5ce9a27a6 --- /dev/null +++ b/tools/jinja2/asyncsupport.py @@ -0,0 +1,256 @@ +# -*- coding: utf-8 -*- +""" + jinja2.asyncsupport + ~~~~~~~~~~~~~~~~~~~ + + Has all the code for async support which is implemented as a patch + for supported Python versions. + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import sys +import asyncio +import inspect +from functools import update_wrapper + +from jinja2.utils import concat, internalcode, Markup +from jinja2.environment import TemplateModule +from jinja2.runtime import LoopContextBase, _last_iteration + + +async def concat_async(async_gen): + rv = [] + async def collect(): + async for event in async_gen: + rv.append(event) + await collect() + return concat(rv) + + +async def generate_async(self, *args, **kwargs): + vars = dict(*args, **kwargs) + try: + async for event in self.root_render_func(self.new_context(vars)): + yield event + except Exception: + exc_info = sys.exc_info() + else: + return + yield self.environment.handle_exception(exc_info, True) + + +def wrap_generate_func(original_generate): + def _convert_generator(self, loop, args, kwargs): + async_gen = self.generate_async(*args, **kwargs) + try: + while 1: + yield loop.run_until_complete(async_gen.__anext__()) + except StopAsyncIteration: + pass + def generate(self, *args, **kwargs): + if not self.environment.is_async: + return original_generate(self, *args, **kwargs) + return _convert_generator(self, asyncio.get_event_loop(), args, kwargs) + return update_wrapper(generate, original_generate) + + +async def render_async(self, *args, **kwargs): + if not self.environment.is_async: + raise RuntimeError('The environment was not created with async mode ' + 'enabled.') + + vars = dict(*args, **kwargs) + ctx = self.new_context(vars) + + try: + return await concat_async(self.root_render_func(ctx)) + except Exception: + exc_info = sys.exc_info() + return self.environment.handle_exception(exc_info, True) + + +def wrap_render_func(original_render): + def render(self, *args, **kwargs): + if not self.environment.is_async: + return original_render(self, *args, **kwargs) + loop = asyncio.get_event_loop() + return loop.run_until_complete(self.render_async(*args, **kwargs)) + return update_wrapper(render, original_render) + + +def wrap_block_reference_call(original_call): + @internalcode + async def async_call(self): + rv = await concat_async(self._stack[self._depth](self._context)) + if self._context.eval_ctx.autoescape: + rv = Markup(rv) + return rv + + @internalcode + def __call__(self): + if not self._context.environment.is_async: + return original_call(self) + return async_call(self) + + return update_wrapper(__call__, original_call) + + +def wrap_macro_invoke(original_invoke): + @internalcode + async def async_invoke(self, arguments, autoescape): + rv = await self._func(*arguments) + if autoescape: + rv = Markup(rv) + return rv + + @internalcode + def _invoke(self, arguments, autoescape): + if not self._environment.is_async: + return original_invoke(self, arguments, autoescape) + return async_invoke(self, arguments, autoescape) + return update_wrapper(_invoke, original_invoke) + + +@internalcode +async def get_default_module_async(self): + if self._module is not None: + return self._module + self._module = rv = await self.make_module_async() + return rv + + +def wrap_default_module(original_default_module): + @internalcode + def _get_default_module(self): + if self.environment.is_async: + raise RuntimeError('Template module attribute is unavailable ' + 'in async mode') + return original_default_module(self) + return _get_default_module + + +async def make_module_async(self, vars=None, shared=False, locals=None): + context = self.new_context(vars, shared, locals) + body_stream = [] + async for item in self.root_render_func(context): + body_stream.append(item) + return TemplateModule(self, context, body_stream) + + +def patch_template(): + from jinja2 import Template + Template.generate = wrap_generate_func(Template.generate) + Template.generate_async = update_wrapper( + generate_async, Template.generate_async) + Template.render_async = update_wrapper( + render_async, Template.render_async) + Template.render = wrap_render_func(Template.render) + Template._get_default_module = wrap_default_module( + Template._get_default_module) + Template._get_default_module_async = get_default_module_async + Template.make_module_async = update_wrapper( + make_module_async, Template.make_module_async) + + +def patch_runtime(): + from jinja2.runtime import BlockReference, Macro + BlockReference.__call__ = wrap_block_reference_call( + BlockReference.__call__) + Macro._invoke = wrap_macro_invoke(Macro._invoke) + + +def patch_filters(): + from jinja2.filters import FILTERS + from jinja2.asyncfilters import ASYNC_FILTERS + FILTERS.update(ASYNC_FILTERS) + + +def patch_all(): + patch_template() + patch_runtime() + patch_filters() + + +async def auto_await(value): + if inspect.isawaitable(value): + return await value + return value + + +async def auto_aiter(iterable): + if hasattr(iterable, '__aiter__'): + async for item in iterable: + yield item + return + for item in iterable: + yield item + + +class AsyncLoopContext(LoopContextBase): + + def __init__(self, async_iterator, undefined, after, length, recurse=None, + depth0=0): + LoopContextBase.__init__(self, undefined, recurse, depth0) + self._async_iterator = async_iterator + self._after = after + self._length = length + + @property + def length(self): + if self._length is None: + raise TypeError('Loop length for some iterators cannot be ' + 'lazily calculated in async mode') + return self._length + + def __aiter__(self): + return AsyncLoopContextIterator(self) + + +class AsyncLoopContextIterator(object): + __slots__ = ('context',) + + def __init__(self, context): + self.context = context + + def __aiter__(self): + return self + + async def __anext__(self): + ctx = self.context + ctx.index0 += 1 + if ctx._after is _last_iteration: + raise StopAsyncIteration() + ctx._before = ctx._current + ctx._current = ctx._after + try: + ctx._after = await ctx._async_iterator.__anext__() + except StopAsyncIteration: + ctx._after = _last_iteration + return ctx._current, ctx + + +async def make_async_loop_context(iterable, undefined, recurse=None, depth0=0): + # Length is more complicated and less efficient in async mode. The + # reason for this is that we cannot know if length will be used + # upfront but because length is a property we cannot lazily execute it + # later. This means that we need to buffer it up and measure :( + # + # We however only do this for actual iterators, not for async + # iterators as blocking here does not seem like the best idea in the + # world. + try: + length = len(iterable) + except (TypeError, AttributeError): + if not hasattr(iterable, '__aiter__'): + iterable = tuple(iterable) + length = len(iterable) + else: + length = None + async_iterator = auto_aiter(iterable) + try: + after = await async_iterator.__anext__() + except StopAsyncIteration: + after = _last_iteration + return AsyncLoopContext(async_iterator, undefined, after, length, recurse, + depth0) diff --git a/tools/jinja2/bccache.py b/tools/jinja2/bccache.py new file mode 100644 index 00000000000000..080e527cabf33b --- /dev/null +++ b/tools/jinja2/bccache.py @@ -0,0 +1,362 @@ +# -*- coding: utf-8 -*- +""" + jinja2.bccache + ~~~~~~~~~~~~~~ + + This module implements the bytecode cache system Jinja is optionally + using. This is useful if you have very complex template situations and + the compiliation of all those templates slow down your application too + much. + + Situations where this is useful are often forking web applications that + are initialized on the first request. + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD. +""" +from os import path, listdir +import os +import sys +import stat +import errno +import marshal +import tempfile +import fnmatch +from hashlib import sha1 +from jinja2.utils import open_if_exists +from jinja2._compat import BytesIO, pickle, PY2, text_type + + +# marshal works better on 3.x, one hack less required +if not PY2: + marshal_dump = marshal.dump + marshal_load = marshal.load +else: + + def marshal_dump(code, f): + if isinstance(f, file): + marshal.dump(code, f) + else: + f.write(marshal.dumps(code)) + + def marshal_load(f): + if isinstance(f, file): + return marshal.load(f) + return marshal.loads(f.read()) + + +bc_version = 3 + +# magic version used to only change with new jinja versions. With 2.6 +# we change this to also take Python version changes into account. The +# reason for this is that Python tends to segfault if fed earlier bytecode +# versions because someone thought it would be a good idea to reuse opcodes +# or make Python incompatible with earlier versions. +bc_magic = 'j2'.encode('ascii') + \ + pickle.dumps(bc_version, 2) + \ + pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1]) + + +class Bucket(object): + """Buckets are used to store the bytecode for one template. It's created + and initialized by the bytecode cache and passed to the loading functions. + + The buckets get an internal checksum from the cache assigned and use this + to automatically reject outdated cache material. Individual bytecode + cache subclasses don't have to care about cache invalidation. + """ + + def __init__(self, environment, key, checksum): + self.environment = environment + self.key = key + self.checksum = checksum + self.reset() + + def reset(self): + """Resets the bucket (unloads the bytecode).""" + self.code = None + + def load_bytecode(self, f): + """Loads bytecode from a file or file like object.""" + # make sure the magic header is correct + magic = f.read(len(bc_magic)) + if magic != bc_magic: + self.reset() + return + # the source code of the file changed, we need to reload + checksum = pickle.load(f) + if self.checksum != checksum: + self.reset() + return + # if marshal_load fails then we need to reload + try: + self.code = marshal_load(f) + except (EOFError, ValueError, TypeError): + self.reset() + return + + def write_bytecode(self, f): + """Dump the bytecode into the file or file like object passed.""" + if self.code is None: + raise TypeError('can\'t write empty bucket') + f.write(bc_magic) + pickle.dump(self.checksum, f, 2) + marshal_dump(self.code, f) + + def bytecode_from_string(self, string): + """Load bytecode from a string.""" + self.load_bytecode(BytesIO(string)) + + def bytecode_to_string(self): + """Return the bytecode as string.""" + out = BytesIO() + self.write_bytecode(out) + return out.getvalue() + + +class BytecodeCache(object): + """To implement your own bytecode cache you have to subclass this class + and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of + these methods are passed a :class:`~jinja2.bccache.Bucket`. + + A very basic bytecode cache that saves the bytecode on the file system:: + + from os import path + + class MyCache(BytecodeCache): + + def __init__(self, directory): + self.directory = directory + + def load_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + if path.exists(filename): + with open(filename, 'rb') as f: + bucket.load_bytecode(f) + + def dump_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + with open(filename, 'wb') as f: + bucket.write_bytecode(f) + + A more advanced version of a filesystem based bytecode cache is part of + Jinja2. + """ + + def load_bytecode(self, bucket): + """Subclasses have to override this method to load bytecode into a + bucket. If they are not able to find code in the cache for the + bucket, it must not do anything. + """ + raise NotImplementedError() + + def dump_bytecode(self, bucket): + """Subclasses have to override this method to write the bytecode + from a bucket back to the cache. If it unable to do so it must not + fail silently but raise an exception. + """ + raise NotImplementedError() + + def clear(self): + """Clears the cache. This method is not used by Jinja2 but should be + implemented to allow applications to clear the bytecode cache used + by a particular environment. + """ + + def get_cache_key(self, name, filename=None): + """Returns the unique hash key for this template name.""" + hash = sha1(name.encode('utf-8')) + if filename is not None: + filename = '|' + filename + if isinstance(filename, text_type): + filename = filename.encode('utf-8') + hash.update(filename) + return hash.hexdigest() + + def get_source_checksum(self, source): + """Returns a checksum for the source.""" + return sha1(source.encode('utf-8')).hexdigest() + + def get_bucket(self, environment, name, filename, source): + """Return a cache bucket for the given template. All arguments are + mandatory but filename may be `None`. + """ + key = self.get_cache_key(name, filename) + checksum = self.get_source_checksum(source) + bucket = Bucket(environment, key, checksum) + self.load_bytecode(bucket) + return bucket + + def set_bucket(self, bucket): + """Put the bucket into the cache.""" + self.dump_bytecode(bucket) + + +class FileSystemBytecodeCache(BytecodeCache): + """A bytecode cache that stores bytecode on the filesystem. It accepts + two arguments: The directory where the cache items are stored and a + pattern string that is used to build the filename. + + If no directory is specified a default cache directory is selected. On + Windows the user's temp directory is used, on UNIX systems a directory + is created for the user in the system temp directory. + + The pattern can be used to have multiple separate caches operate on the + same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s`` + is replaced with the cache key. + + >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache') + + This bytecode cache supports clearing of the cache using the clear method. + """ + + def __init__(self, directory=None, pattern='__jinja2_%s.cache'): + if directory is None: + directory = self._get_default_cache_dir() + self.directory = directory + self.pattern = pattern + + def _get_default_cache_dir(self): + def _unsafe_dir(): + raise RuntimeError('Cannot determine safe temp directory. You ' + 'need to explicitly provide one.') + + tmpdir = tempfile.gettempdir() + + # On windows the temporary directory is used specific unless + # explicitly forced otherwise. We can just use that. + if os.name == 'nt': + return tmpdir + if not hasattr(os, 'getuid'): + _unsafe_dir() + + dirname = '_jinja2-cache-%d' % os.getuid() + actual_dir = os.path.join(tmpdir, dirname) + + try: + os.mkdir(actual_dir, stat.S_IRWXU) + except OSError as e: + if e.errno != errno.EEXIST: + raise + try: + os.chmod(actual_dir, stat.S_IRWXU) + actual_dir_stat = os.lstat(actual_dir) + if actual_dir_stat.st_uid != os.getuid() \ + or not stat.S_ISDIR(actual_dir_stat.st_mode) \ + or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU: + _unsafe_dir() + except OSError as e: + if e.errno != errno.EEXIST: + raise + + actual_dir_stat = os.lstat(actual_dir) + if actual_dir_stat.st_uid != os.getuid() \ + or not stat.S_ISDIR(actual_dir_stat.st_mode) \ + or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU: + _unsafe_dir() + + return actual_dir + + def _get_cache_filename(self, bucket): + return path.join(self.directory, self.pattern % bucket.key) + + def load_bytecode(self, bucket): + f = open_if_exists(self._get_cache_filename(bucket), 'rb') + if f is not None: + try: + bucket.load_bytecode(f) + finally: + f.close() + + def dump_bytecode(self, bucket): + f = open(self._get_cache_filename(bucket), 'wb') + try: + bucket.write_bytecode(f) + finally: + f.close() + + def clear(self): + # imported lazily here because google app-engine doesn't support + # write access on the file system and the function does not exist + # normally. + from os import remove + files = fnmatch.filter(listdir(self.directory), self.pattern % '*') + for filename in files: + try: + remove(path.join(self.directory, filename)) + except OSError: + pass + + +class MemcachedBytecodeCache(BytecodeCache): + """This class implements a bytecode cache that uses a memcache cache for + storing the information. It does not enforce a specific memcache library + (tummy's memcache or cmemcache) but will accept any class that provides + the minimal interface required. + + Libraries compatible with this class: + + - `werkzeug `_.contrib.cache + - `python-memcached `_ + - `cmemcache `_ + + (Unfortunately the django cache interface is not compatible because it + does not support storing binary data, only unicode. You can however pass + the underlying cache client to the bytecode cache which is available + as `django.core.cache.cache._client`.) + + The minimal interface for the client passed to the constructor is this: + + .. class:: MinimalClientInterface + + .. method:: set(key, value[, timeout]) + + Stores the bytecode in the cache. `value` is a string and + `timeout` the timeout of the key. If timeout is not provided + a default timeout or no timeout should be assumed, if it's + provided it's an integer with the number of seconds the cache + item should exist. + + .. method:: get(key) + + Returns the value for the cache key. If the item does not + exist in the cache the return value must be `None`. + + The other arguments to the constructor are the prefix for all keys that + is added before the actual cache key and the timeout for the bytecode in + the cache system. We recommend a high (or no) timeout. + + This bytecode cache does not support clearing of used items in the cache. + The clear method is a no-operation function. + + .. versionadded:: 2.7 + Added support for ignoring memcache errors through the + `ignore_memcache_errors` parameter. + """ + + def __init__(self, client, prefix='jinja2/bytecode/', timeout=None, + ignore_memcache_errors=True): + self.client = client + self.prefix = prefix + self.timeout = timeout + self.ignore_memcache_errors = ignore_memcache_errors + + def load_bytecode(self, bucket): + try: + code = self.client.get(self.prefix + bucket.key) + except Exception: + if not self.ignore_memcache_errors: + raise + code = None + if code is not None: + bucket.bytecode_from_string(code) + + def dump_bytecode(self, bucket): + args = (self.prefix + bucket.key, bucket.bytecode_to_string()) + if self.timeout is not None: + args += (self.timeout,) + try: + self.client.set(*args) + except Exception: + if not self.ignore_memcache_errors: + raise diff --git a/tools/jinja2/compiler.py b/tools/jinja2/compiler.py new file mode 100644 index 00000000000000..d534a827391aeb --- /dev/null +++ b/tools/jinja2/compiler.py @@ -0,0 +1,1721 @@ +# -*- coding: utf-8 -*- +""" + jinja2.compiler + ~~~~~~~~~~~~~~~ + + Compiles nodes into python code. + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +from itertools import chain +from copy import deepcopy +from keyword import iskeyword as is_python_keyword +from functools import update_wrapper +from jinja2 import nodes +from jinja2.nodes import EvalContext +from jinja2.visitor import NodeVisitor +from jinja2.optimizer import Optimizer +from jinja2.exceptions import TemplateAssertionError +from jinja2.utils import Markup, concat, escape +from jinja2._compat import range_type, text_type, string_types, \ + iteritems, NativeStringIO, imap, izip +from jinja2.idtracking import Symbols, VAR_LOAD_PARAMETER, \ + VAR_LOAD_RESOLVE, VAR_LOAD_ALIAS, VAR_LOAD_UNDEFINED + + +operators = { + 'eq': '==', + 'ne': '!=', + 'gt': '>', + 'gteq': '>=', + 'lt': '<', + 'lteq': '<=', + 'in': 'in', + 'notin': 'not in' +} + +# what method to iterate over items do we want to use for dict iteration +# in generated code? on 2.x let's go with iteritems, on 3.x with items +if hasattr(dict, 'iteritems'): + dict_item_iter = 'iteritems' +else: + dict_item_iter = 'items' + +code_features = ['division'] + +# does this python version support generator stops? (PEP 0479) +try: + exec('from __future__ import generator_stop') + code_features.append('generator_stop') +except SyntaxError: + pass + +# does this python version support yield from? +try: + exec('def f(): yield from x()') +except SyntaxError: + supports_yield_from = False +else: + supports_yield_from = True + + +def optimizeconst(f): + def new_func(self, node, frame, **kwargs): + # Only optimize if the frame is not volatile + if self.optimized and not frame.eval_ctx.volatile: + new_node = self.optimizer.visit(node, frame.eval_ctx) + if new_node != node: + return self.visit(new_node, frame) + return f(self, node, frame, **kwargs) + return update_wrapper(new_func, f) + + +def generate(node, environment, name, filename, stream=None, + defer_init=False, optimized=True): + """Generate the python source for a node tree.""" + if not isinstance(node, nodes.Template): + raise TypeError('Can\'t compile non template nodes') + generator = environment.code_generator_class(environment, name, filename, + stream, defer_init, + optimized) + generator.visit(node) + if stream is None: + return generator.stream.getvalue() + + +def has_safe_repr(value): + """Does the node have a safe representation?""" + if value is None or value is NotImplemented or value is Ellipsis: + return True + if type(value) in (bool, int, float, complex, range_type, Markup) + string_types: + return True + if type(value) in (tuple, list, set, frozenset): + for item in value: + if not has_safe_repr(item): + return False + return True + elif type(value) is dict: + for key, value in iteritems(value): + if not has_safe_repr(key): + return False + if not has_safe_repr(value): + return False + return True + return False + + +def find_undeclared(nodes, names): + """Check if the names passed are accessed undeclared. The return value + is a set of all the undeclared names from the sequence of names found. + """ + visitor = UndeclaredNameVisitor(names) + try: + for node in nodes: + visitor.visit(node) + except VisitorExit: + pass + return visitor.undeclared + + +class MacroRef(object): + + def __init__(self, node): + self.node = node + self.accesses_caller = False + self.accesses_kwargs = False + self.accesses_varargs = False + + +class Frame(object): + """Holds compile time information for us.""" + + def __init__(self, eval_ctx, parent=None, level=None): + self.eval_ctx = eval_ctx + self.symbols = Symbols(parent and parent.symbols or None, + level=level) + + # a toplevel frame is the root + soft frames such as if conditions. + self.toplevel = False + + # the root frame is basically just the outermost frame, so no if + # conditions. This information is used to optimize inheritance + # situations. + self.rootlevel = False + + # in some dynamic inheritance situations the compiler needs to add + # write tests around output statements. + self.require_output_check = parent and parent.require_output_check + + # inside some tags we are using a buffer rather than yield statements. + # this for example affects {% filter %} or {% macro %}. If a frame + # is buffered this variable points to the name of the list used as + # buffer. + self.buffer = None + + # the name of the block we're in, otherwise None. + self.block = parent and parent.block or None + + # the parent of this frame + self.parent = parent + + if parent is not None: + self.buffer = parent.buffer + + def copy(self): + """Create a copy of the current one.""" + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.symbols = self.symbols.copy() + return rv + + def inner(self, isolated=False): + """Return an inner frame.""" + if isolated: + return Frame(self.eval_ctx, level=self.symbols.level + 1) + return Frame(self.eval_ctx, self) + + def soft(self): + """Return a soft frame. A soft frame may not be modified as + standalone thing as it shares the resources with the frame it + was created of, but it's not a rootlevel frame any longer. + + This is only used to implement if-statements. + """ + rv = self.copy() + rv.rootlevel = False + return rv + + __copy__ = copy + + +class VisitorExit(RuntimeError): + """Exception used by the `UndeclaredNameVisitor` to signal a stop.""" + + +class DependencyFinderVisitor(NodeVisitor): + """A visitor that collects filter and test calls.""" + + def __init__(self): + self.filters = set() + self.tests = set() + + def visit_Filter(self, node): + self.generic_visit(node) + self.filters.add(node.name) + + def visit_Test(self, node): + self.generic_visit(node) + self.tests.add(node.name) + + def visit_Block(self, node): + """Stop visiting at blocks.""" + + +class UndeclaredNameVisitor(NodeVisitor): + """A visitor that checks if a name is accessed without being + declared. This is different from the frame visitor as it will + not stop at closure frames. + """ + + def __init__(self, names): + self.names = set(names) + self.undeclared = set() + + def visit_Name(self, node): + if node.ctx == 'load' and node.name in self.names: + self.undeclared.add(node.name) + if self.undeclared == self.names: + raise VisitorExit() + else: + self.names.discard(node.name) + + def visit_Block(self, node): + """Stop visiting a blocks.""" + + +class CompilerExit(Exception): + """Raised if the compiler encountered a situation where it just + doesn't make sense to further process the code. Any block that + raises such an exception is not further processed. + """ + + +class CodeGenerator(NodeVisitor): + + def __init__(self, environment, name, filename, stream=None, + defer_init=False, optimized=True): + if stream is None: + stream = NativeStringIO() + self.environment = environment + self.name = name + self.filename = filename + self.stream = stream + self.created_block_context = False + self.defer_init = defer_init + self.optimized = optimized + if optimized: + self.optimizer = Optimizer(environment) + + # aliases for imports + self.import_aliases = {} + + # a registry for all blocks. Because blocks are moved out + # into the global python scope they are registered here + self.blocks = {} + + # the number of extends statements so far + self.extends_so_far = 0 + + # some templates have a rootlevel extends. In this case we + # can safely assume that we're a child template and do some + # more optimizations. + self.has_known_extends = False + + # the current line number + self.code_lineno = 1 + + # registry of all filters and tests (global, not block local) + self.tests = {} + self.filters = {} + + # the debug information + self.debug_info = [] + self._write_debug_info = None + + # the number of new lines before the next write() + self._new_lines = 0 + + # the line number of the last written statement + self._last_line = 0 + + # true if nothing was written so far. + self._first_write = True + + # used by the `temporary_identifier` method to get new + # unique, temporary identifier + self._last_identifier = 0 + + # the current indentation + self._indentation = 0 + + # Tracks toplevel assignments + self._assign_stack = [] + + # Tracks parameter definition blocks + self._param_def_block = [] + + # Tracks the current context. + self._context_reference_stack = ['context'] + + # -- Various compilation helpers + + def fail(self, msg, lineno): + """Fail with a :exc:`TemplateAssertionError`.""" + raise TemplateAssertionError(msg, lineno, self.name, self.filename) + + def temporary_identifier(self): + """Get a new unique identifier.""" + self._last_identifier += 1 + return 't_%d' % self._last_identifier + + def buffer(self, frame): + """Enable buffering for the frame from that point onwards.""" + frame.buffer = self.temporary_identifier() + self.writeline('%s = []' % frame.buffer) + + def return_buffer_contents(self, frame, force_unescaped=False): + """Return the buffer contents of the frame.""" + if not force_unescaped: + if frame.eval_ctx.volatile: + self.writeline('if context.eval_ctx.autoescape:') + self.indent() + self.writeline('return Markup(concat(%s))' % frame.buffer) + self.outdent() + self.writeline('else:') + self.indent() + self.writeline('return concat(%s)' % frame.buffer) + self.outdent() + return + elif frame.eval_ctx.autoescape: + self.writeline('return Markup(concat(%s))' % frame.buffer) + return + self.writeline('return concat(%s)' % frame.buffer) + + def indent(self): + """Indent by one.""" + self._indentation += 1 + + def outdent(self, step=1): + """Outdent by step.""" + self._indentation -= step + + def start_write(self, frame, node=None): + """Yield or write into the frame buffer.""" + if frame.buffer is None: + self.writeline('yield ', node) + else: + self.writeline('%s.append(' % frame.buffer, node) + + def end_write(self, frame): + """End the writing process started by `start_write`.""" + if frame.buffer is not None: + self.write(')') + + def simple_write(self, s, frame, node=None): + """Simple shortcut for start_write + write + end_write.""" + self.start_write(frame, node) + self.write(s) + self.end_write(frame) + + def blockvisit(self, nodes, frame): + """Visit a list of nodes as block in a frame. If the current frame + is no buffer a dummy ``if 0: yield None`` is written automatically. + """ + try: + self.writeline('pass') + for node in nodes: + self.visit(node, frame) + except CompilerExit: + pass + + def write(self, x): + """Write a string into the output stream.""" + if self._new_lines: + if not self._first_write: + self.stream.write('\n' * self._new_lines) + self.code_lineno += self._new_lines + if self._write_debug_info is not None: + self.debug_info.append((self._write_debug_info, + self.code_lineno)) + self._write_debug_info = None + self._first_write = False + self.stream.write(' ' * self._indentation) + self._new_lines = 0 + self.stream.write(x) + + def writeline(self, x, node=None, extra=0): + """Combination of newline and write.""" + self.newline(node, extra) + self.write(x) + + def newline(self, node=None, extra=0): + """Add one or more newlines before the next write.""" + self._new_lines = max(self._new_lines, 1 + extra) + if node is not None and node.lineno != self._last_line: + self._write_debug_info = node.lineno + self._last_line = node.lineno + + def signature(self, node, frame, extra_kwargs=None): + """Writes a function call to the stream for the current node. + A leading comma is added automatically. The extra keyword + arguments may not include python keywords otherwise a syntax + error could occour. The extra keyword arguments should be given + as python dict. + """ + # if any of the given keyword arguments is a python keyword + # we have to make sure that no invalid call is created. + kwarg_workaround = False + for kwarg in chain((x.key for x in node.kwargs), extra_kwargs or ()): + if is_python_keyword(kwarg): + kwarg_workaround = True + break + + for arg in node.args: + self.write(', ') + self.visit(arg, frame) + + if not kwarg_workaround: + for kwarg in node.kwargs: + self.write(', ') + self.visit(kwarg, frame) + if extra_kwargs is not None: + for key, value in iteritems(extra_kwargs): + self.write(', %s=%s' % (key, value)) + if node.dyn_args: + self.write(', *') + self.visit(node.dyn_args, frame) + + if kwarg_workaround: + if node.dyn_kwargs is not None: + self.write(', **dict({') + else: + self.write(', **{') + for kwarg in node.kwargs: + self.write('%r: ' % kwarg.key) + self.visit(kwarg.value, frame) + self.write(', ') + if extra_kwargs is not None: + for key, value in iteritems(extra_kwargs): + self.write('%r: %s, ' % (key, value)) + if node.dyn_kwargs is not None: + self.write('}, **') + self.visit(node.dyn_kwargs, frame) + self.write(')') + else: + self.write('}') + + elif node.dyn_kwargs is not None: + self.write(', **') + self.visit(node.dyn_kwargs, frame) + + def pull_dependencies(self, nodes): + """Pull all the dependencies.""" + visitor = DependencyFinderVisitor() + for node in nodes: + visitor.visit(node) + for dependency in 'filters', 'tests': + mapping = getattr(self, dependency) + for name in getattr(visitor, dependency): + if name not in mapping: + mapping[name] = self.temporary_identifier() + self.writeline('%s = environment.%s[%r]' % + (mapping[name], dependency, name)) + + def enter_frame(self, frame): + undefs = [] + for target, (action, param) in iteritems(frame.symbols.loads): + if action == VAR_LOAD_PARAMETER: + pass + elif action == VAR_LOAD_RESOLVE: + self.writeline('%s = %s(%r)' % + (target, self.get_resolve_func(), param)) + elif action == VAR_LOAD_ALIAS: + self.writeline('%s = %s' % (target, param)) + elif action == VAR_LOAD_UNDEFINED: + undefs.append(target) + else: + raise NotImplementedError('unknown load instruction') + if undefs: + self.writeline('%s = missing' % ' = '.join(undefs)) + + def leave_frame(self, frame, with_python_scope=False): + if not with_python_scope: + undefs = [] + for target, _ in iteritems(frame.symbols.loads): + undefs.append(target) + if undefs: + self.writeline('%s = missing' % ' = '.join(undefs)) + + def func(self, name): + if self.environment.is_async: + return 'async def %s' % name + return 'def %s' % name + + def macro_body(self, node, frame): + """Dump the function def of a macro or call block.""" + frame = frame.inner() + frame.symbols.analyze_node(node) + macro_ref = MacroRef(node) + + explicit_caller = None + skip_special_params = set() + args = [] + for idx, arg in enumerate(node.args): + if arg.name == 'caller': + explicit_caller = idx + if arg.name in ('kwargs', 'varargs'): + skip_special_params.add(arg.name) + args.append(frame.symbols.ref(arg.name)) + + undeclared = find_undeclared(node.body, ('caller', 'kwargs', 'varargs')) + + if 'caller' in undeclared: + # In older Jinja2 versions there was a bug that allowed caller + # to retain the special behavior even if it was mentioned in + # the argument list. However thankfully this was only really + # working if it was the last argument. So we are explicitly + # checking this now and error out if it is anywhere else in + # the argument list. + if explicit_caller is not None: + try: + node.defaults[explicit_caller - len(node.args)] + except IndexError: + self.fail('When defining macros or call blocks the ' + 'special "caller" argument must be omitted ' + 'or be given a default.', node.lineno) + else: + args.append(frame.symbols.declare_parameter('caller')) + macro_ref.accesses_caller = True + if 'kwargs' in undeclared and not 'kwargs' in skip_special_params: + args.append(frame.symbols.declare_parameter('kwargs')) + macro_ref.accesses_kwargs = True + if 'varargs' in undeclared and not 'varargs' in skip_special_params: + args.append(frame.symbols.declare_parameter('varargs')) + macro_ref.accesses_varargs = True + + # macros are delayed, they never require output checks + frame.require_output_check = False + frame.symbols.analyze_node(node) + self.writeline('%s(%s):' % (self.func('macro'), ', '.join(args)), node) + self.indent() + + self.buffer(frame) + self.enter_frame(frame) + + self.push_parameter_definitions(frame) + for idx, arg in enumerate(node.args): + ref = frame.symbols.ref(arg.name) + self.writeline('if %s is missing:' % ref) + self.indent() + try: + default = node.defaults[idx - len(node.args)] + except IndexError: + self.writeline('%s = undefined(%r, name=%r)' % ( + ref, + 'parameter %r was not provided' % arg.name, + arg.name)) + else: + self.writeline('%s = ' % ref) + self.visit(default, frame) + self.mark_parameter_stored(ref) + self.outdent() + self.pop_parameter_definitions() + + self.blockvisit(node.body, frame) + self.return_buffer_contents(frame, force_unescaped=True) + self.leave_frame(frame, with_python_scope=True) + self.outdent() + + return frame, macro_ref + + def macro_def(self, macro_ref, frame): + """Dump the macro definition for the def created by macro_body.""" + arg_tuple = ', '.join(repr(x.name) for x in macro_ref.node.args) + name = getattr(macro_ref.node, 'name', None) + if len(macro_ref.node.args) == 1: + arg_tuple += ',' + self.write('Macro(environment, macro, %r, (%s), %r, %r, %r, ' + 'context.eval_ctx.autoescape)' % + (name, arg_tuple, macro_ref.accesses_kwargs, + macro_ref.accesses_varargs, macro_ref.accesses_caller)) + + def position(self, node): + """Return a human readable position for the node.""" + rv = 'line %d' % node.lineno + if self.name is not None: + rv += ' in ' + repr(self.name) + return rv + + def dump_local_context(self, frame): + return '{%s}' % ', '.join( + '%r: %s' % (name, target) for name, target + in iteritems(frame.symbols.dump_stores())) + + def write_commons(self): + """Writes a common preamble that is used by root and block functions. + Primarily this sets up common local helpers and enforces a generator + through a dead branch. + """ + self.writeline('resolve = context.resolve_or_missing') + self.writeline('undefined = environment.undefined') + self.writeline('if 0: yield None') + + def push_parameter_definitions(self, frame): + """Pushes all parameter targets from the given frame into a local + stack that permits tracking of yet to be assigned parameters. In + particular this enables the optimization from `visit_Name` to skip + undefined expressions for parameters in macros as macros can reference + otherwise unbound parameters. + """ + self._param_def_block.append(frame.symbols.dump_param_targets()) + + def pop_parameter_definitions(self): + """Pops the current parameter definitions set.""" + self._param_def_block.pop() + + def mark_parameter_stored(self, target): + """Marks a parameter in the current parameter definitions as stored. + This will skip the enforced undefined checks. + """ + if self._param_def_block: + self._param_def_block[-1].discard(target) + + def push_context_reference(self, target): + self._context_reference_stack.append(target) + + def pop_context_reference(self): + self._context_reference_stack.pop() + + def get_context_ref(self): + return self._context_reference_stack[-1] + + def get_resolve_func(self): + target = self._context_reference_stack[-1] + if target == 'context': + return 'resolve' + return '%s.resolve' % target + + def derive_context(self, frame): + return '%s.derived(%s)' % ( + self.get_context_ref(), + self.dump_local_context(frame), + ) + + def parameter_is_undeclared(self, target): + """Checks if a given target is an undeclared parameter.""" + if not self._param_def_block: + return False + return target in self._param_def_block[-1] + + def push_assign_tracking(self): + """Pushes a new layer for assignment tracking.""" + self._assign_stack.append(set()) + + def pop_assign_tracking(self, frame): + """Pops the topmost level for assignment tracking and updates the + context variables if necessary. + """ + vars = self._assign_stack.pop() + if not frame.toplevel or not vars: + return + public_names = [x for x in vars if x[:1] != '_'] + if len(vars) == 1: + name = next(iter(vars)) + ref = frame.symbols.ref(name) + self.writeline('context.vars[%r] = %s' % (name, ref)) + else: + self.writeline('context.vars.update({') + for idx, name in enumerate(vars): + if idx: + self.write(', ') + ref = frame.symbols.ref(name) + self.write('%r: %s' % (name, ref)) + self.write('})') + if public_names: + if len(public_names) == 1: + self.writeline('context.exported_vars.add(%r)' % + public_names[0]) + else: + self.writeline('context.exported_vars.update((%s))' % + ', '.join(imap(repr, public_names))) + + # -- Statement Visitors + + def visit_Template(self, node, frame=None): + assert frame is None, 'no root frame allowed' + eval_ctx = EvalContext(self.environment, self.name) + + from jinja2.runtime import __all__ as exported + self.writeline('from __future__ import %s' % ', '.join(code_features)) + self.writeline('from jinja2.runtime import ' + ', '.join(exported)) + + if self.environment.is_async: + self.writeline('from jinja2.asyncsupport import auto_await, ' + 'auto_aiter, make_async_loop_context') + + # if we want a deferred initialization we cannot move the + # environment into a local name + envenv = not self.defer_init and ', environment=environment' or '' + + # do we have an extends tag at all? If not, we can save some + # overhead by just not processing any inheritance code. + have_extends = node.find(nodes.Extends) is not None + + # find all blocks + for block in node.find_all(nodes.Block): + if block.name in self.blocks: + self.fail('block %r defined twice' % block.name, block.lineno) + self.blocks[block.name] = block + + # find all imports and import them + for import_ in node.find_all(nodes.ImportedName): + if import_.importname not in self.import_aliases: + imp = import_.importname + self.import_aliases[imp] = alias = self.temporary_identifier() + if '.' in imp: + module, obj = imp.rsplit('.', 1) + self.writeline('from %s import %s as %s' % + (module, obj, alias)) + else: + self.writeline('import %s as %s' % (imp, alias)) + + # add the load name + self.writeline('name = %r' % self.name) + + # generate the root render function. + self.writeline('%s(context, missing=missing%s):' % + (self.func('root'), envenv), extra=1) + self.indent() + self.write_commons() + + # process the root + frame = Frame(eval_ctx) + if 'self' in find_undeclared(node.body, ('self',)): + ref = frame.symbols.declare_parameter('self') + self.writeline('%s = TemplateReference(context)' % ref) + frame.symbols.analyze_node(node) + frame.toplevel = frame.rootlevel = True + frame.require_output_check = have_extends and not self.has_known_extends + if have_extends: + self.writeline('parent_template = None') + self.enter_frame(frame) + self.pull_dependencies(node.body) + self.blockvisit(node.body, frame) + self.leave_frame(frame, with_python_scope=True) + self.outdent() + + # make sure that the parent root is called. + if have_extends: + if not self.has_known_extends: + self.indent() + self.writeline('if parent_template is not None:') + self.indent() + if supports_yield_from and not self.environment.is_async: + self.writeline('yield from parent_template.' + 'root_render_func(context)') + else: + self.writeline('%sfor event in parent_template.' + 'root_render_func(context):' % + (self.environment.is_async and 'async ' or '')) + self.indent() + self.writeline('yield event') + self.outdent() + self.outdent(1 + (not self.has_known_extends)) + + # at this point we now have the blocks collected and can visit them too. + for name, block in iteritems(self.blocks): + self.writeline('%s(context, missing=missing%s):' % + (self.func('block_' + name), envenv), + block, 1) + self.indent() + self.write_commons() + # It's important that we do not make this frame a child of the + # toplevel template. This would cause a variety of + # interesting issues with identifier tracking. + block_frame = Frame(eval_ctx) + undeclared = find_undeclared(block.body, ('self', 'super')) + if 'self' in undeclared: + ref = block_frame.symbols.declare_parameter('self') + self.writeline('%s = TemplateReference(context)' % ref) + if 'super' in undeclared: + ref = block_frame.symbols.declare_parameter('super') + self.writeline('%s = context.super(%r, ' + 'block_%s)' % (ref, name, name)) + block_frame.symbols.analyze_node(block) + block_frame.block = name + self.enter_frame(block_frame) + self.pull_dependencies(block.body) + self.blockvisit(block.body, block_frame) + self.leave_frame(block_frame, with_python_scope=True) + self.outdent() + + self.writeline('blocks = {%s}' % ', '.join('%r: block_%s' % (x, x) + for x in self.blocks), + extra=1) + + # add a function that returns the debug info + self.writeline('debug_info = %r' % '&'.join('%s=%s' % x for x + in self.debug_info)) + + def visit_Block(self, node, frame): + """Call a block and register it for the template.""" + level = 0 + if frame.toplevel: + # if we know that we are a child template, there is no need to + # check if we are one + if self.has_known_extends: + return + if self.extends_so_far > 0: + self.writeline('if parent_template is None:') + self.indent() + level += 1 + + if node.scoped: + context = self.derive_context(frame) + else: + context = self.get_context_ref() + + if supports_yield_from and not self.environment.is_async and \ + frame.buffer is None: + self.writeline('yield from context.blocks[%r][0](%s)' % ( + node.name, context), node) + else: + loop = self.environment.is_async and 'async for' or 'for' + self.writeline('%s event in context.blocks[%r][0](%s):' % ( + loop, node.name, context), node) + self.indent() + self.simple_write('event', frame) + self.outdent() + + self.outdent(level) + + def visit_Extends(self, node, frame): + """Calls the extender.""" + if not frame.toplevel: + self.fail('cannot use extend from a non top-level scope', + node.lineno) + + # if the number of extends statements in general is zero so + # far, we don't have to add a check if something extended + # the template before this one. + if self.extends_so_far > 0: + + # if we have a known extends we just add a template runtime + # error into the generated code. We could catch that at compile + # time too, but i welcome it not to confuse users by throwing the + # same error at different times just "because we can". + if not self.has_known_extends: + self.writeline('if parent_template is not None:') + self.indent() + self.writeline('raise TemplateRuntimeError(%r)' % + 'extended multiple times') + + # if we have a known extends already we don't need that code here + # as we know that the template execution will end here. + if self.has_known_extends: + raise CompilerExit() + else: + self.outdent() + + self.writeline('parent_template = environment.get_template(', node) + self.visit(node.template, frame) + self.write(', %r)' % self.name) + self.writeline('for name, parent_block in parent_template.' + 'blocks.%s():' % dict_item_iter) + self.indent() + self.writeline('context.blocks.setdefault(name, []).' + 'append(parent_block)') + self.outdent() + + # if this extends statement was in the root level we can take + # advantage of that information and simplify the generated code + # in the top level from this point onwards + if frame.rootlevel: + self.has_known_extends = True + + # and now we have one more + self.extends_so_far += 1 + + def visit_Include(self, node, frame): + """Handles includes.""" + if node.ignore_missing: + self.writeline('try:') + self.indent() + + func_name = 'get_or_select_template' + if isinstance(node.template, nodes.Const): + if isinstance(node.template.value, string_types): + func_name = 'get_template' + elif isinstance(node.template.value, (tuple, list)): + func_name = 'select_template' + elif isinstance(node.template, (nodes.Tuple, nodes.List)): + func_name = 'select_template' + + self.writeline('template = environment.%s(' % func_name, node) + self.visit(node.template, frame) + self.write(', %r)' % self.name) + if node.ignore_missing: + self.outdent() + self.writeline('except TemplateNotFound:') + self.indent() + self.writeline('pass') + self.outdent() + self.writeline('else:') + self.indent() + + skip_event_yield = False + if node.with_context: + loop = self.environment.is_async and 'async for' or 'for' + self.writeline('%s event in template.root_render_func(' + 'template.new_context(context.get_all(), True, ' + '%s)):' % (loop, self.dump_local_context(frame))) + elif self.environment.is_async: + self.writeline('for event in (await ' + 'template._get_default_module_async())' + '._body_stream:') + else: + if supports_yield_from: + self.writeline('yield from template._get_default_module()' + '._body_stream') + skip_event_yield = True + else: + self.writeline('for event in template._get_default_module()' + '._body_stream:') + + if not skip_event_yield: + self.indent() + self.simple_write('event', frame) + self.outdent() + + if node.ignore_missing: + self.outdent() + + def visit_Import(self, node, frame): + """Visit regular imports.""" + self.writeline('%s = ' % frame.symbols.ref(node.target), node) + if frame.toplevel: + self.write('context.vars[%r] = ' % node.target) + if self.environment.is_async: + self.write('await ') + self.write('environment.get_template(') + self.visit(node.template, frame) + self.write(', %r).' % self.name) + if node.with_context: + self.write('make_module%s(context.get_all(), True, %s)' + % (self.environment.is_async and '_async' or '', + self.dump_local_context(frame))) + elif self.environment.is_async: + self.write('_get_default_module_async()') + else: + self.write('_get_default_module()') + if frame.toplevel and not node.target.startswith('_'): + self.writeline('context.exported_vars.discard(%r)' % node.target) + + def visit_FromImport(self, node, frame): + """Visit named imports.""" + self.newline(node) + self.write('included_template = %senvironment.get_template(' + % (self.environment.is_async and 'await ' or '')) + self.visit(node.template, frame) + self.write(', %r).' % self.name) + if node.with_context: + self.write('make_module%s(context.get_all(), True, %s)' + % (self.environment.is_async and '_async' or '', + self.dump_local_context(frame))) + elif self.environment.is_async: + self.write('_get_default_module_async()') + else: + self.write('_get_default_module()') + + var_names = [] + discarded_names = [] + for name in node.names: + if isinstance(name, tuple): + name, alias = name + else: + alias = name + self.writeline('%s = getattr(included_template, ' + '%r, missing)' % (frame.symbols.ref(alias), name)) + self.writeline('if %s is missing:' % frame.symbols.ref(alias)) + self.indent() + self.writeline('%s = undefined(%r %% ' + 'included_template.__name__, ' + 'name=%r)' % + (frame.symbols.ref(alias), + 'the template %%r (imported on %s) does ' + 'not export the requested name %s' % ( + self.position(node), + repr(name) + ), name)) + self.outdent() + if frame.toplevel: + var_names.append(alias) + if not alias.startswith('_'): + discarded_names.append(alias) + + if var_names: + if len(var_names) == 1: + name = var_names[0] + self.writeline('context.vars[%r] = %s' % + (name, frame.symbols.ref(name))) + else: + self.writeline('context.vars.update({%s})' % ', '.join( + '%r: %s' % (name, frame.symbols.ref(name)) for name in var_names + )) + if discarded_names: + if len(discarded_names) == 1: + self.writeline('context.exported_vars.discard(%r)' % + discarded_names[0]) + else: + self.writeline('context.exported_vars.difference_' + 'update((%s))' % ', '.join(imap(repr, discarded_names))) + + def visit_For(self, node, frame): + loop_frame = frame.inner() + test_frame = frame.inner() + else_frame = frame.inner() + + # try to figure out if we have an extended loop. An extended loop + # is necessary if the loop is in recursive mode if the special loop + # variable is accessed in the body. + extended_loop = node.recursive or 'loop' in \ + find_undeclared(node.iter_child_nodes( + only=('body',)), ('loop',)) + + loop_ref = None + if extended_loop: + loop_ref = loop_frame.symbols.declare_parameter('loop') + + loop_frame.symbols.analyze_node(node, for_branch='body') + if node.else_: + else_frame.symbols.analyze_node(node, for_branch='else') + + if node.test: + loop_filter_func = self.temporary_identifier() + test_frame.symbols.analyze_node(node, for_branch='test') + self.writeline('%s(fiter):' % self.func(loop_filter_func), node.test) + self.indent() + self.enter_frame(test_frame) + self.writeline(self.environment.is_async and 'async for ' or 'for ') + self.visit(node.target, loop_frame) + self.write(' in ') + self.write(self.environment.is_async and 'auto_aiter(fiter)' or 'fiter') + self.write(':') + self.indent() + self.writeline('if ', node.test) + self.visit(node.test, test_frame) + self.write(':') + self.indent() + self.writeline('yield ') + self.visit(node.target, loop_frame) + self.outdent(3) + self.leave_frame(test_frame, with_python_scope=True) + + # if we don't have an recursive loop we have to find the shadowed + # variables at that point. Because loops can be nested but the loop + # variable is a special one we have to enforce aliasing for it. + if node.recursive: + self.writeline('%s(reciter, loop_render_func, depth=0):' % + self.func('loop'), node) + self.indent() + self.buffer(loop_frame) + + # Use the same buffer for the else frame + else_frame.buffer = loop_frame.buffer + + # make sure the loop variable is a special one and raise a template + # assertion error if a loop tries to write to loop + if extended_loop: + self.writeline('%s = missing' % loop_ref) + + for name in node.find_all(nodes.Name): + if name.ctx == 'store' and name.name == 'loop': + self.fail('Can\'t assign to special loop variable ' + 'in for-loop target', name.lineno) + + if node.else_: + iteration_indicator = self.temporary_identifier() + self.writeline('%s = 1' % iteration_indicator) + + self.writeline(self.environment.is_async and 'async for ' or 'for ', node) + self.visit(node.target, loop_frame) + if extended_loop: + if self.environment.is_async: + self.write(', %s in await make_async_loop_context(' % loop_ref) + else: + self.write(', %s in LoopContext(' % loop_ref) + else: + self.write(' in ') + + if node.test: + self.write('%s(' % loop_filter_func) + if node.recursive: + self.write('reciter') + else: + if self.environment.is_async and not extended_loop: + self.write('auto_aiter(') + self.visit(node.iter, frame) + if self.environment.is_async and not extended_loop: + self.write(')') + if node.test: + self.write(')') + + if node.recursive: + self.write(', undefined, loop_render_func, depth):') + else: + self.write(extended_loop and ', undefined):' or ':') + + self.indent() + self.enter_frame(loop_frame) + + self.blockvisit(node.body, loop_frame) + if node.else_: + self.writeline('%s = 0' % iteration_indicator) + self.outdent() + self.leave_frame(loop_frame, with_python_scope=node.recursive + and not node.else_) + + if node.else_: + self.writeline('if %s:' % iteration_indicator) + self.indent() + self.enter_frame(else_frame) + self.blockvisit(node.else_, else_frame) + self.leave_frame(else_frame) + self.outdent() + + # if the node was recursive we have to return the buffer contents + # and start the iteration code + if node.recursive: + self.return_buffer_contents(loop_frame) + self.outdent() + self.start_write(frame, node) + if self.environment.is_async: + self.write('await ') + self.write('loop(') + if self.environment.is_async: + self.write('auto_aiter(') + self.visit(node.iter, frame) + if self.environment.is_async: + self.write(')') + self.write(', loop)') + self.end_write(frame) + + def visit_If(self, node, frame): + if_frame = frame.soft() + self.writeline('if ', node) + self.visit(node.test, if_frame) + self.write(':') + self.indent() + self.blockvisit(node.body, if_frame) + self.outdent() + for elif_ in node.elif_: + self.writeline('elif ', elif_) + self.visit(elif_.test, if_frame) + self.write(':') + self.indent() + self.blockvisit(elif_.body, if_frame) + self.outdent() + if node.else_: + self.writeline('else:') + self.indent() + self.blockvisit(node.else_, if_frame) + self.outdent() + + def visit_Macro(self, node, frame): + macro_frame, macro_ref = self.macro_body(node, frame) + self.newline() + if frame.toplevel: + if not node.name.startswith('_'): + self.write('context.exported_vars.add(%r)' % node.name) + ref = frame.symbols.ref(node.name) + self.writeline('context.vars[%r] = ' % node.name) + self.write('%s = ' % frame.symbols.ref(node.name)) + self.macro_def(macro_ref, macro_frame) + + def visit_CallBlock(self, node, frame): + call_frame, macro_ref = self.macro_body(node, frame) + self.writeline('caller = ') + self.macro_def(macro_ref, call_frame) + self.start_write(frame, node) + self.visit_Call(node.call, frame, forward_caller=True) + self.end_write(frame) + + def visit_FilterBlock(self, node, frame): + filter_frame = frame.inner() + filter_frame.symbols.analyze_node(node) + self.enter_frame(filter_frame) + self.buffer(filter_frame) + self.blockvisit(node.body, filter_frame) + self.start_write(frame, node) + self.visit_Filter(node.filter, filter_frame) + self.end_write(frame) + self.leave_frame(filter_frame) + + def visit_With(self, node, frame): + with_frame = frame.inner() + with_frame.symbols.analyze_node(node) + self.enter_frame(with_frame) + for idx, (target, expr) in enumerate(izip(node.targets, node.values)): + self.newline() + self.visit(target, with_frame) + self.write(' = ') + self.visit(expr, frame) + self.blockvisit(node.body, with_frame) + self.leave_frame(with_frame) + + def visit_ExprStmt(self, node, frame): + self.newline(node) + self.visit(node.node, frame) + + def visit_Output(self, node, frame): + # if we have a known extends statement, we don't output anything + # if we are in a require_output_check section + if self.has_known_extends and frame.require_output_check: + return + + allow_constant_finalize = True + if self.environment.finalize: + func = self.environment.finalize + if getattr(func, 'contextfunction', False) or \ + getattr(func, 'evalcontextfunction', False): + allow_constant_finalize = False + elif getattr(func, 'environmentfunction', False): + finalize = lambda x: text_type( + self.environment.finalize(self.environment, x)) + else: + finalize = lambda x: text_type(self.environment.finalize(x)) + else: + finalize = text_type + + # if we are inside a frame that requires output checking, we do so + outdent_later = False + if frame.require_output_check: + self.writeline('if parent_template is None:') + self.indent() + outdent_later = True + + # try to evaluate as many chunks as possible into a static + # string at compile time. + body = [] + for child in node.nodes: + try: + if not allow_constant_finalize: + raise nodes.Impossible() + const = child.as_const(frame.eval_ctx) + except nodes.Impossible: + body.append(child) + continue + # the frame can't be volatile here, becaus otherwise the + # as_const() function would raise an Impossible exception + # at that point. + try: + if frame.eval_ctx.autoescape: + if hasattr(const, '__html__'): + const = const.__html__() + else: + const = escape(const) + const = finalize(const) + except Exception: + # if something goes wrong here we evaluate the node + # at runtime for easier debugging + body.append(child) + continue + if body and isinstance(body[-1], list): + body[-1].append(const) + else: + body.append([const]) + + # if we have less than 3 nodes or a buffer we yield or extend/append + if len(body) < 3 or frame.buffer is not None: + if frame.buffer is not None: + # for one item we append, for more we extend + if len(body) == 1: + self.writeline('%s.append(' % frame.buffer) + else: + self.writeline('%s.extend((' % frame.buffer) + self.indent() + for item in body: + if isinstance(item, list): + val = repr(concat(item)) + if frame.buffer is None: + self.writeline('yield ' + val) + else: + self.writeline(val + ',') + else: + if frame.buffer is None: + self.writeline('yield ', item) + else: + self.newline(item) + close = 1 + if frame.eval_ctx.volatile: + self.write('(escape if context.eval_ctx.autoescape' + ' else to_string)(') + elif frame.eval_ctx.autoescape: + self.write('escape(') + else: + self.write('to_string(') + if self.environment.finalize is not None: + self.write('environment.finalize(') + if getattr(self.environment.finalize, + "contextfunction", False): + self.write('context, ') + close += 1 + self.visit(item, frame) + self.write(')' * close) + if frame.buffer is not None: + self.write(',') + if frame.buffer is not None: + # close the open parentheses + self.outdent() + self.writeline(len(body) == 1 and ')' or '))') + + # otherwise we create a format string as this is faster in that case + else: + format = [] + arguments = [] + for item in body: + if isinstance(item, list): + format.append(concat(item).replace('%', '%%')) + else: + format.append('%s') + arguments.append(item) + self.writeline('yield ') + self.write(repr(concat(format)) + ' % (') + self.indent() + for argument in arguments: + self.newline(argument) + close = 0 + if frame.eval_ctx.volatile: + self.write('(escape if context.eval_ctx.autoescape else' + ' to_string)(') + close += 1 + elif frame.eval_ctx.autoescape: + self.write('escape(') + close += 1 + if self.environment.finalize is not None: + self.write('environment.finalize(') + if getattr(self.environment.finalize, + 'contextfunction', False): + self.write('context, ') + elif getattr(self.environment.finalize, + 'evalcontextfunction', False): + self.write('context.eval_ctx, ') + elif getattr(self.environment.finalize, + 'environmentfunction', False): + self.write('environment, ') + close += 1 + self.visit(argument, frame) + self.write(')' * close + ', ') + self.outdent() + self.writeline(')') + + if outdent_later: + self.outdent() + + def visit_Assign(self, node, frame): + self.push_assign_tracking() + self.newline(node) + self.visit(node.target, frame) + self.write(' = ') + self.visit(node.node, frame) + self.pop_assign_tracking(frame) + + def visit_AssignBlock(self, node, frame): + self.push_assign_tracking() + block_frame = frame.inner() + # This is a special case. Since a set block always captures we + # will disable output checks. This way one can use set blocks + # toplevel even in extended templates. + block_frame.require_output_check = False + block_frame.symbols.analyze_node(node) + self.enter_frame(block_frame) + self.buffer(block_frame) + self.blockvisit(node.body, block_frame) + self.newline(node) + self.visit(node.target, frame) + self.write(' = (Markup if context.eval_ctx.autoescape ' + 'else identity)(') + if node.filter is not None: + self.visit_Filter(node.filter, block_frame) + else: + self.write('concat(%s)' % block_frame.buffer) + self.write(')') + self.pop_assign_tracking(frame) + self.leave_frame(block_frame) + + # -- Expression Visitors + + def visit_Name(self, node, frame): + if node.ctx == 'store' and frame.toplevel: + if self._assign_stack: + self._assign_stack[-1].add(node.name) + ref = frame.symbols.ref(node.name) + + # If we are looking up a variable we might have to deal with the + # case where it's undefined. We can skip that case if the load + # instruction indicates a parameter which are always defined. + if node.ctx == 'load': + load = frame.symbols.find_load(ref) + if not (load is not None and load[0] == VAR_LOAD_PARAMETER and \ + not self.parameter_is_undeclared(ref)): + self.write('(undefined(name=%r) if %s is missing else %s)' % + (node.name, ref, ref)) + return + + self.write(ref) + + def visit_NSRef(self, node, frame): + # NSRefs can only be used to store values; since they use the normal + # `foo.bar` notation they will be parsed as a normal attribute access + # when used anywhere but in a `set` context + ref = frame.symbols.ref(node.name) + self.writeline('if not isinstance(%s, Namespace):' % ref) + self.indent() + self.writeline('raise TemplateRuntimeError(%r)' % + 'cannot assign attribute on non-namespace object') + self.outdent() + self.writeline('%s[%r]' % (ref, node.attr)) + + def visit_Const(self, node, frame): + val = node.as_const(frame.eval_ctx) + if isinstance(val, float): + self.write(str(val)) + else: + self.write(repr(val)) + + def visit_TemplateData(self, node, frame): + try: + self.write(repr(node.as_const(frame.eval_ctx))) + except nodes.Impossible: + self.write('(Markup if context.eval_ctx.autoescape else identity)(%r)' + % node.data) + + def visit_Tuple(self, node, frame): + self.write('(') + idx = -1 + for idx, item in enumerate(node.items): + if idx: + self.write(', ') + self.visit(item, frame) + self.write(idx == 0 and ',)' or ')') + + def visit_List(self, node, frame): + self.write('[') + for idx, item in enumerate(node.items): + if idx: + self.write(', ') + self.visit(item, frame) + self.write(']') + + def visit_Dict(self, node, frame): + self.write('{') + for idx, item in enumerate(node.items): + if idx: + self.write(', ') + self.visit(item.key, frame) + self.write(': ') + self.visit(item.value, frame) + self.write('}') + + def binop(operator, interceptable=True): + @optimizeconst + def visitor(self, node, frame): + if self.environment.sandboxed and \ + operator in self.environment.intercepted_binops: + self.write('environment.call_binop(context, %r, ' % operator) + self.visit(node.left, frame) + self.write(', ') + self.visit(node.right, frame) + else: + self.write('(') + self.visit(node.left, frame) + self.write(' %s ' % operator) + self.visit(node.right, frame) + self.write(')') + return visitor + + def uaop(operator, interceptable=True): + @optimizeconst + def visitor(self, node, frame): + if self.environment.sandboxed and \ + operator in self.environment.intercepted_unops: + self.write('environment.call_unop(context, %r, ' % operator) + self.visit(node.node, frame) + else: + self.write('(' + operator) + self.visit(node.node, frame) + self.write(')') + return visitor + + visit_Add = binop('+') + visit_Sub = binop('-') + visit_Mul = binop('*') + visit_Div = binop('/') + visit_FloorDiv = binop('//') + visit_Pow = binop('**') + visit_Mod = binop('%') + visit_And = binop('and', interceptable=False) + visit_Or = binop('or', interceptable=False) + visit_Pos = uaop('+') + visit_Neg = uaop('-') + visit_Not = uaop('not ', interceptable=False) + del binop, uaop + + @optimizeconst + def visit_Concat(self, node, frame): + if frame.eval_ctx.volatile: + func_name = '(context.eval_ctx.volatile and' \ + ' markup_join or unicode_join)' + elif frame.eval_ctx.autoescape: + func_name = 'markup_join' + else: + func_name = 'unicode_join' + self.write('%s((' % func_name) + for arg in node.nodes: + self.visit(arg, frame) + self.write(', ') + self.write('))') + + @optimizeconst + def visit_Compare(self, node, frame): + self.visit(node.expr, frame) + for op in node.ops: + self.visit(op, frame) + + def visit_Operand(self, node, frame): + self.write(' %s ' % operators[node.op]) + self.visit(node.expr, frame) + + @optimizeconst + def visit_Getattr(self, node, frame): + self.write('environment.getattr(') + self.visit(node.node, frame) + self.write(', %r)' % node.attr) + + @optimizeconst + def visit_Getitem(self, node, frame): + # slices bypass the environment getitem method. + if isinstance(node.arg, nodes.Slice): + self.visit(node.node, frame) + self.write('[') + self.visit(node.arg, frame) + self.write(']') + else: + self.write('environment.getitem(') + self.visit(node.node, frame) + self.write(', ') + self.visit(node.arg, frame) + self.write(')') + + def visit_Slice(self, node, frame): + if node.start is not None: + self.visit(node.start, frame) + self.write(':') + if node.stop is not None: + self.visit(node.stop, frame) + if node.step is not None: + self.write(':') + self.visit(node.step, frame) + + @optimizeconst + def visit_Filter(self, node, frame): + if self.environment.is_async: + self.write('await auto_await(') + self.write(self.filters[node.name] + '(') + func = self.environment.filters.get(node.name) + if func is None: + self.fail('no filter named %r' % node.name, node.lineno) + if getattr(func, 'contextfilter', False): + self.write('context, ') + elif getattr(func, 'evalcontextfilter', False): + self.write('context.eval_ctx, ') + elif getattr(func, 'environmentfilter', False): + self.write('environment, ') + + # if the filter node is None we are inside a filter block + # and want to write to the current buffer + if node.node is not None: + self.visit(node.node, frame) + elif frame.eval_ctx.volatile: + self.write('(context.eval_ctx.autoescape and' + ' Markup(concat(%s)) or concat(%s))' % + (frame.buffer, frame.buffer)) + elif frame.eval_ctx.autoescape: + self.write('Markup(concat(%s))' % frame.buffer) + else: + self.write('concat(%s)' % frame.buffer) + self.signature(node, frame) + self.write(')') + if self.environment.is_async: + self.write(')') + + @optimizeconst + def visit_Test(self, node, frame): + self.write(self.tests[node.name] + '(') + if node.name not in self.environment.tests: + self.fail('no test named %r' % node.name, node.lineno) + self.visit(node.node, frame) + self.signature(node, frame) + self.write(')') + + @optimizeconst + def visit_CondExpr(self, node, frame): + def write_expr2(): + if node.expr2 is not None: + return self.visit(node.expr2, frame) + self.write('undefined(%r)' % ('the inline if-' + 'expression on %s evaluated to false and ' + 'no else section was defined.' % self.position(node))) + + self.write('(') + self.visit(node.expr1, frame) + self.write(' if ') + self.visit(node.test, frame) + self.write(' else ') + write_expr2() + self.write(')') + + @optimizeconst + def visit_Call(self, node, frame, forward_caller=False): + if self.environment.is_async: + self.write('await auto_await(') + if self.environment.sandboxed: + self.write('environment.call(context, ') + else: + self.write('context.call(') + self.visit(node.node, frame) + extra_kwargs = forward_caller and {'caller': 'caller'} or None + self.signature(node, frame, extra_kwargs) + self.write(')') + if self.environment.is_async: + self.write(')') + + def visit_Keyword(self, node, frame): + self.write(node.key + '=') + self.visit(node.value, frame) + + # -- Unused nodes for extensions + + def visit_MarkSafe(self, node, frame): + self.write('Markup(') + self.visit(node.expr, frame) + self.write(')') + + def visit_MarkSafeIfAutoescape(self, node, frame): + self.write('(context.eval_ctx.autoescape and Markup or identity)(') + self.visit(node.expr, frame) + self.write(')') + + def visit_EnvironmentAttribute(self, node, frame): + self.write('environment.' + node.name) + + def visit_ExtensionAttribute(self, node, frame): + self.write('environment.extensions[%r].%s' % (node.identifier, node.name)) + + def visit_ImportedName(self, node, frame): + self.write(self.import_aliases[node.importname]) + + def visit_InternalName(self, node, frame): + self.write(node.name) + + def visit_ContextReference(self, node, frame): + self.write('context') + + def visit_Continue(self, node, frame): + self.writeline('continue', node) + + def visit_Break(self, node, frame): + self.writeline('break', node) + + def visit_Scope(self, node, frame): + scope_frame = frame.inner() + scope_frame.symbols.analyze_node(node) + self.enter_frame(scope_frame) + self.blockvisit(node.body, scope_frame) + self.leave_frame(scope_frame) + + def visit_OverlayScope(self, node, frame): + ctx = self.temporary_identifier() + self.writeline('%s = %s' % (ctx, self.derive_context(frame))) + self.writeline('%s.vars = ' % ctx) + self.visit(node.context, frame) + self.push_context_reference(ctx) + + scope_frame = frame.inner(isolated=True) + scope_frame.symbols.analyze_node(node) + self.enter_frame(scope_frame) + self.blockvisit(node.body, scope_frame) + self.leave_frame(scope_frame) + self.pop_context_reference() + + def visit_EvalContextModifier(self, node, frame): + for keyword in node.options: + self.writeline('context.eval_ctx.%s = ' % keyword.key) + self.visit(keyword.value, frame) + try: + val = keyword.value.as_const(frame.eval_ctx) + except nodes.Impossible: + frame.eval_ctx.volatile = True + else: + setattr(frame.eval_ctx, keyword.key, val) + + def visit_ScopedEvalContextModifier(self, node, frame): + old_ctx_name = self.temporary_identifier() + saved_ctx = frame.eval_ctx.save() + self.writeline('%s = context.eval_ctx.save()' % old_ctx_name) + self.visit_EvalContextModifier(node, frame) + for child in node.body: + self.visit(child, frame) + frame.eval_ctx.revert(saved_ctx) + self.writeline('context.eval_ctx.revert(%s)' % old_ctx_name) diff --git a/tools/jinja2/constants.py b/tools/jinja2/constants.py new file mode 100644 index 00000000000000..11efd1ed15832d --- /dev/null +++ b/tools/jinja2/constants.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +""" + jinja.constants + ~~~~~~~~~~~~~~~ + + Various constants. + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" + + +#: list of lorem ipsum words used by the lipsum() helper function +LOREM_IPSUM_WORDS = u'''\ +a ac accumsan ad adipiscing aenean aliquam aliquet amet ante aptent arcu at +auctor augue bibendum blandit class commodo condimentum congue consectetuer +consequat conubia convallis cras cubilia cum curabitur curae cursus dapibus +diam dictum dictumst dignissim dis dolor donec dui duis egestas eget eleifend +elementum elit enim erat eros est et etiam eu euismod facilisi facilisis fames +faucibus felis fermentum feugiat fringilla fusce gravida habitant habitasse hac +hendrerit hymenaeos iaculis id imperdiet in inceptos integer interdum ipsum +justo lacinia lacus laoreet lectus leo libero ligula litora lobortis lorem +luctus maecenas magna magnis malesuada massa mattis mauris metus mi molestie +mollis montes morbi mus nam nascetur natoque nec neque netus nibh nisi nisl non +nonummy nostra nulla nullam nunc odio orci ornare parturient pede pellentesque +penatibus per pharetra phasellus placerat platea porta porttitor posuere +potenti praesent pretium primis proin pulvinar purus quam quis quisque rhoncus +ridiculus risus rutrum sagittis sapien scelerisque sed sem semper senectus sit +sociis sociosqu sodales sollicitudin suscipit suspendisse taciti tellus tempor +tempus tincidunt torquent tortor tristique turpis ullamcorper ultrices +ultricies urna ut varius vehicula vel velit venenatis vestibulum vitae vivamus +viverra volutpat vulputate''' diff --git a/tools/jinja2/debug.py b/tools/jinja2/debug.py new file mode 100644 index 00000000000000..b61139f0cde17f --- /dev/null +++ b/tools/jinja2/debug.py @@ -0,0 +1,372 @@ +# -*- coding: utf-8 -*- +""" + jinja2.debug + ~~~~~~~~~~~~ + + Implements the debug interface for Jinja. This module does some pretty + ugly stuff with the Python traceback system in order to achieve tracebacks + with correct line numbers, locals and contents. + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import sys +import traceback +from types import TracebackType, CodeType +from jinja2.utils import missing, internal_code +from jinja2.exceptions import TemplateSyntaxError +from jinja2._compat import iteritems, reraise, PY2 + +# on pypy we can take advantage of transparent proxies +try: + from __pypy__ import tproxy +except ImportError: + tproxy = None + + +# how does the raise helper look like? +try: + exec("raise TypeError, 'foo'") +except SyntaxError: + raise_helper = 'raise __jinja_exception__[1]' +except TypeError: + raise_helper = 'raise __jinja_exception__[0], __jinja_exception__[1]' + + +class TracebackFrameProxy(object): + """Proxies a traceback frame.""" + + def __init__(self, tb): + self.tb = tb + self._tb_next = None + + @property + def tb_next(self): + return self._tb_next + + def set_next(self, next): + if tb_set_next is not None: + try: + tb_set_next(self.tb, next and next.tb or None) + except Exception: + # this function can fail due to all the hackery it does + # on various python implementations. We just catch errors + # down and ignore them if necessary. + pass + self._tb_next = next + + @property + def is_jinja_frame(self): + return '__jinja_template__' in self.tb.tb_frame.f_globals + + def __getattr__(self, name): + return getattr(self.tb, name) + + +def make_frame_proxy(frame): + proxy = TracebackFrameProxy(frame) + if tproxy is None: + return proxy + def operation_handler(operation, *args, **kwargs): + if operation in ('__getattribute__', '__getattr__'): + return getattr(proxy, args[0]) + elif operation == '__setattr__': + proxy.__setattr__(*args, **kwargs) + else: + return getattr(proxy, operation)(*args, **kwargs) + return tproxy(TracebackType, operation_handler) + + +class ProcessedTraceback(object): + """Holds a Jinja preprocessed traceback for printing or reraising.""" + + def __init__(self, exc_type, exc_value, frames): + assert frames, 'no frames for this traceback?' + self.exc_type = exc_type + self.exc_value = exc_value + self.frames = frames + + # newly concatenate the frames (which are proxies) + prev_tb = None + for tb in self.frames: + if prev_tb is not None: + prev_tb.set_next(tb) + prev_tb = tb + prev_tb.set_next(None) + + def render_as_text(self, limit=None): + """Return a string with the traceback.""" + lines = traceback.format_exception(self.exc_type, self.exc_value, + self.frames[0], limit=limit) + return ''.join(lines).rstrip() + + def render_as_html(self, full=False): + """Return a unicode string with the traceback as rendered HTML.""" + from jinja2.debugrenderer import render_traceback + return u'%s\n\n' % ( + render_traceback(self, full=full), + self.render_as_text().decode('utf-8', 'replace') + ) + + @property + def is_template_syntax_error(self): + """`True` if this is a template syntax error.""" + return isinstance(self.exc_value, TemplateSyntaxError) + + @property + def exc_info(self): + """Exception info tuple with a proxy around the frame objects.""" + return self.exc_type, self.exc_value, self.frames[0] + + @property + def standard_exc_info(self): + """Standard python exc_info for re-raising""" + tb = self.frames[0] + # the frame will be an actual traceback (or transparent proxy) if + # we are on pypy or a python implementation with support for tproxy + if type(tb) is not TracebackType: + tb = tb.tb + return self.exc_type, self.exc_value, tb + + +def make_traceback(exc_info, source_hint=None): + """Creates a processed traceback object from the exc_info.""" + exc_type, exc_value, tb = exc_info + if isinstance(exc_value, TemplateSyntaxError): + exc_info = translate_syntax_error(exc_value, source_hint) + initial_skip = 0 + else: + initial_skip = 1 + return translate_exception(exc_info, initial_skip) + + +def translate_syntax_error(error, source=None): + """Rewrites a syntax error to please traceback systems.""" + error.source = source + error.translated = True + exc_info = (error.__class__, error, None) + filename = error.filename + if filename is None: + filename = '' + return fake_exc_info(exc_info, filename, error.lineno) + + +def translate_exception(exc_info, initial_skip=0): + """If passed an exc_info it will automatically rewrite the exceptions + all the way down to the correct line numbers and frames. + """ + tb = exc_info[2] + frames = [] + + # skip some internal frames if wanted + for x in range(initial_skip): + if tb is not None: + tb = tb.tb_next + initial_tb = tb + + while tb is not None: + # skip frames decorated with @internalcode. These are internal + # calls we can't avoid and that are useless in template debugging + # output. + if tb.tb_frame.f_code in internal_code: + tb = tb.tb_next + continue + + # save a reference to the next frame if we override the current + # one with a faked one. + next = tb.tb_next + + # fake template exceptions + template = tb.tb_frame.f_globals.get('__jinja_template__') + if template is not None: + lineno = template.get_corresponding_lineno(tb.tb_lineno) + tb = fake_exc_info(exc_info[:2] + (tb,), template.filename, + lineno)[2] + + frames.append(make_frame_proxy(tb)) + tb = next + + # if we don't have any exceptions in the frames left, we have to + # reraise it unchanged. + # XXX: can we backup here? when could this happen? + if not frames: + reraise(exc_info[0], exc_info[1], exc_info[2]) + + return ProcessedTraceback(exc_info[0], exc_info[1], frames) + + +def get_jinja_locals(real_locals): + ctx = real_locals.get('context') + if ctx: + locals = ctx.get_all().copy() + else: + locals = {} + + local_overrides = {} + + for name, value in iteritems(real_locals): + if not name.startswith('l_') or value is missing: + continue + try: + _, depth, name = name.split('_', 2) + depth = int(depth) + except ValueError: + continue + cur_depth = local_overrides.get(name, (-1,))[0] + if cur_depth < depth: + local_overrides[name] = (depth, value) + + for name, (_, value) in iteritems(local_overrides): + if value is missing: + locals.pop(name, None) + else: + locals[name] = value + + return locals + + +def fake_exc_info(exc_info, filename, lineno): + """Helper for `translate_exception`.""" + exc_type, exc_value, tb = exc_info + + # figure the real context out + if tb is not None: + locals = get_jinja_locals(tb.tb_frame.f_locals) + + # if there is a local called __jinja_exception__, we get + # rid of it to not break the debug functionality. + locals.pop('__jinja_exception__', None) + else: + locals = {} + + # assamble fake globals we need + globals = { + '__name__': filename, + '__file__': filename, + '__jinja_exception__': exc_info[:2], + + # we don't want to keep the reference to the template around + # to not cause circular dependencies, but we mark it as Jinja + # frame for the ProcessedTraceback + '__jinja_template__': None + } + + # and fake the exception + code = compile('\n' * (lineno - 1) + raise_helper, filename, 'exec') + + # if it's possible, change the name of the code. This won't work + # on some python environments such as google appengine + try: + if tb is None: + location = 'template' + else: + function = tb.tb_frame.f_code.co_name + if function == 'root': + location = 'top-level template code' + elif function.startswith('block_'): + location = 'block "%s"' % function[6:] + else: + location = 'template' + + if PY2: + code = CodeType(0, code.co_nlocals, code.co_stacksize, + code.co_flags, code.co_code, code.co_consts, + code.co_names, code.co_varnames, filename, + location, code.co_firstlineno, + code.co_lnotab, (), ()) + else: + code = CodeType(0, code.co_kwonlyargcount, + code.co_nlocals, code.co_stacksize, + code.co_flags, code.co_code, code.co_consts, + code.co_names, code.co_varnames, filename, + location, code.co_firstlineno, + code.co_lnotab, (), ()) + except Exception as e: + pass + + # execute the code and catch the new traceback + try: + exec(code, globals, locals) + except: + exc_info = sys.exc_info() + new_tb = exc_info[2].tb_next + + # return without this frame + return exc_info[:2] + (new_tb,) + + +def _init_ugly_crap(): + """This function implements a few ugly things so that we can patch the + traceback objects. The function returned allows resetting `tb_next` on + any python traceback object. Do not attempt to use this on non cpython + interpreters + """ + import ctypes + from types import TracebackType + + if PY2: + # figure out size of _Py_ssize_t for Python 2: + if hasattr(ctypes.pythonapi, 'Py_InitModule4_64'): + _Py_ssize_t = ctypes.c_int64 + else: + _Py_ssize_t = ctypes.c_int + else: + # platform ssize_t on Python 3 + _Py_ssize_t = ctypes.c_ssize_t + + # regular python + class _PyObject(ctypes.Structure): + pass + _PyObject._fields_ = [ + ('ob_refcnt', _Py_ssize_t), + ('ob_type', ctypes.POINTER(_PyObject)) + ] + + # python with trace + if hasattr(sys, 'getobjects'): + class _PyObject(ctypes.Structure): + pass + _PyObject._fields_ = [ + ('_ob_next', ctypes.POINTER(_PyObject)), + ('_ob_prev', ctypes.POINTER(_PyObject)), + ('ob_refcnt', _Py_ssize_t), + ('ob_type', ctypes.POINTER(_PyObject)) + ] + + class _Traceback(_PyObject): + pass + _Traceback._fields_ = [ + ('tb_next', ctypes.POINTER(_Traceback)), + ('tb_frame', ctypes.POINTER(_PyObject)), + ('tb_lasti', ctypes.c_int), + ('tb_lineno', ctypes.c_int) + ] + + def tb_set_next(tb, next): + """Set the tb_next attribute of a traceback object.""" + if not (isinstance(tb, TracebackType) and + (next is None or isinstance(next, TracebackType))): + raise TypeError('tb_set_next arguments must be traceback objects') + obj = _Traceback.from_address(id(tb)) + if tb.tb_next is not None: + old = _Traceback.from_address(id(tb.tb_next)) + old.ob_refcnt -= 1 + if next is None: + obj.tb_next = ctypes.POINTER(_Traceback)() + else: + next = _Traceback.from_address(id(next)) + next.ob_refcnt += 1 + obj.tb_next = ctypes.pointer(next) + + return tb_set_next + + +# try to get a tb_set_next implementation if we don't have transparent +# proxies. +tb_set_next = None +if tproxy is None: + try: + tb_set_next = _init_ugly_crap() + except: + pass + del _init_ugly_crap diff --git a/tools/jinja2/defaults.py b/tools/jinja2/defaults.py new file mode 100644 index 00000000000000..7c93dec0aeb202 --- /dev/null +++ b/tools/jinja2/defaults.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +""" + jinja2.defaults + ~~~~~~~~~~~~~~~ + + Jinja default filters and tags. + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +from jinja2._compat import range_type +from jinja2.utils import generate_lorem_ipsum, Cycler, Joiner, Namespace + + +# defaults for the parser / lexer +BLOCK_START_STRING = '{%' +BLOCK_END_STRING = '%}' +VARIABLE_START_STRING = '{{' +VARIABLE_END_STRING = '}}' +COMMENT_START_STRING = '{#' +COMMENT_END_STRING = '#}' +LINE_STATEMENT_PREFIX = None +LINE_COMMENT_PREFIX = None +TRIM_BLOCKS = False +LSTRIP_BLOCKS = False +NEWLINE_SEQUENCE = '\n' +KEEP_TRAILING_NEWLINE = False + + +# default filters, tests and namespace +from jinja2.filters import FILTERS as DEFAULT_FILTERS +from jinja2.tests import TESTS as DEFAULT_TESTS +DEFAULT_NAMESPACE = { + 'range': range_type, + 'dict': dict, + 'lipsum': generate_lorem_ipsum, + 'cycler': Cycler, + 'joiner': Joiner, + 'namespace': Namespace +} + + +# default policies +DEFAULT_POLICIES = { + 'compiler.ascii_str': True, + 'urlize.rel': 'noopener', + 'urlize.target': None, + 'truncate.leeway': 5, + 'json.dumps_function': None, + 'json.dumps_kwargs': {'sort_keys': True}, + 'ext.i18n.trimmed': False, +} + + +# export all constants +__all__ = tuple(x for x in locals().keys() if x.isupper()) diff --git a/tools/jinja2/environment.py b/tools/jinja2/environment.py new file mode 100644 index 00000000000000..549d9afab456b4 --- /dev/null +++ b/tools/jinja2/environment.py @@ -0,0 +1,1276 @@ +# -*- coding: utf-8 -*- +""" + jinja2.environment + ~~~~~~~~~~~~~~~~~~ + + Provides a class that holds runtime and parsing time options. + + :copyright: (c) 2017 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import os +import sys +import weakref +from functools import reduce, partial +from jinja2 import nodes +from jinja2.defaults import BLOCK_START_STRING, \ + BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \ + COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \ + LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \ + DEFAULT_FILTERS, DEFAULT_TESTS, DEFAULT_NAMESPACE, \ + DEFAULT_POLICIES, KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS +from jinja2.lexer import get_lexer, TokenStream +from jinja2.parser import Parser +from jinja2.nodes import EvalContext +from jinja2.compiler import generate, CodeGenerator +from jinja2.runtime import Undefined, new_context, Context +from jinja2.exceptions import TemplateSyntaxError, TemplateNotFound, \ + TemplatesNotFound, TemplateRuntimeError +from jinja2.utils import import_string, LRUCache, Markup, missing, \ + concat, consume, internalcode, have_async_gen +from jinja2._compat import imap, ifilter, string_types, iteritems, \ + text_type, reraise, implements_iterator, implements_to_string, \ + encode_filename, PY2, PYPY + + +# for direct template usage we have up to ten living environments +_spontaneous_environments = LRUCache(10) + +# the function to create jinja traceback objects. This is dynamically +# imported on the first exception in the exception handler. +_make_traceback = None + + +def get_spontaneous_environment(*args): + """Return a new spontaneous environment. A spontaneous environment is an + unnamed and unaccessible (in theory) environment that is used for + templates generated from a string and not from the file system. + """ + try: + env = _spontaneous_environments.get(args) + except TypeError: + return Environment(*args) + if env is not None: + return env + _spontaneous_environments[args] = env = Environment(*args) + env.shared = True + return env + + +def create_cache(size): + """Return the cache class for the given size.""" + if size == 0: + return None + if size < 0: + return {} + return LRUCache(size) + + +def copy_cache(cache): + """Create an empty copy of the given cache.""" + if cache is None: + return None + elif type(cache) is dict: + return {} + return LRUCache(cache.capacity) + + +def load_extensions(environment, extensions): + """Load the extensions from the list and bind it to the environment. + Returns a dict of instantiated environments. + """ + result = {} + for extension in extensions: + if isinstance(extension, string_types): + extension = import_string(extension) + result[extension.identifier] = extension(environment) + return result + + +def fail_for_missing_callable(string, name): + msg = string % name + if isinstance(name, Undefined): + try: + name._fail_with_undefined_error() + except Exception as e: + msg = '%s (%s; did you forget to quote the callable name?)' % (msg, e) + raise TemplateRuntimeError(msg) + + +def _environment_sanity_check(environment): + """Perform a sanity check on the environment.""" + assert issubclass(environment.undefined, Undefined), 'undefined must ' \ + 'be a subclass of undefined because filters depend on it.' + assert environment.block_start_string != \ + environment.variable_start_string != \ + environment.comment_start_string, 'block, variable and comment ' \ + 'start strings must be different' + assert environment.newline_sequence in ('\r', '\r\n', '\n'), \ + 'newline_sequence set to unknown line ending string.' + return environment + + +class Environment(object): + r"""The core component of Jinja is the `Environment`. It contains + important shared variables like configuration, filters, tests, + globals and others. Instances of this class may be modified if + they are not shared and if no template was loaded so far. + Modifications on environments after the first template was loaded + will lead to surprising effects and undefined behavior. + + Here are the possible initialization parameters: + + `block_start_string` + The string marking the beginning of a block. Defaults to ``'{%'``. + + `block_end_string` + The string marking the end of a block. Defaults to ``'%}'``. + + `variable_start_string` + The string marking the beginning of a print statement. + Defaults to ``'{{'``. + + `variable_end_string` + The string marking the end of a print statement. Defaults to + ``'}}'``. + + `comment_start_string` + The string marking the beginning of a comment. Defaults to ``'{#'``. + + `comment_end_string` + The string marking the end of a comment. Defaults to ``'#}'``. + + `line_statement_prefix` + If given and a string, this will be used as prefix for line based + statements. See also :ref:`line-statements`. + + `line_comment_prefix` + If given and a string, this will be used as prefix for line based + comments. See also :ref:`line-statements`. + + .. versionadded:: 2.2 + + `trim_blocks` + If this is set to ``True`` the first newline after a block is + removed (block, not variable tag!). Defaults to `False`. + + `lstrip_blocks` + If this is set to ``True`` leading spaces and tabs are stripped + from the start of a line to a block. Defaults to `False`. + + `newline_sequence` + The sequence that starts a newline. Must be one of ``'\r'``, + ``'\n'`` or ``'\r\n'``. The default is ``'\n'`` which is a + useful default for Linux and OS X systems as well as web + applications. + + `keep_trailing_newline` + Preserve the trailing newline when rendering templates. + The default is ``False``, which causes a single newline, + if present, to be stripped from the end of the template. + + .. versionadded:: 2.7 + + `extensions` + List of Jinja extensions to use. This can either be import paths + as strings or extension classes. For more information have a + look at :ref:`the extensions documentation `. + + `optimized` + should the optimizer be enabled? Default is ``True``. + + `undefined` + :class:`Undefined` or a subclass of it that is used to represent + undefined values in the template. + + `finalize` + A callable that can be used to process the result of a variable + expression before it is output. For example one can convert + ``None`` implicitly into an empty string here. + + `autoescape` + If set to ``True`` the XML/HTML autoescaping feature is enabled by + default. For more details about autoescaping see + :class:`~jinja2.utils.Markup`. As of Jinja 2.4 this can also + be a callable that is passed the template name and has to + return ``True`` or ``False`` depending on autoescape should be + enabled by default. + + .. versionchanged:: 2.4 + `autoescape` can now be a function + + `loader` + The template loader for this environment. + + `cache_size` + The size of the cache. Per default this is ``400`` which means + that if more than 400 templates are loaded the loader will clean + out the least recently used template. If the cache size is set to + ``0`` templates are recompiled all the time, if the cache size is + ``-1`` the cache will not be cleaned. + + .. versionchanged:: 2.8 + The cache size was increased to 400 from a low 50. + + `auto_reload` + Some loaders load templates from locations where the template + sources may change (ie: file system or database). If + ``auto_reload`` is set to ``True`` (default) every time a template is + requested the loader checks if the source changed and if yes, it + will reload the template. For higher performance it's possible to + disable that. + + `bytecode_cache` + If set to a bytecode cache object, this object will provide a + cache for the internal Jinja bytecode so that templates don't + have to be parsed if they were not changed. + + See :ref:`bytecode-cache` for more information. + + `enable_async` + If set to true this enables async template execution which allows + you to take advantage of newer Python features. This requires + Python 3.6 or later. + """ + + #: if this environment is sandboxed. Modifying this variable won't make + #: the environment sandboxed though. For a real sandboxed environment + #: have a look at jinja2.sandbox. This flag alone controls the code + #: generation by the compiler. + sandboxed = False + + #: True if the environment is just an overlay + overlayed = False + + #: the environment this environment is linked to if it is an overlay + linked_to = None + + #: shared environments have this set to `True`. A shared environment + #: must not be modified + shared = False + + #: these are currently EXPERIMENTAL undocumented features. + exception_handler = None + exception_formatter = None + + #: the class that is used for code generation. See + #: :class:`~jinja2.compiler.CodeGenerator` for more information. + code_generator_class = CodeGenerator + + #: the context class thatis used for templates. See + #: :class:`~jinja2.runtime.Context` for more information. + context_class = Context + + def __init__(self, + block_start_string=BLOCK_START_STRING, + block_end_string=BLOCK_END_STRING, + variable_start_string=VARIABLE_START_STRING, + variable_end_string=VARIABLE_END_STRING, + comment_start_string=COMMENT_START_STRING, + comment_end_string=COMMENT_END_STRING, + line_statement_prefix=LINE_STATEMENT_PREFIX, + line_comment_prefix=LINE_COMMENT_PREFIX, + trim_blocks=TRIM_BLOCKS, + lstrip_blocks=LSTRIP_BLOCKS, + newline_sequence=NEWLINE_SEQUENCE, + keep_trailing_newline=KEEP_TRAILING_NEWLINE, + extensions=(), + optimized=True, + undefined=Undefined, + finalize=None, + autoescape=False, + loader=None, + cache_size=400, + auto_reload=True, + bytecode_cache=None, + enable_async=False): + # !!Important notice!! + # The constructor accepts quite a few arguments that should be + # passed by keyword rather than position. However it's important to + # not change the order of arguments because it's used at least + # internally in those cases: + # - spontaneous environments (i18n extension and Template) + # - unittests + # If parameter changes are required only add parameters at the end + # and don't change the arguments (or the defaults!) of the arguments + # existing already. + + # lexer / parser information + self.block_start_string = block_start_string + self.block_end_string = block_end_string + self.variable_start_string = variable_start_string + self.variable_end_string = variable_end_string + self.comment_start_string = comment_start_string + self.comment_end_string = comment_end_string + self.line_statement_prefix = line_statement_prefix + self.line_comment_prefix = line_comment_prefix + self.trim_blocks = trim_blocks + self.lstrip_blocks = lstrip_blocks + self.newline_sequence = newline_sequence + self.keep_trailing_newline = keep_trailing_newline + + # runtime information + self.undefined = undefined + self.optimized = optimized + self.finalize = finalize + self.autoescape = autoescape + + # defaults + self.filters = DEFAULT_FILTERS.copy() + self.tests = DEFAULT_TESTS.copy() + self.globals = DEFAULT_NAMESPACE.copy() + + # set the loader provided + self.loader = loader + self.cache = create_cache(cache_size) + self.bytecode_cache = bytecode_cache + self.auto_reload = auto_reload + + # configurable policies + self.policies = DEFAULT_POLICIES.copy() + + # load extensions + self.extensions = load_extensions(self, extensions) + + self.enable_async = enable_async + self.is_async = self.enable_async and have_async_gen + + _environment_sanity_check(self) + + def add_extension(self, extension): + """Adds an extension after the environment was created. + + .. versionadded:: 2.5 + """ + self.extensions.update(load_extensions(self, [extension])) + + def extend(self, **attributes): + """Add the items to the instance of the environment if they do not exist + yet. This is used by :ref:`extensions ` to register + callbacks and configuration values without breaking inheritance. + """ + for key, value in iteritems(attributes): + if not hasattr(self, key): + setattr(self, key, value) + + def overlay(self, block_start_string=missing, block_end_string=missing, + variable_start_string=missing, variable_end_string=missing, + comment_start_string=missing, comment_end_string=missing, + line_statement_prefix=missing, line_comment_prefix=missing, + trim_blocks=missing, lstrip_blocks=missing, + extensions=missing, optimized=missing, + undefined=missing, finalize=missing, autoescape=missing, + loader=missing, cache_size=missing, auto_reload=missing, + bytecode_cache=missing): + """Create a new overlay environment that shares all the data with the + current environment except for cache and the overridden attributes. + Extensions cannot be removed for an overlayed environment. An overlayed + environment automatically gets all the extensions of the environment it + is linked to plus optional extra extensions. + + Creating overlays should happen after the initial environment was set + up completely. Not all attributes are truly linked, some are just + copied over so modifications on the original environment may not shine + through. + """ + args = dict(locals()) + del args['self'], args['cache_size'], args['extensions'] + + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.overlayed = True + rv.linked_to = self + + for key, value in iteritems(args): + if value is not missing: + setattr(rv, key, value) + + if cache_size is not missing: + rv.cache = create_cache(cache_size) + else: + rv.cache = copy_cache(self.cache) + + rv.extensions = {} + for key, value in iteritems(self.extensions): + rv.extensions[key] = value.bind(rv) + if extensions is not missing: + rv.extensions.update(load_extensions(rv, extensions)) + + return _environment_sanity_check(rv) + + lexer = property(get_lexer, doc="The lexer for this environment.") + + def iter_extensions(self): + """Iterates over the extensions by priority.""" + return iter(sorted(self.extensions.values(), + key=lambda x: x.priority)) + + def getitem(self, obj, argument): + """Get an item or attribute of an object but prefer the item.""" + try: + return obj[argument] + except (AttributeError, TypeError, LookupError): + if isinstance(argument, string_types): + try: + attr = str(argument) + except Exception: + pass + else: + try: + return getattr(obj, attr) + except AttributeError: + pass + return self.undefined(obj=obj, name=argument) + + def getattr(self, obj, attribute): + """Get an item or attribute of an object but prefer the attribute. + Unlike :meth:`getitem` the attribute *must* be a bytestring. + """ + try: + return getattr(obj, attribute) + except AttributeError: + pass + try: + return obj[attribute] + except (TypeError, LookupError, AttributeError): + return self.undefined(obj=obj, name=attribute) + + def call_filter(self, name, value, args=None, kwargs=None, + context=None, eval_ctx=None): + """Invokes a filter on a value the same way the compiler does it. + + Note that on Python 3 this might return a coroutine in case the + filter is running from an environment in async mode and the filter + supports async execution. It's your responsibility to await this + if needed. + + .. versionadded:: 2.7 + """ + func = self.filters.get(name) + if func is None: + fail_for_missing_callable('no filter named %r', name) + args = [value] + list(args or ()) + if getattr(func, 'contextfilter', False): + if context is None: + raise TemplateRuntimeError('Attempted to invoke context ' + 'filter without context') + args.insert(0, context) + elif getattr(func, 'evalcontextfilter', False): + if eval_ctx is None: + if context is not None: + eval_ctx = context.eval_ctx + else: + eval_ctx = EvalContext(self) + args.insert(0, eval_ctx) + elif getattr(func, 'environmentfilter', False): + args.insert(0, self) + return func(*args, **(kwargs or {})) + + def call_test(self, name, value, args=None, kwargs=None): + """Invokes a test on a value the same way the compiler does it. + + .. versionadded:: 2.7 + """ + func = self.tests.get(name) + if func is None: + fail_for_missing_callable('no test named %r', name) + return func(value, *(args or ()), **(kwargs or {})) + + @internalcode + def parse(self, source, name=None, filename=None): + """Parse the sourcecode and return the abstract syntax tree. This + tree of nodes is used by the compiler to convert the template into + executable source- or bytecode. This is useful for debugging or to + extract information from templates. + + If you are :ref:`developing Jinja2 extensions ` + this gives you a good overview of the node tree generated. + """ + try: + return self._parse(source, name, filename) + except TemplateSyntaxError: + exc_info = sys.exc_info() + self.handle_exception(exc_info, source_hint=source) + + def _parse(self, source, name, filename): + """Internal parsing function used by `parse` and `compile`.""" + return Parser(self, source, name, encode_filename(filename)).parse() + + def lex(self, source, name=None, filename=None): + """Lex the given sourcecode and return a generator that yields + tokens as tuples in the form ``(lineno, token_type, value)``. + This can be useful for :ref:`extension development ` + and debugging templates. + + This does not perform preprocessing. If you want the preprocessing + of the extensions to be applied you have to filter source through + the :meth:`preprocess` method. + """ + source = text_type(source) + try: + return self.lexer.tokeniter(source, name, filename) + except TemplateSyntaxError: + exc_info = sys.exc_info() + self.handle_exception(exc_info, source_hint=source) + + def preprocess(self, source, name=None, filename=None): + """Preprocesses the source with all extensions. This is automatically + called for all parsing and compiling methods but *not* for :meth:`lex` + because there you usually only want the actual source tokenized. + """ + return reduce(lambda s, e: e.preprocess(s, name, filename), + self.iter_extensions(), text_type(source)) + + def _tokenize(self, source, name, filename=None, state=None): + """Called by the parser to do the preprocessing and filtering + for all the extensions. Returns a :class:`~jinja2.lexer.TokenStream`. + """ + source = self.preprocess(source, name, filename) + stream = self.lexer.tokenize(source, name, filename, state) + for ext in self.iter_extensions(): + stream = ext.filter_stream(stream) + if not isinstance(stream, TokenStream): + stream = TokenStream(stream, name, filename) + return stream + + def _generate(self, source, name, filename, defer_init=False): + """Internal hook that can be overridden to hook a different generate + method in. + + .. versionadded:: 2.5 + """ + return generate(source, self, name, filename, defer_init=defer_init, + optimized=self.optimized) + + def _compile(self, source, filename): + """Internal hook that can be overridden to hook a different compile + method in. + + .. versionadded:: 2.5 + """ + return compile(source, filename, 'exec') + + @internalcode + def compile(self, source, name=None, filename=None, raw=False, + defer_init=False): + """Compile a node or template source code. The `name` parameter is + the load name of the template after it was joined using + :meth:`join_path` if necessary, not the filename on the file system. + the `filename` parameter is the estimated filename of the template on + the file system. If the template came from a database or memory this + can be omitted. + + The return value of this method is a python code object. If the `raw` + parameter is `True` the return value will be a string with python + code equivalent to the bytecode returned otherwise. This method is + mainly used internally. + + `defer_init` is use internally to aid the module code generator. This + causes the generated code to be able to import without the global + environment variable to be set. + + .. versionadded:: 2.4 + `defer_init` parameter added. + """ + source_hint = None + try: + if isinstance(source, string_types): + source_hint = source + source = self._parse(source, name, filename) + source = self._generate(source, name, filename, + defer_init=defer_init) + if raw: + return source + if filename is None: + filename = '