From a304743fbe3a967db1892ca3a0c2c0489cfbd00a Mon Sep 17 00:00:00 2001 From: Jae Kim Date: Mon, 10 Apr 2017 14:18:01 -0700 Subject: [PATCH] Merge rate_limit branch to master (#137) * Added AllocateQuotaRequest/Response support (#99) * Added AllocateQuotaRequest/Response support * Added AllocateQuotaRequest/Response support * Added test cases for rate limiting * Fixed failed test cases caused by recent changes * Removed unnecessary variable and applied server config to disable report cache * Removed unnecessary variable and applied server config to disable report cache * Updated commit id of istio/proxy repo * Set max column to 80, Added report request handler * Merge latest changes from master (#107) * not to use api_key if service is not activated. (#98) * Escape run description Change-Id: I6a66f9be92314218ee6a19b2d82897b6c55c42d2 * Fix grpc interop stress test script. (#103) * try to escape json again. (#106) * Add t test for fail wrong api key. (#104) Change-Id: I0ea9ed4ba94f13283616eae00b93f067f9c00cf5 * Allow grpc interop stress test fail once. (#105) Change-Id: I5a090eae45c1d9b7d0641b2f09757bb373746aa1 * Updated the latest commit from istio/proxy (#109) * Updated commit id of istio/proxy (#117) * Updated commit id of istio/proxy * Fixed proto_pass_perf tool compilation error * Fixed proto_pass_perf tool compilation error * Recover codes acdientally removed * Fixed code style * Update istio/proxy commitid (#131) * Update commitid of istio/proxy --- WORKSPACE | 2 +- src/nginx/t/BUILD | 3 + src/nginx/t/cloud_trace.t | 10 +- src/nginx/t/quota.t | 253 ++++++++++++++++++++++++++ src/nginx/t/quota_api_not_available.t | 200 ++++++++++++++++++++ src/nginx/t/quota_exhausted.t | 218 ++++++++++++++++++++++ src/tools/proto_pass_perf.cc | 9 + src/tools/service_control_json_gen.cc | 28 ++- 8 files changed, 716 insertions(+), 7 deletions(-) create mode 100644 src/nginx/t/quota.t create mode 100644 src/nginx/t/quota_api_not_available.t create mode 100644 src/nginx/t/quota_exhausted.t diff --git a/WORKSPACE b/WORKSPACE index c72bdb7de..4b5dabbb1 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -26,7 +26,7 @@ # # A Bazel (http://bazel.io) workspace for the Google Cloud Endpoints runtime. -ISTIO_PROXY = "987223f6ce6998a8af0f3524545bed7f0d0a1ebb" +ISTIO_PROXY = "d602c7aef35399f7d0c74a7b34175ade3066f44c" git_repository( name = "nginx", diff --git a/src/nginx/t/BUILD b/src/nginx/t/BUILD index ddd6243cd..8295b47b0 100644 --- a/src/nginx/t/BUILD +++ b/src/nginx/t/BUILD @@ -190,6 +190,9 @@ nginx_suite( "multiple_apis.t", "no_backend.t", "no_service_control.t", + "quota.t", + "quota_api_not_available.t", + "quota_exhausted.t", "reject_unrecognized.t", "report_3xx.t", "report_4xx.t", diff --git a/src/nginx/t/cloud_trace.t b/src/nginx/t/cloud_trace.t index de001b46c..01406ff6e 100644 --- a/src/nginx/t/cloud_trace.t +++ b/src/nginx/t/cloud_trace.t @@ -44,7 +44,7 @@ my $BackendPort = ApiManager::pick_port(); my $ServiceControlPort = ApiManager::pick_port(); my $CloudTracePort = ApiManager::pick_port(); -my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(27); +my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(29); my $config = ApiManager::get_bookstore_service_config_allow_unregistered . ApiManager::read_test_file('testdata/logs_metrics.pb.txt') . <<"EOF"; @@ -151,11 +151,15 @@ is($json_obj->{traces}->[0]->{spans}->[3]->{name}, 'Call ServiceControl server', 'Next trace span is Call ServiceControl server'); is($json_obj->{traces}->[0]->{spans}->[3]->{parentSpanId}, $check_service_control_cache_id, 'Parent of Call ServiceControl sever span is CheckServiceControlCache'); -is($json_obj->{traces}->[0]->{spans}->[4]->{name}, 'Backend', +is($json_obj->{traces}->[0]->{spans}->[4]->{name}, 'QuotaControl', 'Next trace span is Backend'); is($json_obj->{traces}->[0]->{spans}->[4]->{parentSpanId}, $rootid, 'Parent of Beckend span is root'); -my $backend_span_id = $json_obj->{traces}->[0]->{spans}->[4]->{spanId}; +is($json_obj->{traces}->[0]->{spans}->[5]->{name}, 'Backend', + 'Next trace span is Backend'); +is($json_obj->{traces}->[0]->{spans}->[5]->{parentSpanId}, $rootid, + 'Parent of Beckend span is root'); +my $backend_span_id = $json_obj->{traces}->[0]->{spans}->[5]->{spanId}; my @bookstore_requests = ApiManager::read_http_stream($t, 'bookstore.log'); is(scalar @bookstore_requests, 1, 'Bookstore received 1 request.'); diff --git a/src/nginx/t/quota.t b/src/nginx/t/quota.t new file mode 100644 index 000000000..a4cc1998d --- /dev/null +++ b/src/nginx/t/quota.t @@ -0,0 +1,253 @@ +# Copyright (C) Extensible Service Proxy Authors +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +################################################################################ +# +use strict; +use warnings; +use JSON::PP; + +################################################################################ + +use src::nginx::t::ApiManager; # Must be first (sets up import path to + # the Nginx test module) +use src::nginx::t::HttpServer; +use src::nginx::t::ServiceControl; +use Test::Nginx; # Imports Nginx's test module +use Test::More; # And the test framework + +################################################################################ + +# Port assignments +my $NginxPort = ApiManager::pick_port(); +my $BackendPort = ApiManager::pick_port(); +my $ServiceControlPort = ApiManager::pick_port(); + +my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(26); + +# Save servce configuration that disables the report cache. +# Report request will be sent for each client request +$t->write_file('server.pb.txt', <<"EOF"); +service_control_config { + report_aggregator_config { + cache_entries: 0 + flush_interval_ms: 1000 + } +} +EOF + +# Save service name in the service configuration protocol buffer file. +$t->write_file( 'service.pb.txt', + ApiManager::get_bookstore_service_config . <<"EOF"); +control { + environment: "http://127.0.0.1:${ServiceControlPort}" +} +quota { + metric_rules [ + { + selector: "ListShelves" + metric_costs: [ + { + key: "metrics_first" + value: 2 + }, + { + key: "metrics_second" + value: 1 + } + ] + } + ] +} +EOF + +ApiManager::write_file_expand( $t, 'nginx.conf', <<"EOF"); +%%TEST_GLOBALS%% +daemon off; +events { + worker_connections 32; +} +http { + %%TEST_GLOBALS_HTTP%% + server_tokens off; + server { + listen 127.0.0.1:${NginxPort}; + server_name localhost; + location / { + endpoints { + api service.pb.txt; + server_config server.pb.txt; + %%TEST_CONFIG%% + on; + } + proxy_pass http://127.0.0.1:${BackendPort}; + } + } +} +EOF + +$t->run_daemon( \&bookstore, $t, $BackendPort, 'bookstore.log' ); +$t->run_daemon( \&servicecontrol, $t, $ServiceControlPort, 'servicecontrol.log' ); +is( $t->waitforsocket("127.0.0.1:${BackendPort}"), 1, 'Bookstore socket ready.' ); +is( $t->waitforsocket("127.0.0.1:${ServiceControlPort}"), 1, + 'Service control socket ready.' ); +$t->run(); + +################################################################################ + +my $response = ApiManager::http_get( + $NginxPort, '/shelves?key=this-is-an-api-key' ); + +$t->stop_daemons(); + +my ( $response_headers, $response_body ) = split /\r\n\r\n/, $response, 2; + +like( $response_headers, qr/HTTP\/1\.1 200 OK/, 'Returned HTTP 200.' ); +is( $response_body, <<'EOF', 'Shelves returned in the response body.' ); +{ "shelves": [ + { "name": "shelves/1", "theme": "Fiction" }, + { "name": "shelves/2", "theme": "Fantasy" } + ] +} +EOF + +my @requests = ApiManager::read_http_stream( $t, 'bookstore.log' ); +is( scalar @requests, 1, 'Backend received one request' ); + +my $r = shift @requests; + +is( $r->{verb}, 'GET', 'Backend request was a get' ); +is( $r->{uri}, '/shelves?key=this-is-an-api-key', 'Backend uri was /shelves' ); +is( $r->{headers}->{host}, "127.0.0.1:${BackendPort}", 'Host header was set' ); + +@requests = ApiManager::read_http_stream( $t, 'servicecontrol.log' ); +is( scalar @requests, 3, 'Service control received three requests' ); + +# check +$r = shift @requests; +is( $r->{verb}, 'POST', ':check verb was post' ); +is( $r->{uri}, '/v1/services/endpoints-test.cloudendpointsapis.com:check', + ':check was called'); +is( $r->{headers}->{host}, "127.0.0.1:${ServiceControlPort}", + 'Host header was set'); +is( $r->{headers}->{'content-type'}, 'application/x-protobuf', + ':check Content-Type was protocol buffer'); + +# test allocateQuota request was requested +$r = shift @requests; +is( $r->{verb}, 'POST', ':allocateQuota verb was post' ); +is( $r->{uri}, + '/v1/services/endpoints-test.cloudendpointsapis.com:allocateQuota', + ':allocateQuota was called'); +is( $r->{headers}->{host}, "127.0.0.1:${ServiceControlPort}", + 'Host header was set'); +is( $r->{headers}->{'content-type'}, 'application/x-protobuf', + ':check Content-Type was protocol buffer' ); + +my $allocate_quota_request = decode_json(ServiceControl::convert_proto( + $r->{body}, 'quota_request', 'json' ) ); + +my @quotaMetrics = + @{ $allocate_quota_request->{allocateOperation}->{quotaMetrics} }; +is( @quotaMetrics, 2, "Quota metrics should have two elements" ); + +my @sorted_quotaMetrics = + sort { $a->{metricName} cmp $b->{metricName} } @quotaMetrics; + +is( $sorted_quotaMetrics[0]->{metricName}, "metrics_first", + "Quota metric name is 'metrics_first'" ); +is( $sorted_quotaMetrics[0]->{metricValues}[0]->{int64Value}, 2, + "Quota metric value is 2" ); +is( $sorted_quotaMetrics[1]->{metricName}, "metrics_second", + "Quota metric name is 'metrics_second'" ); +is( $sorted_quotaMetrics[1]->{metricValues}[0]->{int64Value}, 1, + "Quota metric value is 1" ); + +# check report +$r = shift @requests; + +is( $r->{verb}, 'POST', ':report verb was post' ); +is( $r->{uri}, '/v1/services/endpoints-test.cloudendpointsapis.com:report', + ':report was called'); +is( $r->{headers}->{host}, "127.0.0.1:${ServiceControlPort}", + 'Host header was set'); +is( $r->{headers}->{'content-type'}, 'application/x-protobuf', + ':check Content-Type was protocol buffer' ); + +################################################################################ + +sub bookstore { + my ( $t, $port, $file ) = @_; + my $server = HttpServer->new( $port, $t->testdir() . '/' . $file ) + or die "Can't create test server socket: $!\n"; + local $SIG{PIPE} = 'IGNORE'; + + $server->on( 'GET', '/shelves?key=this-is-an-api-key', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +{ "shelves": [ + { "name": "shelves/1", "theme": "Fiction" }, + { "name": "shelves/2", "theme": "Fantasy" } + ] +} +EOF + $server->run(); +} + +my @quota_responses = (); +my $quota_response_index = 0; + +sub servicecontrol { + my ( $t, $port, $file ) = @_; + my $server = HttpServer->new( $port, $t->testdir() . '/' . $file ) + or die "Can't create test server socket: $!\n"; + local $SIG{PIPE} = 'IGNORE'; + + $server->on( 'POST', + '/v1/services/endpoints-test.cloudendpointsapis.com:check', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +EOF + + $server->on('POST', + '/v1/services/endpoints-test.cloudendpointsapis.com:allocateQuota', + <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +EOF + + $server->on( 'POST', + '/v1/services/endpoints-test.cloudendpointsapis.com:report', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +EOF + + $server->run(); +} + +################################################################################ diff --git a/src/nginx/t/quota_api_not_available.t b/src/nginx/t/quota_api_not_available.t new file mode 100644 index 000000000..277b1c3f7 --- /dev/null +++ b/src/nginx/t/quota_api_not_available.t @@ -0,0 +1,200 @@ +# Copyright (C) Extensible Service Proxy Authors +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +################################################################################ +# +use strict; +use warnings; + +use JSON::PP; +use Data::Dumper; + +################################################################################ + +use src::nginx::t::ApiManager; # Must be first (sets up import path + # to the Nginx test module) +use src::nginx::t::HttpServer; +use src::nginx::t::ServiceControl; +use Test::Nginx; # Imports Nginx's test module +use Test::More; # And the test framework + +################################################################################ + +# Port assignments +my $NginxPort = ApiManager::pick_port(); +my $BackendPort = ApiManager::pick_port(); +my $ServiceControlPort = ApiManager::pick_port(); + +my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(9); + +# Save servce configuration that disables the report cache. +# Report request will be sent for each client request +$t->write_file('server.pb.txt', <<"EOF"); +service_control_config { + report_aggregator_config { + cache_entries: 0 + flush_interval_ms: 1000 + } +} +EOF + +# Save service name in the service configuration protocol buffer file. +$t->write_file( 'service.pb.txt', + ApiManager::get_bookstore_service_config . <<"EOF"); +control { + environment: "http://127.0.0.1:${ServiceControlPort}" +} +quota { + metric_rules [ + { + selector: "ListShelves" + metric_costs: [ + { + key: "metrics_first" + value: 2 + }, + { + key: "metrics_second" + value: 1 + } + ] + } + ] +} +EOF + +ApiManager::write_file_expand( $t, 'nginx.conf', <<"EOF"); +%%TEST_GLOBALS%% +daemon off; +events { + worker_connections 32; +} +http { + %%TEST_GLOBALS_HTTP%% + server_tokens off; + server { + listen 127.0.0.1:${NginxPort}; + server_name localhost; + location / { + endpoints { + api service.pb.txt; + server_config server.pb.txt; + %%TEST_CONFIG%% + on; + } + proxy_pass http://127.0.0.1:${BackendPort}; + } + } +} +EOF + +$t->run_daemon( \&bookstore, $t, $BackendPort, 'bookstore.log' ); +$t->run_daemon( \&servicecontrol, $t, $ServiceControlPort, + 'servicecontrol.log' ); +is( $t->waitforsocket("127.0.0.1:${BackendPort}"), 1, + 'Bookstore socket ready.' ); +is( $t->waitforsocket("127.0.0.1:${ServiceControlPort}"), 1, + 'Service control socket ready.' ); +$t->run(); + +################################################################################ + +my $response = ApiManager::http_get( $NginxPort, + '/shelves?key=this-is-an-api-key' ); + +$t->stop_daemons(); + +my ( $response_headers, $response_body ) = split /\r\n\r\n/, $response, 2; + +like( $response_headers, qr/HTTP\/1\.1 200 OK/, 'Returned HTTP 200.' ); +is( $response_body, <<'EOF', 'Shelves returned in the response body.' ); +{ "shelves": [ + { "name": "shelves/1", "theme": "Fiction" }, + { "name": "shelves/2", "theme": "Fantasy" } + ] +} +EOF + +my @requests = ApiManager::read_http_stream( $t, 'bookstore.log' ); +is( scalar @requests, 1, 'Backend received empty request' ); + +my $r = shift @requests; + +is( $r->{verb}, 'GET', 'Backend request was a get' ); +is( $r->{uri}, '/shelves?key=this-is-an-api-key', 'Backend uri was /shelves' ); +is( $r->{headers}->{host}, "127.0.0.1:${BackendPort}", 'Host header was set' ); + +@requests = ApiManager::read_http_stream( $t, 'servicecontrol.log' ); +is( scalar @requests, 3, 'Service control received three requests' ); + +################################################################################ + +sub bookstore { + my ( $t, $port, $file ) = @_; + my $server = HttpServer->new( $port, $t->testdir() . '/' . $file ) + or die "Can't create test server socket: $!\n"; + local $SIG{PIPE} = 'IGNORE'; + + $server->on( 'GET', '/shelves?key=this-is-an-api-key', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +{ "shelves": [ + { "name": "shelves/1", "theme": "Fiction" }, + { "name": "shelves/2", "theme": "Fantasy" } + ] +} +EOF + $server->run(); +} + +my @quota_responses = (); +my $quota_response_index = 0; + +sub servicecontrol { + my ( $t, $port, $file ) = @_; + my $server = HttpServer->new( $port, $t->testdir() . '/' . $file ) + or die "Can't create test server socket: $!\n"; + local $SIG{PIPE} = 'IGNORE'; + + $server->on( 'POST', + '/v1/services/endpoints-test.cloudendpointsapis.com:check', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +EOF + # The allocateQuota request receives HTTP 404 Not Found error code. + # This simulates QuotaController is not available. + + $server->on( 'POST', + '/v1/services/endpoints-test.cloudendpointsapis.com:report', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +EOF + + $server->run(); +} + +################################################################################ diff --git a/src/nginx/t/quota_exhausted.t b/src/nginx/t/quota_exhausted.t new file mode 100644 index 000000000..0fdd8f404 --- /dev/null +++ b/src/nginx/t/quota_exhausted.t @@ -0,0 +1,218 @@ +# Copyright (C) Extensible Service Proxy Authors +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +################################################################################ +# +use strict; +use warnings; + +use JSON::PP; +use Data::Dumper; + +################################################################################ + +use src::nginx::t::ApiManager; # Must be first (sets up import path to + # the Nginx test module) +use src::nginx::t::HttpServer; +use src::nginx::t::ServiceControl; +use Test::Nginx; # Imports Nginx's test module +use Test::More; # And the test framework + +################################################################################ + +# Port assignments +my $NginxPort = ApiManager::pick_port(); +my $BackendPort = ApiManager::pick_port(); +my $ServiceControlPort = ApiManager::pick_port(); + +my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(6); + +# Save servce configuration that disables the report cache. +# Report request will be sent for each client request +$t->write_file('server.pb.txt', <<"EOF"); +service_control_config { + report_aggregator_config { + cache_entries: 0 + flush_interval_ms: 1000 + } +} +EOF + +# Save service name in the service configuration protocol buffer file. +$t->write_file( 'service.pb.txt', + ApiManager::get_bookstore_service_config . <<"EOF"); +control { + environment: "http://127.0.0.1:${ServiceControlPort}" +} +quota { + metric_rules [ + { + selector: "ListShelves" + metric_costs: [ + { + key: "metrics_first" + value: 2 + }, + { + key: "metrics_second" + value: 1 + } + ] + } + ] +} +EOF + +ApiManager::write_file_expand( $t, 'nginx.conf', <<"EOF"); +%%TEST_GLOBALS%% +daemon off; +events { + worker_connections 32; +} +http { + %%TEST_GLOBALS_HTTP%% + server_tokens off; + server { + listen 127.0.0.1:${NginxPort}; + server_name localhost; + location / { + endpoints { + api service.pb.txt; + server_config server.pb.txt; + %%TEST_CONFIG%% + on; + } + proxy_pass http://127.0.0.1:${BackendPort}; + } + } +} +EOF + +$t->run_daemon( \&bookstore, $t, $BackendPort, 'bookstore.log' ); +$t->run_daemon( \&servicecontrol, $t, $ServiceControlPort, + 'servicecontrol.log' ); +is( $t->waitforsocket("127.0.0.1:${BackendPort}"), + 1, 'Bookstore socket ready.' ); +is( $t->waitforsocket("127.0.0.1:${ServiceControlPort}"), + 1, 'Service control socket ready.' ); +$t->run(); + +################################################################################ + +my $response = + ApiManager::http_get( $NginxPort, '/shelves?key=this-is-an-api-key' ); + +$t->stop_daemons(); + +my ( $response_headers, $response_body ) = split /\r\n\r\n/, $response, 2; + +like( $response_headers, qr/HTTP\/1\.1 429/, 'Returned HTTP 429.' ); +is( $response_body, <<'EOF', 'Shelves returned in the response body.' ); +{ + "code": 8, + "message": "Quota allocation failed.", + "details": [ + { + "@type": "type.googleapis.com/google.rpc.DebugInfo", + "stackEntries": [], + "detail": "internal" + } + ] +} +EOF + +my @requests = ApiManager::read_http_stream( $t, 'bookstore.log' ); +is( scalar @requests, 0, 'Backend received empty request' ); + + +@requests = ApiManager::read_http_stream( $t, 'servicecontrol.log' ); +is( scalar @requests, 3, 'Service control received three requests' ); + + +################################################################################ + +sub bookstore { + my ( $t, $port, $file ) = @_; + my $server = HttpServer->new( $port, $t->testdir() . '/' . $file ) + or die "Can't create test server socket: $!\n"; + local $SIG{PIPE} = 'IGNORE'; + + $server->on( 'GET', '/shelves?key=this-is-an-api-key', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +{ "shelves": [ + { "name": "shelves/1", "theme": "Fiction" }, + { "name": "shelves/2", "theme": "Fantasy" } + ] +} +EOF + $server->run(); +} + +my @quota_responses = (); +my $quota_response_index = 0; + +sub servicecontrol { + my ( $t, $port, $file ) = @_; + my $server = HttpServer->new( $port, $t->testdir() . '/' . $file ) + or die "Can't create test server socket: $!\n"; + local $SIG{PIPE} = 'IGNORE'; + + my $quota_response_exhausted = + ServiceControl::convert_proto( <<'EOF', 'quota_response', 'binary' ); +operation_id: "006eaa26-5c2f-41bc-b6d8-0972eff8bdf6" +allocate_errors { + code: RESOURCE_EXHAUSTED + description: "Insufficient tokens for quota group and limit \'apiWriteQpsPerProject_LOW\' of service \'jaebonginternal.sandbox.google.com\', using the limit by ID \'container:1002409420961\'." +} +service_config_id: "2017-02-08r9" +EOF + + $server->on( 'POST', + '/v1/services/endpoints-test.cloudendpointsapis.com:check', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +EOF + + $server->on('POST', + '/v1/services/endpoints-test.cloudendpointsapis.com:allocateQuota', + <<'EOF' . $quota_response_exhausted); +HTTP/1.1 200 OK +Connection: close + +EOF + + $server->on( 'POST', + '/v1/services/endpoints-test.cloudendpointsapis.com:report', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +EOF + + $server->run(); +} + +################################################################################ diff --git a/src/tools/proto_pass_perf.cc b/src/tools/proto_pass_perf.cc index a68548800..b890fc4c1 100644 --- a/src/tools/proto_pass_perf.cc +++ b/src/tools/proto_pass_perf.cc @@ -43,12 +43,15 @@ using google::api_manager::service_control::Proto; using ::google::api::servicecontrol::v1::CheckRequest; using ::google::api::servicecontrol::v1::CheckResponse; +using ::google::api::servicecontrol::v1::AllocateQuotaRequest; +using ::google::api::servicecontrol::v1::AllocateQuotaResponse; using ::google::api::servicecontrol::v1::ReportRequest; using ::google::api::servicecontrol::v1::ReportResponse; using ::google::protobuf::Arena; using ::google::protobuf::util::Status; using ::google::service_control_client::CheckAggregationOptions; +using ::google::service_control_client::QuotaAggregationOptions; using ::google::service_control_client::ReportAggregationOptions; using ::google::service_control_client::ServiceControlClient; using ::google::service_control_client::ServiceControlClientOptions; @@ -87,6 +90,7 @@ void FillReportRequestInfo(ReportRequestInfo* request) { int total_called_checks = 0; int total_called_reports = 0; +int total_called_quotas = 0; std::string request_text; // Compare the performance for passing Service Control Report protobuf to its @@ -103,10 +107,15 @@ int main() { CheckAggregationOptions(1000000 /*entries*/, 1000000 /* refresh_interval_ms */, 1000000 /*flush_interval_ms*/), + QuotaAggregationOptions(1000000 /*entries*/, + 1000000 /* refresh_interval_ms */), ReportAggregationOptions(1000000 /*entries*/, 1000000 /* refresh_interval_ms */)); options.check_transport = [](const CheckRequest&, CheckResponse*, TransportDoneFunc) { ++total_called_checks; }; + options.quota_transport = [](const AllocateQuotaRequest&, + AllocateQuotaResponse*, + TransportDoneFunc) { ++total_called_quotas; }; options.report_transport = [](const ReportRequest&, ReportResponse*, TransportDoneFunc) { ++total_called_reports; }; client = CreateServiceControlClient(kServiceName, kServiceConfigId, options); diff --git a/src/tools/service_control_json_gen.cc b/src/tools/service_control_json_gen.cc index de7f2594f..f63e557ec 100644 --- a/src/tools/service_control_json_gen.cc +++ b/src/tools/service_control_json_gen.cc @@ -46,6 +46,8 @@ enum OutputType { BINARY, TEXT, JSON }; enum ProtoMessageType { CHECK_REQUEST, CHECK_RESPONSE, + QUOTA_REQUEST, + QUOTA_RESPONSE, REPORT_REQUEST, REPORT_RESPONSE }; @@ -69,12 +71,16 @@ const int kTextOutput = 4; const int kBinaryOutput = 5; const int kCheckRequstProto = 6; const int kCheckResponseProto = 7; -const int kReportRequstProto = 8; -const int kReportResponseProto = 9; -const int kReportRequestSize = 10; +const int kAllocateQuotaRequstProto = 8; +const int kAllocateQuotaResponseProto = 9; +const int kReportRequstProto = 10; +const int kReportResponseProto = 11; +const int kReportRequestSize = 12; ::google::api::servicecontrol::v1::CheckRequest check_request; ::google::api::servicecontrol::v1::CheckResponse check_response; +::google::api::servicecontrol::v1::AllocateQuotaRequest quota_request; +::google::api::servicecontrol::v1::AllocateQuotaResponse quota_response; ::google::api::servicecontrol::v1::ReportRequest report_request; ::google::api::servicecontrol::v1::ReportResponse report_response; @@ -94,6 +100,8 @@ void ProcessCmdLine(int argc, char** argv) { {"binary", no_argument, nullptr, kBinaryOutput}, {"check_request", no_argument, nullptr, kCheckRequstProto}, {"check_response", no_argument, nullptr, kCheckResponseProto}, + {"quota_request", no_argument, nullptr, kAllocateQuotaRequstProto}, + {"quota_response", no_argument, nullptr, kAllocateQuotaResponseProto}, {"report_request", no_argument, nullptr, kReportRequstProto}, {"report_response", no_argument, nullptr, kReportResponseProto}, {"report_request_size", required_argument, nullptr, kReportRequestSize}, @@ -133,6 +141,12 @@ void ProcessCmdLine(int argc, char** argv) { case kCheckResponseProto: proto_type = CHECK_RESPONSE; break; + case kAllocateQuotaRequstProto: + proto_type = QUOTA_REQUEST; + break; + case kAllocateQuotaResponseProto: + proto_type = QUOTA_RESPONSE; + break; case kReportRequstProto: proto_type = REPORT_REQUEST; break; @@ -163,6 +177,8 @@ void ProcessCmdLine(int argc, char** argv) { "Protobuf message type:\n" " --check_request: CheckRequest protobuf message.\n" " --check_response: CheckResponse protobuf message.\n" + " --quota_request: AllocateQuotaRequest protobuf message.\n" + " --quota_response: AllocateQuotaResponse protobuf message.\n" " --report_request: ReportRequest protobuf message.\n" " --report_response: ReportResponse protobuf message.\n" "Input:\n" @@ -316,6 +332,12 @@ int main(int argc, char** argv) { case CHECK_RESPONSE: request = &check_response; break; + case QUOTA_REQUEST: + request = "a_request; + break; + case QUOTA_RESPONSE: + request = "a_response; + break; case REPORT_REQUEST: request = &report_request; break;