From 678988696a9f3d8412f1f14a757ab07e126a9497 Mon Sep 17 00:00:00 2001 From: lijunlong Date: Sat, 13 Jun 2020 10:57:08 +0800 Subject: [PATCH 01/19] bugfix: set_by_lua_block allowed more than one arg (in addition to the block). --- src/ngx_http_lua_module.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ngx_http_lua_module.c b/src/ngx_http_lua_module.c index 5303ae76f8..9816d86441 100644 --- a/src/ngx_http_lua_module.c +++ b/src/ngx_http_lua_module.c @@ -230,10 +230,10 @@ static ngx_command_t ngx_http_lua_cmds[] = { (void *) ngx_http_lua_init_worker_by_file }, #if defined(NDK) && NDK - /* set_by_lua $res { inline Lua code } [$arg1 [$arg2 [...]]] */ + /* set_by_lua_block $res { inline Lua code } */ { ngx_string("set_by_lua_block"), NGX_HTTP_SRV_CONF|NGX_HTTP_SIF_CONF|NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF - |NGX_CONF_1MORE|NGX_CONF_BLOCK, + |NGX_CONF_TAKE1|NGX_CONF_BLOCK, ngx_http_lua_set_by_lua_block, NGX_HTTP_LOC_CONF_OFFSET, 0, From 23d2bc10a16fc56fef36df3344d4cf5ffa1681f1 Mon Sep 17 00:00:00 2001 From: zhuizhuhaomeng <151090888@qq.com> Date: Sat, 25 Apr 2020 16:41:11 +0800 Subject: [PATCH 02/19] bugfix: ngx.req.set_uri_args() threw an exception with wrong argument info. --- src/ngx_http_lua_args.c | 2 +- t/030-uri-args.t | 71 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/src/ngx_http_lua_args.c b/src/ngx_http_lua_args.c index 6da3759212..45ee6023ca 100644 --- a/src/ngx_http_lua_args.c +++ b/src/ngx_http_lua_args.c @@ -64,7 +64,7 @@ ngx_http_lua_ngx_req_set_uri_args(lua_State *L) default: msg = lua_pushfstring(L, "string, number, or table expected, " - "but got %s", luaL_typename(L, 2)); + "but got %s", luaL_typename(L, 1)); return luaL_argerror(L, 1, msg); } diff --git a/t/030-uri-args.t b/t/030-uri-args.t index 02fe6e1042..951355e527 100644 --- a/t/030-uri-args.t +++ b/t/030-uri-args.t @@ -9,7 +9,7 @@ log_level('warn'); repeat_each(2); #repeat_each(1); -plan tests => repeat_each() * (blocks() * 2 + 21); +plan tests => repeat_each() * (blocks() * 2 + 24); no_root_location(); @@ -1617,3 +1617,72 @@ request_uri: /foo%20bar uri: /foo bar --- no_error_log [error] + + + +=== TEST 61: set_uri_args with boolean +--- config + location /bar { + echo $query_string; + } + location /foo { + #set $args 'hello'; + rewrite_by_lua_block { + ngx.req.set_uri_args(true) + ngx.req.set_uri("/bar", true) + } + proxy_pass http://127.0.0.2:12345; + } +--- request + GET /foo?world +--- response_body_like: 500 Internal Server Error +--- log_level: debug +--- error_code: 500 +--- error_log +bad argument #1 to 'set_uri_args' (string, number, or table expected, but got boolean) + + + +=== TEST 62: set_uri_args with nil +--- config + location /bar { + echo $query_string; + } + location /foo { + #set $args 'hello'; + rewrite_by_lua_block { + ngx.req.set_uri_args(nil) + ngx.req.set_uri("/bar", true) + } + proxy_pass http://127.0.0.2:12345; + } +--- request + GET /foo?world +--- response_body_like: 500 Internal Server Error +--- log_level: debug +--- error_code: 500 +--- error_log +bad argument #1 to 'set_uri_args' (string, number, or table expected, but got nil) + + + +=== TEST 63: set_uri_args with userdata +--- config + location /bar { + echo $query_string; + } + location /foo { + #set $args 'hello'; + rewrite_by_lua_block { + ngx.req.set_uri_args(ngx.null) + ngx.req.set_uri("/bar", true) + } + proxy_pass http://127.0.0.2:12345; + } +--- request + GET /foo?world +--- response_body_like: 500 Internal Server Error +--- log_level: debug +--- error_code: 500 +--- error_log +bad argument #1 to 'set_uri_args' (string, number, or table expected, but got userdata) From d278043fb0f9856dac00a428c90f237ae43ffc9a Mon Sep 17 00:00:00 2001 From: wanghuizzz Date: Fri, 17 Apr 2020 01:48:46 -0400 Subject: [PATCH 03/19] doc: added docs for working around memory fragmentation issues to the shdict:set() method. --- README.markdown | 4 ++++ doc/HttpLuaModule.wiki | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/README.markdown b/README.markdown index ee87baeee7..e36a4c83d6 100644 --- a/README.markdown +++ b/README.markdown @@ -6548,6 +6548,10 @@ The optional `flags` argument specifies a user flags value associated with the e When it fails to allocate memory for the current key-value item, then `set` will try removing existing items in the storage according to the Least-Recently Used (LRU) algorithm. Note that, LRU takes priority over expiration time here. If up to tens of existing items have been removed and the storage left is still insufficient (either due to the total capacity limit specified by [lua_shared_dict](#lua_shared_dict) or memory segmentation), then the `err` return value will be `no memory` and `success` will be `false`. +If the sizes of items in the dictionary are not multiples or even powers of a certain value (like 2), it is easier to encounter `no memory` error because of memory fragmentation. It is recommended to use different dictionaries for different sizes of items. + +When you encounter `no memory` error, you can also evict more least-recently-used items by retrying this method call more times to to make room for the current item. + If this method succeeds in storing the current item by forcibly removing other not-yet-expired items in the dictionary via LRU, the `forcible` return value will be `true`. If it stores the item without forcibly removing other valid items, then the return value `forcible` will be `false`. The first argument to this method must be the dictionary object itself, for example, diff --git a/doc/HttpLuaModule.wiki b/doc/HttpLuaModule.wiki index 97f7a4a0ea..a3b6ddab46 100644 --- a/doc/HttpLuaModule.wiki +++ b/doc/HttpLuaModule.wiki @@ -5528,6 +5528,10 @@ The optional flags argument specifies a user flags value associated When it fails to allocate memory for the current key-value item, then set will try removing existing items in the storage according to the Least-Recently Used (LRU) algorithm. Note that, LRU takes priority over expiration time here. If up to tens of existing items have been removed and the storage left is still insufficient (either due to the total capacity limit specified by [[#lua_shared_dict|lua_shared_dict]] or memory segmentation), then the err return value will be no memory and success will be false. +If the sizes of items in the dictionary are not multiples or even powers of a certain value (like 2), it is easier to encounter no memory error because of memory fragmentation. It is recommended to use different dictionaries for different sizes of items. + +When you encounter no memory error, you can also evict more least-recently-used items by retrying this method call more times to to make room for the current item. + If this method succeeds in storing the current item by forcibly removing other not-yet-expired items in the dictionary via LRU, the forcible return value will be true. If it stores the item without forcibly removing other valid items, then the return value forcible will be false. The first argument to this method must be the dictionary object itself, for example, From 8ac6cc7285994026645deb8d24f185e64db8c2d8 Mon Sep 17 00:00:00 2001 From: rainingmaster <312841925@qq.com> Date: Sat, 4 Apr 2020 16:58:16 +0800 Subject: [PATCH 04/19] doc: made the code examples more realistic (better for direct copy&paste use). --- README.markdown | 49 ++++++++++++++++++++++++++++-------------- doc/HttpLuaModule.wiki | 49 ++++++++++++++++++++++++++++-------------- 2 files changed, 66 insertions(+), 32 deletions(-) diff --git a/README.markdown b/README.markdown index e36a4c83d6..21917d4bf4 100644 --- a/README.markdown +++ b/README.markdown @@ -1512,6 +1512,8 @@ This hook is often used to create per-worker reoccurring timers (via the [ngx.ti return end end + + -- do something in timer end local hdl, err = new_timer(delay, check) @@ -1519,6 +1521,8 @@ This hook is often used to create per-worker reoccurring timers (via the [ngx.ti log(ERR, "failed to create timer: ", err) return end + + -- other job in init_worker_by_lua '; ``` @@ -6292,13 +6296,13 @@ When the `replace` is a string, then it is treated as a special template for str ```lua local newstr, n, err = ngx.re.sub("hello, 1234", "([0-9])[0-9]", "[$0][$1]") - if newstr then - -- newstr == "hello, [12][1]34" - -- n == 1 - else + if not newstr then ngx.log(ngx.ERR, "error: ", err) return end + + -- newstr == "hello, [12][1]34" + -- n == 1 ``` where `$0` referring to the whole substring matched by the pattern and `$1` referring to the first parenthesized capturing substring. @@ -6308,8 +6312,8 @@ Curly braces can also be used to disambiguate variable names from the background ```lua local newstr, n, err = ngx.re.sub("hello, 1234", "[0-9]", "${0}00") - -- newstr == "hello, 100234" - -- n == 1 + -- newstr == "hello, 100234" + -- n == 1 ``` Literal dollar sign characters (`$`) in the `replace` string argument can be escaped by another dollar sign, for instance, @@ -6317,8 +6321,8 @@ Literal dollar sign characters (`$`) in the `replace` string argument can be esc ```lua local newstr, n, err = ngx.re.sub("hello, 1234", "[0-9]", "$$") - -- newstr == "hello, $234" - -- n == 1 + -- newstr == "hello, $234" + -- n == 1 ``` Do not use backlashes to escape dollar signs; it will not work as expected. @@ -6330,9 +6334,10 @@ When the `replace` argument is of type "function", then it will be invoked with local func = function (m) return "[" .. m[0] .. "][" .. m[1] .. "]" end + local newstr, n, err = ngx.re.sub("hello, 1234", "( [0-9] ) [0-9]", func, "x") - -- newstr == "hello, [12][1]34" - -- n == 1 + -- newstr == "hello, [12][1]34" + -- n == 1 ``` The dollar sign characters in the return value of the `replace` function argument are not special at all. @@ -6357,13 +6362,13 @@ Here is some examples: ```lua local newstr, n, err = ngx.re.gsub("hello, world", "([a-z])[a-z]+", "[$0,$1]", "i") - if newstr then - -- newstr == "[hello,h], [world,w]" - -- n == 2 - else + if not newstr then ngx.log(ngx.ERR, "error: ", err) return end + + -- newstr == "[hello,h], [world,w]" + -- n == 2 ``` ```lua @@ -6372,8 +6377,8 @@ Here is some examples: return "[" .. m[0] .. "," .. m[1] .. "]" end local newstr, n, err = ngx.re.gsub("hello, world", "([a-z])[a-z]+", func, "i") - -- newstr == "[hello,h], [world,w]" - -- n == 2 + -- newstr == "[hello,h], [world,w]" + -- n == 2 ``` This method requires the PCRE library enabled in Nginx ([Known Issue With Special Escaping Sequences](#special-escaping-sequences)). @@ -7078,6 +7083,9 @@ Since the `v0.7.18` release, connecting to a datagram unix domain socket file is ngx.say("failed to connect to the datagram unix domain socket: ", err) return end + + -- do something after connect + -- such as sock:send or sock:receive ``` assuming the datagram service is listening on the unix domain socket file `/tmp/some-datagram-service.sock` and the client socket will use the "autobind" feature on Linux. @@ -7282,6 +7290,9 @@ Connecting to a Unix Domain Socket file is also possible: ngx.say("failed to connect to the memcached unix domain socket: ", err) return end + + -- do something after connect + -- such as sock:send or sock:receive ``` assuming memcached (or something else) is listening on the unix domain socket file `/tmp/memcached.sock`. @@ -8160,6 +8171,8 @@ Here is a simple example: ngx.log(ngx.ERR, "failed to create timer: ", err) return end + + -- other job in log_by_lua_block } } ``` @@ -8180,6 +8193,8 @@ One can also create infinite re-occurring timers, for instance, a timer getting ngx.log(ngx.ERR, "failed to create the timer: ", err) return end + + -- do something in timer end local ok, err = ngx.timer.at(delay, handler) @@ -8187,6 +8202,8 @@ One can also create infinite re-occurring timers, for instance, a timer getting ngx.log(ngx.ERR, "failed to create the timer: ", err) return end + + -- do other jobs ``` It is recommended, however, to use the [ngx.timer.every](#ngxtimerevery) API function diff --git a/doc/HttpLuaModule.wiki b/doc/HttpLuaModule.wiki index a3b6ddab46..ee3d9c122c 100644 --- a/doc/HttpLuaModule.wiki +++ b/doc/HttpLuaModule.wiki @@ -1224,6 +1224,8 @@ This hook is often used to create per-worker reoccurring timers (via the [[#ngx. return end end + + -- do something in timer end local hdl, err = new_timer(delay, check) @@ -1231,6 +1233,8 @@ This hook is often used to create per-worker reoccurring timers (via the [[#ngx. log(ERR, "failed to create timer: ", err) return end + + -- other job in init_worker_by_lua '; @@ -5296,13 +5300,13 @@ When the replace is a string, then it is treated as a special templ local newstr, n, err = ngx.re.sub("hello, 1234", "([0-9])[0-9]", "[$0][$1]") - if newstr then - -- newstr == "hello, [12][1]34" - -- n == 1 - else + if not newstr then ngx.log(ngx.ERR, "error: ", err) return end + + -- newstr == "hello, [12][1]34" + -- n == 1 where $0 referring to the whole substring matched by the pattern and $1 referring to the first parenthesized capturing substring. @@ -5311,16 +5315,16 @@ Curly braces can also be used to disambiguate variable names from the background local newstr, n, err = ngx.re.sub("hello, 1234", "[0-9]", "${0}00") - -- newstr == "hello, 100234" - -- n == 1 + -- newstr == "hello, 100234" + -- n == 1 Literal dollar sign characters ($) in the replace string argument can be escaped by another dollar sign, for instance, local newstr, n, err = ngx.re.sub("hello, 1234", "[0-9]", "$$") - -- newstr == "hello, $234" - -- n == 1 + -- newstr == "hello, $234" + -- n == 1 Do not use backlashes to escape dollar signs; it will not work as expected. @@ -5331,9 +5335,10 @@ When the replace argument is of type "function", then it will be in local func = function (m) return "[" .. m[0] .. "][" .. m[1] .. "]" end + local newstr, n, err = ngx.re.sub("hello, 1234", "( [0-9] ) [0-9]", func, "x") - -- newstr == "hello, [12][1]34" - -- n == 1 + -- newstr == "hello, [12][1]34" + -- n == 1 The dollar sign characters in the return value of the replace function argument are not special at all. @@ -5354,13 +5359,13 @@ Here is some examples: local newstr, n, err = ngx.re.gsub("hello, world", "([a-z])[a-z]+", "[$0,$1]", "i") - if newstr then - -- newstr == "[hello,h], [world,w]" - -- n == 2 - else + if not newstr then ngx.log(ngx.ERR, "error: ", err) return end + + -- newstr == "[hello,h], [world,w]" + -- n == 2 @@ -5368,8 +5373,8 @@ Here is some examples: return "[" .. m[0] .. "," .. m[1] .. "]" end local newstr, n, err = ngx.re.gsub("hello, world", "([a-z])[a-z]+", func, "i") - -- newstr == "[hello,h], [world,w]" - -- n == 2 + -- newstr == "[hello,h], [world,w]" + -- n == 2 This method requires the PCRE library enabled in Nginx ([[#Special Escaping Sequences|Known Issue With Special Escaping Sequences]]). @@ -5988,6 +5993,9 @@ Since the v0.7.18 release, connecting to a datagram unix domain soc ngx.say("failed to connect to the datagram unix domain socket: ", err) return end + + -- do something after connect + -- such as sock:send or sock:receive assuming the datagram service is listening on the unix domain socket file /tmp/some-datagram-service.sock and the client socket will use the "autobind" feature on Linux. @@ -6167,6 +6175,9 @@ Connecting to a Unix Domain Socket file is also possible: ngx.say("failed to connect to the memcached unix domain socket: ", err) return end + + -- do something after connect + -- such as sock:send or sock:receive assuming memcached (or something else) is listening on the unix domain socket file /tmp/memcached.sock. @@ -6976,6 +6987,8 @@ Here is a simple example: ngx.log(ngx.ERR, "failed to create timer: ", err) return end + + -- other job in log_by_lua_block } } @@ -6995,6 +7008,8 @@ One can also create infinite re-occurring timers, for instance, a timer getting ngx.log(ngx.ERR, "failed to create the timer: ", err) return end + + -- do something in timer end local ok, err = ngx.timer.at(delay, handler) @@ -7002,6 +7017,8 @@ One can also create infinite re-occurring timers, for instance, a timer getting ngx.log(ngx.ERR, "failed to create the timer: ", err) return end + + -- do other jobs It is recommended, however, to use the [[#ngx.timer.every|ngx.timer.every]] API function From c86819f0b0972d2d05679c790ecbbbfa73339ae9 Mon Sep 17 00:00:00 2001 From: Yuchen Wu Date: Fri, 19 Jun 2020 13:59:43 -0700 Subject: [PATCH 05/19] doc: updated the docs on the limits of subrequests to reflect recent changes in the nginx core. --- README.markdown | 2 +- doc/HttpLuaModule.wiki | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.markdown b/README.markdown index 21917d4bf4..77ca7ce32f 100644 --- a/README.markdown +++ b/README.markdown @@ -4044,7 +4044,7 @@ in gzipped responses that cannot be handled properly in Lua code. Original reque When the `body` option is not specified and the `always_forward_body` option is false (the default value), the `POST` and `PUT` subrequests will inherit the request bodies of the parent request (if any). -There is a hard-coded upper limit on the number of concurrent subrequests possible for every main request. In older versions of Nginx, the limit was `50` concurrent subrequests and in more recent versions, Nginx `1.1.x` onwards, this was increased to `200` concurrent subrequests. When this limit is exceeded, the following error message is added to the `error.log` file: +There is a hard-coded upper limit on the number of subrequests possible for every main request. In older versions of Nginx, the limit was `50` concurrent subrequests and in more recent versions, Nginx `1.9.5` onwards, the same limit is changed to limit the depth of recursive subrequests. When this limit is exceeded, the following error message is added to the `error.log` file: [error] 13983#0: *1 subrequests cycle while processing "/uri" diff --git a/doc/HttpLuaModule.wiki b/doc/HttpLuaModule.wiki index ee3d9c122c..db9367a6ca 100644 --- a/doc/HttpLuaModule.wiki +++ b/doc/HttpLuaModule.wiki @@ -3348,7 +3348,7 @@ in gzipped responses that cannot be handled properly in Lua code. Original reque When the body option is not specified and the always_forward_body option is false (the default value), the POST and PUT subrequests will inherit the request bodies of the parent request (if any). -There is a hard-coded upper limit on the number of concurrent subrequests possible for every main request. In older versions of Nginx, the limit was 50 concurrent subrequests and in more recent versions, Nginx 1.1.x onwards, this was increased to 200 concurrent subrequests. When this limit is exceeded, the following error message is added to the error.log file: +There is a hard-coded upper limit on the number of subrequests possible for every main request. In older versions of Nginx, the limit was 50 concurrent subrequests and in more recent versions, Nginx 1.9.5 onwards, the same limit is changed to limit the depth of recursive subrequests. When this limit is exceeded, the following error message is added to the error.log file: [error] 13983#0: *1 subrequests cycle while processing "/uri" From f655b0e19a34c206ce8a2dddbb1b93b9293c221d Mon Sep 17 00:00:00 2001 From: rainingmaster <312841925@qq.com> Date: Wed, 10 Jun 2020 11:55:52 +0800 Subject: [PATCH 06/19] feature: add mergify.yml to manage our pr --- .mergify.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .mergify.yml diff --git a/.mergify.yml b/.mergify.yml new file mode 100644 index 0000000000..c89be4fb54 --- /dev/null +++ b/.mergify.yml @@ -0,0 +1,35 @@ +--- +pull_request_rules: + - name: warn on conflicts + conditions: + - conflict + actions: + comment: + message: This pull request is now in conflict :( + label: + add: + - conflict + - name: remove conflict label if not needed + conditions: + - -conflict + actions: + label: + remove: + - conflict + - name: add label needs-test-cases + conditions: + - files~=^src/ + - -files~=^t/ + actions: + label: + add: + - needs-test-cases + - name: remove label needs-test-cases + conditions: + - label=needs-test-cases + - files~=^src/ + - files~=^t/ + actions: + label: + remove: + - needs-test-cases From 99a5a6bcb4d72d07467f1dd9c82b7a8d2c6055ec Mon Sep 17 00:00:00 2001 From: lijunlong Date: Mon, 8 Jun 2020 14:27:40 +0800 Subject: [PATCH 07/19] test: used the iptable -I option to insert rule to the head of the filter chain. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b7fa83a91a..fc2e4c20bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -94,8 +94,8 @@ before_script: script: - sudo iptables -I OUTPUT 1 -p udp --dport 10086 -j REJECT - - sudo iptables -A OUTPUT -p tcp --dst 127.0.0.2 --dport 12345 -j DROP - - sudo iptables -A OUTPUT -p udp --dst 127.0.0.2 --dport 12345 -j DROP + - sudo iptables -I OUTPUT -p tcp --dst 127.0.0.2 --dport 12345 -j DROP + - sudo iptables -I OUTPUT -p udp --dst 127.0.0.2 --dport 12345 -j DROP - cd luajit2/ - make -j$JOBS CCDEBUG=-g Q= PREFIX=$LUAJIT_PREFIX CC=$CC XCFLAGS='-DLUA_USE_APICHECK -DLUA_USE_ASSERT -msse4.2' > build.log 2>&1 || (cat build.log && exit 1) - sudo make install PREFIX=$LUAJIT_PREFIX > build.log 2>&1 || (cat build.log && exit 1) From 785fb57094e3dbb161962875800e9cf92ce3717b Mon Sep 17 00:00:00 2001 From: lijunlong Date: Sat, 6 Jun 2020 14:51:07 +0800 Subject: [PATCH 08/19] feature: ngx.req.set_uri_args() now automatically escapes control and whitespace characters if the query-string is provided directly. Signed-off-by: Yichun Zhang (agentzh) --- README.markdown | 7 +- doc/HttpLuaModule.wiki | 7 +- src/ngx_http_lua_args.c | 88 +++++++++++++++++++++++- t/030-uri-args-with-ctrl.t | 137 +++++++++++++++++++++++++++++++++++++ 4 files changed, 236 insertions(+), 3 deletions(-) create mode 100644 t/030-uri-args-with-ctrl.t diff --git a/README.markdown b/README.markdown index 77ca7ce32f..c822b3d498 100644 --- a/README.markdown +++ b/README.markdown @@ -4552,7 +4552,12 @@ or a Lua table holding the query arguments' key-value pairs, as in ngx.req.set_uri_args({ a = 3, b = "hello world" }) ``` -where in the latter case, this method will escape argument keys and values according to the URI escaping rule. +In the former case, i.e., when the whole query-strng is provided directly, +the input Lua string should already be well-formed with the URI encoding. +For security considerations, his method will autoamticaly escape any control and +whitespace characters (ASCII code 0x00 ~ 0x32 and 0x7F) in the Lua string. + +In the latter case, this method will escape argument keys and values according to the URI escaping rule. Multi-value arguments are also supported: diff --git a/doc/HttpLuaModule.wiki b/doc/HttpLuaModule.wiki index db9367a6ca..6f99701605 100644 --- a/doc/HttpLuaModule.wiki +++ b/doc/HttpLuaModule.wiki @@ -3791,7 +3791,12 @@ or a Lua table holding the query arguments' key-value pairs, as in ngx.req.set_uri_args({ a = 3, b = "hello world" }) -where in the latter case, this method will escape argument keys and values according to the URI escaping rule. +In the former case, i.e., when the whole query-strng is provided directly, +the input Lua string should already be well-formed with the URI encoding. +For security considerations, his method will autoamticaly escape any control and +whitespace characters (ASCII code 0x00 ~ 0x32 and 0x7F) in the Lua string. + +In the latter case, this method will escape argument keys and values according to the URI escaping rule. Multi-value arguments are also supported: diff --git a/src/ngx_http_lua_args.c b/src/ngx_http_lua_args.c index 45ee6023ca..7b7d5eae6c 100644 --- a/src/ngx_http_lua_args.c +++ b/src/ngx_http_lua_args.c @@ -19,6 +19,66 @@ static int ngx_http_lua_ngx_req_set_uri_args(lua_State *L); static int ngx_http_lua_ngx_req_get_post_args(lua_State *L); +uintptr_t +ngx_http_lua_escape_args(u_char *dst, u_char *src, size_t size) +{ + ngx_uint_t n; + static u_char hex[] = "0123456789ABCDEF"; + + /* %00-%20 %7F*/ + + static uint32_t escape[] = { + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + + /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */ + 0x00000001, /* 0000 0000 0000 0000 0000 0000 0000 0001 */ + + /* _^]\ [ZYX WVUT SRQP ONML KJIH GFED CBA@ */ + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + + /* ~}| {zyx wvut srqp onml kjih gfed cba` */ + 0x80000000, /* 1000 0000 0000 0000 0000 0000 0000 0000 */ + + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + }; + + if (dst == NULL) { + + /* find the number of the characters to be escaped */ + + n = 0; + + while (size) { + if (escape[*src >> 5] & (1 << (*src & 0x1f))) { + n++; + } + src++; + size--; + } + + return (uintptr_t) n; + } + + while (size) { + if (escape[*src >> 5] & (1 << (*src & 0x1f))) { + *dst++ = '%'; + *dst++ = hex[*src >> 4]; + *dst++ = hex[*src & 0xf]; + src++; + + } else { + *dst++ = *src++; + } + size--; + } + + return (uintptr_t) dst; +} + + static int ngx_http_lua_ngx_req_set_uri_args(lua_State *L) { @@ -27,6 +87,7 @@ ngx_http_lua_ngx_req_set_uri_args(lua_State *L) const char *msg; size_t len; u_char *p; + uintptr_t escape; if (lua_gettop(L) != 1) { return luaL_error(L, "expecting 1 argument but seen %d", @@ -42,7 +103,6 @@ ngx_http_lua_ngx_req_set_uri_args(lua_State *L) switch (lua_type(L, 1)) { case LUA_TNUMBER: - case LUA_TSTRING: p = (u_char *) lua_tolstring(L, 1, &len); args.data = ngx_palloc(r->pool, len); @@ -55,6 +115,32 @@ ngx_http_lua_ngx_req_set_uri_args(lua_State *L) args.len = len; break; + case LUA_TSTRING: + p = (u_char *) lua_tolstring(L, 1, &len); + + escape = ngx_http_lua_escape_args(NULL, p, len); + if (escape > 0) { + args.len = len + 2 * escape; + args.data = ngx_palloc(r->pool, args.len); + if (args.data == NULL) { + return NGX_ERROR; + } + + ngx_http_lua_escape_args(args.data, p, len); + + } else { + args.data = ngx_palloc(r->pool, len); + if (args.data == NULL) { + return luaL_error(L, "no memory"); + } + + ngx_memcpy(args.data, p, len); + + args.len = len; + } + + break; + case LUA_TTABLE: ngx_http_lua_process_args_option(r, L, 1, &args); diff --git a/t/030-uri-args-with-ctrl.t b/t/030-uri-args-with-ctrl.t new file mode 100644 index 0000000000..e74da38eeb --- /dev/null +++ b/t/030-uri-args-with-ctrl.t @@ -0,0 +1,137 @@ +# vim:set ft= ts=4 sw=4 et fdm=marker: +use Test::Nginx::Socket::Lua; + +log_level('warn'); + +repeat_each(2); + +plan tests => repeat_each() * (blocks() * 2 + 3); + +no_root_location(); + +no_long_string(); +run_tests(); + +__DATA__ + +=== TEST 1: rewrite args (string with \r) +--- config + location /foo { + rewrite_by_lua_block { + ngx.req.set_uri_args("a\rb") + } + proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/echo; + } + location /echo { + content_by_lua_block { + ngx.say(ngx.var.request_uri); + } + } +--- request +GET /foo?world +--- error_code: 200 +--- response_body +/echo?a%0Db + + + +=== TEST 2: rewrite args (string with \n) +--- config + location /foo { + rewrite_by_lua_block { + ngx.req.set_uri_args("a\nb") + } + proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/echo; + } + location /echo { + content_by_lua_block { + ngx.say(ngx.var.request_uri); + } + } +--- request +GET /foo?world +--- response_body +/echo?a%0Ab + + + +=== TEST 3: rewrite args (string with \0) +--- config + location /foo { + rewrite_by_lua_block { + ngx.req.set_uri_args("a\0b") + } + proxy_pass http://127.0.0.1:$TEST_NGINX_SERVER_PORT/echo; + } + location /echo { + content_by_lua_block { + ngx.say(ngx.var.request_uri); + } + } +--- request +GET /foo?world +--- response_body +/echo?a%00b + + + +=== TEST 4: rewrite args (string arg with 'lang=中文') +ngx.req.set_uri_args with string argument should be carefully encoded. +For backward compatibility, we are allowed to pass such parameters. +--- config + location /foo { + rewrite_by_lua_block { + ngx.req.set_uri_args("lang=中文") + } + content_by_lua_block { + ngx.say(ngx.var.arg_lang) + } + } +--- request +GET /foo?world +--- response_body +中文 +--- no_error_log +[error] + + + +=== TEST 5: rewrite args (string arg with '语言=chinese') +ngx.req.set_uri_args with string argument should be carefully encoded. +For backward compatibility, we are allowed to pass such parameters. +--- config + location /foo { + rewrite_by_lua_block { + ngx.req.set_uri_args("语言=chinese") + } + content_by_lua_block { + ngx.say(ngx.var.arg_语言) + } + } +--- request +GET /foo?world +--- response_body +chinese +--- no_error_log +[error] + + + +=== TEST 6: rewrite args (string arg with '语言=中文') +ngx.req.set_uri_args with string argument should be carefully encoded. +For backward compatibility, we are allowed to pass such parameters. +--- config + location /foo { + rewrite_by_lua_block { + ngx.req.set_uri_args("语言=中文") + } + content_by_lua_block { + ngx.say(ngx.var.arg_语言) + } + } +--- request +GET /foo?world +--- response_body +中文 +--- no_error_log +[error] From c6ffcfbf375b044792db17c739df8e5dcf99298a Mon Sep 17 00:00:00 2001 From: lijunlong Date: Sat, 6 Jun 2020 10:47:39 +0800 Subject: [PATCH 09/19] tests: added test cases for the nginx core patch static_mod_escape_loc_hdr. See https://github.com/openresty/openresty/blob/master/patches/nginx-1.17.8-static_mod_escape_loc_hdr.patch and https://github.com/openresty/openresty/commit/6985198d. --- src/ngx_http_lua_uri.c | 4 -- t/030-uri-args.t | 16 +++--- t/162-static-module-location.t | 95 ++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 13 deletions(-) create mode 100644 t/162-static-module-location.t diff --git a/src/ngx_http_lua_uri.c b/src/ngx_http_lua_uri.c index 569cb50270..b605bf2b89 100644 --- a/src/ngx_http_lua_uri.c +++ b/src/ngx_http_lua_uri.c @@ -55,10 +55,6 @@ ngx_http_lua_ngx_req_set_uri(lua_State *L) return luaL_error(L, "attempt to use zero-length uri"); } - if (ngx_http_lua_check_unsafe_string(r, p, len, "uri") != NGX_OK) { - return luaL_error(L, "attempt to set unsafe uri"); - } - if (n == 2) { luaL_checktype(L, 2, LUA_TBOOLEAN); diff --git a/t/030-uri-args.t b/t/030-uri-args.t index 951355e527..b9dba2b736 100644 --- a/t/030-uri-args.t +++ b/t/030-uri-args.t @@ -9,7 +9,7 @@ log_level('warn'); repeat_each(2); #repeat_each(1); -plan tests => repeat_each() * (blocks() * 2 + 24); +plan tests => repeat_each() * (blocks() * 2 + 22); no_root_location(); @@ -1568,10 +1568,9 @@ args: foo=%2C%24%40%7C%60&bar=-_.!~*'() } --- request GET /t ---- error_code: 500 ---- error_log -unsafe byte "0x9" in uri "/foo\x09bar" -attempt to set unsafe uri +--- error_code: 200 +--- response_body eval +qr#/foo\tbar# @@ -1586,10 +1585,9 @@ attempt to set unsafe uri } --- request GET /t ---- error_code: 500 ---- error_log -unsafe byte "0x0" in uri "\x00foo" -attempt to set unsafe uri +--- error_code: 200 +--- response_body eval +qr/\0foo/ diff --git a/t/162-static-module-location.t b/t/162-static-module-location.t new file mode 100644 index 0000000000..1dce330e77 --- /dev/null +++ b/t/162-static-module-location.t @@ -0,0 +1,95 @@ +use Test::Nginx::Socket::Lua; + +repeat_each(2); + +plan tests => repeat_each() * (blocks() * 3); + +our $HtmlDir = html_dir; + +no_long_string(); +run_tests(); + +__DATA__ + +=== TEST 1: decoded url contains '\0' and '\r\n' +--- config + server_tokens off; + location = /t { + rewrite_by_lua_block { + ngx.req.read_body(); + local args, _ = ngx.req.get_post_args(); + ngx.req.set_uri(args["url"], true); + } + } +--- request +POST /t +url=%00%0a%0dset-cookie:1234567 +--- error_code: 301 +--- response_headers +Location: %00%0A%0Dset-cookie:1234567/ +--- response_body_like +.*301 Moved Permanently.* + + + +=== TEST 2: uri contain chinese characters +--- config + server_tokens off; +--- user_files +>>> t/中文/foo.txt +Hello, world +--- request +GET /t/中文 +--- error_code: 301 +--- response_headers +Location: http://localhost:1984/t/%E4%B8%AD%E6%96%87/ +--- response_body_like +.*301 Moved Permanently.* + + + +=== TEST 3: uri contain chinese characters with args +--- config + server_tokens off; +--- user_files +>>> t/中文/foo.txt +Hello, world +--- request +GET /t/中文?q=name +--- error_code: 301 +--- response_headers +Location: http://localhost:1984/t/%E4%B8%AD%E6%96%87/?q=name +--- response_body_like +.*301 Moved Permanently.* + + + +=== TEST 4: uri already encoded +--- config + server_tokens off; +--- user_files +>>> t/中文/foo.txt +Hello, world +--- request +GET /t/%E4%B8%AD%E6%96%87 +--- error_code: 301 +--- response_headers +Location: http://localhost:1984/t/%E4%B8%AD%E6%96%87/ +--- response_body_like +.*301 Moved Permanently.* + + + +=== TEST 5: uri already encoded with args +--- config + server_tokens off; +--- user_files +>>> t/中文/foo.txt +Hello, world +--- request +GET /t/%E4%B8%AD%E6%96%87?q=name +--- error_code: 301 +--- response_headers +Location: http://localhost:1984/t/%E4%B8%AD%E6%96%87/?q=name +--- response_body_like +.*301 Moved Permanently.* From 2a6c9a63e2ef4720509408f9893e8b8aefd0f223 Mon Sep 17 00:00:00 2001 From: "Yichun Zhang (agentzh)" Date: Sat, 27 Jun 2020 17:05:44 -0700 Subject: [PATCH 10/19] doc: minor typo fixes. --- doc/HttpLuaModule.wiki | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/HttpLuaModule.wiki b/doc/HttpLuaModule.wiki index 6f99701605..3f44adfa3e 100644 --- a/doc/HttpLuaModule.wiki +++ b/doc/HttpLuaModule.wiki @@ -3791,9 +3791,9 @@ or a Lua table holding the query arguments' key-value pairs, as in ngx.req.set_uri_args({ a = 3, b = "hello world" }) -In the former case, i.e., when the whole query-strng is provided directly, +In the former case, i.e., when the whole query-string is provided directly, the input Lua string should already be well-formed with the URI encoding. -For security considerations, his method will autoamticaly escape any control and +For security considerations, his method will automatically escape any control and whitespace characters (ASCII code 0x00 ~ 0x32 and 0x7F) in the Lua string. In the latter case, this method will escape argument keys and values according to the URI escaping rule. From be35318272782313e11674304c91d7657742ee41 Mon Sep 17 00:00:00 2001 From: "Yichun Zhang (agentzh)" Date: Sat, 27 Jun 2020 17:09:26 -0700 Subject: [PATCH 11/19] doc: more typo fixes. --- doc/HttpLuaModule.wiki | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/HttpLuaModule.wiki b/doc/HttpLuaModule.wiki index 3f44adfa3e..7a624d4031 100644 --- a/doc/HttpLuaModule.wiki +++ b/doc/HttpLuaModule.wiki @@ -3793,7 +3793,7 @@ or a Lua table holding the query arguments' key-value pairs, as in In the former case, i.e., when the whole query-string is provided directly, the input Lua string should already be well-formed with the URI encoding. -For security considerations, his method will automatically escape any control and +For security considerations, this method will automatically escape any control and whitespace characters (ASCII code 0x00 ~ 0x32 and 0x7F) in the Lua string. In the latter case, this method will escape argument keys and values according to the URI escaping rule. From e1e335fd7f72bf2f1dad20800c33f79ff3e0d1c4 Mon Sep 17 00:00:00 2001 From: lijunlong Date: Sun, 7 Jun 2020 10:17:41 +0800 Subject: [PATCH 12/19] security: ngx.req.set_header(): now we always escape bytes in header names and header values which are prohibited by RFC 7230. See https://tools.ietf.org/html/rfc7230#section-3.2.6 for more details. Signed-off-by: Yichun Zhang (agentzh) --- README.markdown | 9 +++- doc/HttpLuaModule.wiki | 5 ++ src/ngx_http_lua_headers_in.c | 14 +++--- src/ngx_http_lua_headers_out.c | 10 ++-- src/ngx_http_lua_util.c | 80 +++++++++++++++++++++++++++++- src/ngx_http_lua_util.h | 6 +++ t/016-resp-header.t | 57 +++++++++------------ t/028-req-header.t | 91 +++++++++++++++++++++++++++------- t/030-uri-args.t | 8 +-- t/113-req-header-cookie.t | 13 ++--- 10 files changed, 219 insertions(+), 74 deletions(-) diff --git a/README.markdown b/README.markdown index c822b3d498..9aa0e4c4e3 100644 --- a/README.markdown +++ b/README.markdown @@ -4552,9 +4552,9 @@ or a Lua table holding the query arguments' key-value pairs, as in ngx.req.set_uri_args({ a = 3, b = "hello world" }) ``` -In the former case, i.e., when the whole query-strng is provided directly, +In the former case, i.e., when the whole query-string is provided directly, the input Lua string should already be well-formed with the URI encoding. -For security considerations, his method will autoamticaly escape any control and +For security considerations, this method will automatically escape any control and whitespace characters (ASCII code 0x00 ~ 0x32 and 0x7F) in the Lua string. In the latter case, this method will escape argument keys and values according to the URI escaping rule. @@ -4883,6 +4883,11 @@ ngx.req.set_header Set the current request's request header named `header_name` to value `header_value`, overriding any existing ones. +The input Lua string `header_name` and `header_value` should already be well-formed with the URI encoding. +For security considerations, this method will automatically escape " ", """, "(", ")", ",", "/", ":", ";", "?", +"<", "=", ">", "?", "@", "[", "]", "\", "{", "}", 0x00-0x1F, 0x7F-0xFF in `header_name` and automatically escape +"0x00-0x08, 0x0A-0x0F, 0x7F in `header_value`. + By default, all the subrequests subsequently initiated by [ngx.location.capture](#ngxlocationcapture) and [ngx.location.capture_multi](#ngxlocationcapture_multi) will inherit the new header. Here is an example of setting the `Content-Type` header: diff --git a/doc/HttpLuaModule.wiki b/doc/HttpLuaModule.wiki index 7a624d4031..bfacf86cb0 100644 --- a/doc/HttpLuaModule.wiki +++ b/doc/HttpLuaModule.wiki @@ -4084,6 +4084,11 @@ The __index metamethod will not be added when the raw Set the current request's request header named header_name to value header_value, overriding any existing ones. +The input Lua string `header_name` and `header_value` should already be well-formed with the URI encoding. +For security considerations, this method will automatically escape " ", """, "(", ")", ",", "/", ":", ";", "?", +"<", "=", ">", "?", "@", "[", "]", "\", "{", "}", 0x00-0x1F, 0x7F-0xFF in `header_name` and automatically escape +"0x00-0x08, 0x0A-0x0F, 0x7F in `header_value`. + By default, all the subrequests subsequently initiated by [[#ngx.location.capture|ngx.location.capture]] and [[#ngx.location.capture_multi|ngx.location.capture_multi]] will inherit the new header. Here is an example of setting the Content-Type header: diff --git a/src/ngx_http_lua_headers_in.c b/src/ngx_http_lua_headers_in.c index 34f2fb667f..713818efc2 100644 --- a/src/ngx_http_lua_headers_in.c +++ b/src/ngx_http_lua_headers_in.c @@ -653,16 +653,18 @@ ngx_http_lua_set_input_header(ngx_http_request_t *r, ngx_str_t key, { ngx_http_lua_header_val_t hv; ngx_http_lua_set_header_t *handlers = ngx_http_lua_set_handlers; - + ngx_int_t rc; ngx_uint_t i; dd("set header value: %.*s", (int) value.len, value.data); - if (ngx_http_lua_check_unsafe_string(r, key.data, key.len, - "header name") != NGX_OK - || ngx_http_lua_check_unsafe_string(r, value.data, value.len, - "header value") != NGX_OK) - { + rc = ngx_http_lua_copy_escaped_header(r, &key, 1); + if (rc != NGX_OK) { + return NGX_ERROR; + } + + rc = ngx_http_lua_copy_escaped_header(r, &value, 0); + if (rc != NGX_OK) { return NGX_ERROR; } diff --git a/src/ngx_http_lua_headers_out.c b/src/ngx_http_lua_headers_out.c index a0a9012643..a3b0f9e90e 100644 --- a/src/ngx_http_lua_headers_out.c +++ b/src/ngx_http_lua_headers_out.c @@ -491,11 +491,11 @@ ngx_http_lua_set_output_header(ngx_http_request_t *r, ngx_http_lua_ctx_t *ctx, dd("set header value: %.*s", (int) value.len, value.data); - if (ngx_http_lua_check_unsafe_string(r, key.data, key.len, - "header name") != NGX_OK - || ngx_http_lua_check_unsafe_string(r, value.data, value.len, - "header value") != NGX_OK) - { + if (ngx_http_lua_copy_escaped_header(r, &key, 1) != NGX_OK) { + return NGX_ERROR; + } + + if (ngx_http_lua_copy_escaped_header(r, &value, 0) != NGX_OK) { return NGX_ERROR; } diff --git a/src/ngx_http_lua_util.c b/src/ngx_http_lua_util.c index 983f476bb4..0154796b34 100644 --- a/src/ngx_http_lua_util.c +++ b/src/ngx_http_lua_util.c @@ -2008,8 +2008,52 @@ ngx_http_lua_escape_uri(u_char *dst, u_char *src, size_t size, ngx_uint_t type) /* mail_auth is the same as memcached */ + /* " ", """, "(", ")", ",", "/", ":", ";", "?", + * "<", "=", ">", "?", "@", "[", "]", "\", "{", + * "}", %00-%1F, %7F-%FF + */ + + static uint32_t header_name[] = { + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + + /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */ + 0xfc009305, /* 1111 1100 0000 0000 1001 0011 0000 0101 */ + + /* _^]\ [ZYX WVUT SRQP ONML KJIH GFED CBA@ */ + 0x38000001, /* 0011 1000 0000 0000 0000 0000 0000 0001 */ + + /* ~}| {zyx wvut srqp onml kjih gfed cba` */ + 0xa8000000, /* 1010 1000 0000 0000 0000 0000 0000 0000 */ + + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ + }; + + /* "%00-%08, %0A-%0F, %7F */ + + static uint32_t header_value[] = { + 0xfffffdff, /* 1111 1111 1111 1111 1111 1101 1111 1111 */ + + /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */ + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + + /* _^]\ [ZYX WVUT SRQP ONML KJIH GFED CBA@ */ + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + + /* ~}| {zyx wvut srqp onml kjih gfed cba` */ + 0x80000000, /* 1000 0000 0000 0000 0000 0000 0000 0000 */ + + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ + }; + static uint32_t *map[] = - { uri, args, uri_component, html, refresh, memcached, memcached }; + { uri, args, uri_component, html, refresh, memcached, memcached, + header_name, header_value }; escape = map[type]; @@ -4328,4 +4372,38 @@ ngx_http_lua_escape_log(u_char *dst, u_char *src, size_t size) } +ngx_int_t +ngx_http_lua_copy_escaped_header(ngx_http_request_t *r, + ngx_str_t *dst, int is_name) +{ + size_t escape; + size_t len; + u_char *data; + int type; + + type = is_name + ? NGX_HTTP_LUA_ESCAPE_HEADER_NAME : NGX_HTTP_LUA_ESCAPE_HEADER_VALUE; + + data = dst->data; + len = dst->len; + + escape = ngx_http_lua_escape_uri(NULL, data, len, type); + if (escape > 0) { + /* + * we allocate space for the trailling '\0' char here because nginx + * header values must be null-terminated + */ + dst->data = ngx_palloc(r->pool, len + 2 * escape + 1); + if (dst->data == NULL) { + return NGX_ERROR; + } + + ngx_http_lua_escape_uri(dst->data, data, len, type); + dst->len = len + 2 * escape; + dst->data[dst->len] = '\0'; + } + + return NGX_OK; +} + /* vi:set ft=c ts=4 sw=4 et fdm=marker: */ diff --git a/src/ngx_http_lua_util.h b/src/ngx_http_lua_util.h index a0ab2de357..92f3ba4737 100644 --- a/src/ngx_http_lua_util.h +++ b/src/ngx_http_lua_util.h @@ -27,6 +27,9 @@ # define NGX_HTTP_SWITCHING_PROTOCOLS 101 #endif +#define NGX_HTTP_LUA_ESCAPE_HEADER_NAME 7 + +#define NGX_HTTP_LUA_ESCAPE_HEADER_VALUE 8 /* key in Lua vm registry for all the "ngx.ctx" tables */ #define ngx_http_lua_ctx_tables_key "ngx_lua_ctx_tables" @@ -169,6 +172,9 @@ void ngx_http_lua_unescape_uri(u_char **dst, u_char **src, size_t size, uintptr_t ngx_http_lua_escape_uri(u_char *dst, u_char *src, size_t size, ngx_uint_t type); +ngx_int_t ngx_http_lua_copy_escaped_header(ngx_http_request_t *r, + ngx_str_t *dst, int is_name); + void ngx_http_lua_inject_req_api(ngx_log_t *log, lua_State *L); void ngx_http_lua_process_args_option(ngx_http_request_t *r, diff --git a/t/016-resp-header.t b/t/016-resp-header.t index 3d38d8f21b..febd276fef 100644 --- a/t/016-resp-header.t +++ b/t/016-resp-header.t @@ -8,7 +8,7 @@ use Test::Nginx::Socket::Lua; repeat_each(2); -plan tests => repeat_each() * (blocks() * 3 + 80); +plan tests => repeat_each() * (blocks() * 3 + 77); #no_diff(); no_long_string(); @@ -51,7 +51,7 @@ Content-Type: text/html -=== TEST 3: set response content-type header +=== TEST 3: set response content-length header --- config location /read { content_by_lua ' @@ -1976,14 +1976,12 @@ Content-Type: application/json } --- request GET /t ---- error_code: 500 --- response_headers -header: +header: value%0Dfoo:bar%0Abar:foo foo: bar: ---- error_log -unsafe byte "0xd" in header value "value\x0Dfoo:bar\x0Abar:foo" -failed to set header +--- no_error_log +[error] @@ -1997,14 +1995,12 @@ failed to set header } --- request GET /t ---- error_code: 500 --- response_headers -header: +header: value%0Afoo:bar%0Dbar:foo foo: bar: ---- error_log -unsafe byte "0xa" in header value "value\x0Afoo:bar\x0Dbar:foo" -failed to set header +--- no_error_log +[error] @@ -2018,14 +2014,13 @@ failed to set header } --- request GET /t ---- error_code: 500 --- response_headers +header%3A%20value%0Dfoo%3Abar%0Abar%3Afoo: xx header: foo: bar: ---- error_log -unsafe byte "0xd" in header name "header: value\x0Dfoo:bar\x0Abar:foo" -failed to set header +--- no_error_log +[error] @@ -2039,14 +2034,13 @@ failed to set header } --- request GET /t ---- error_code: 500 --- response_headers +header%3A%20value%0Afoo%3Abar%0Dbar%3Afoo: xx header: foo: bar: ---- error_log -unsafe byte "0xa" in header name "header: value\x0Afoo:bar\x0Dbar:foo" -failed to set header +--- no_error_log +[error] @@ -2060,14 +2054,13 @@ failed to set header } --- request GET /t ---- error_code: 500 --- response_headers +%0Dheader%3A%20value%0Dfoo%3Abar%0Abar%3Afoo: xx header: foo: bar: ---- error_log -unsafe byte "0xd" in header name "\x0Dheader: value\x0Dfoo:bar\x0Abar:foo" -failed to set header +--- no_error_log +[error] @@ -2081,14 +2074,13 @@ failed to set header } --- request GET /t ---- error_code: 500 --- response_headers +%0Aheader%3A%20value%0Afoo%3Abar%0Dbar%3Afoo: xx header: foo: bar: ---- error_log -unsafe byte "0xa" in header name "\x0Aheader: value\x0Afoo:bar\x0Dbar:foo" -failed to set header +--- no_error_log +[error] @@ -2105,11 +2097,10 @@ failed to set header } --- request GET /t ---- error_code: 500 --- response_headers -foo: xx: xxx: ---- error_log -unsafe byte "0xa" in header value "foo\x0Axx:bar" -failed to set header +--- raw_response_headers_like chomp +foo: foo%0Axx:bar\r\nfoo: bar%0Dxxx:foo\r\n +--- no_error_log +[error] diff --git a/t/028-req-header.t b/t/028-req-header.t index fbd4e9eac9..c2c58eb52e 100644 --- a/t/028-req-header.t +++ b/t/028-req-header.t @@ -2044,10 +2044,10 @@ new } --- request GET /req-header ---- error_code: 500 ---- error_log -unsafe byte "0xd" in header name "Foo\x0Dfoo" -failed to set header +--- response_body +Foo: +--- no_error_log +[error] @@ -2062,10 +2062,10 @@ failed to set header } --- request GET /req-header ---- error_code: 500 ---- error_log -unsafe byte "0xa" in header value "new\x0Avalue" -failed to set header +--- response_body +Foo: new%0Avalue +--- no_error_log +[error] @@ -2082,10 +2082,10 @@ failed to set header } --- request GET /req-header ---- error_code: 500 ---- error_log -unsafe byte "0xa" in header value "new\x0Avalue" -failed to set header +--- response_body +new%0Avalue, foo bar. +--- no_error_log +[error] @@ -2097,12 +2097,69 @@ failed to set header } content_by_lua_block { - ngx.say(table.concat(ngx.req.get_headers()["foo"], ", "), ".") + ngx.say(ngx.req.get_headers()["foo"]) } } --- request GET /req-header ---- error_code: 500 ---- error_log -unsafe byte "0xa" in header value "\x22new\x0Avalue\x5C\x22" -failed to set header +--- response_body +"new%0Avalue\" +--- no_error_log +[error] + + + +=== TEST 66: add request headers with '\r\n' +--- config + location /bar { + access_by_lua_block { + ngx.req.set_header("Foo\r", "123\r\n") + } + proxy_pass http://127.0.0.1:$server_port/foo; + } + + location = /foo { + echo $echo_client_request_headers; + } +--- request +GET /bar +--- response_body_like chomp +\bFoo%0D: 123%0D%0A\b + + + +=== TEST 67: add request headers with '\0' +--- config + location /bar { + access_by_lua_block { + ngx.req.set_header("Foo", "\0") + } + proxy_pass http://127.0.0.1:$server_port/foo; + } + + location = /foo { + echo $echo_client_request_headers; + } +--- request +GET /bar +--- response_body_like chomp +\bFoo: %00\b + + + +=== TEST 68: add request headers with '中文' +--- config + location /bar { + access_by_lua_block { + ngx.req.set_header("Foo中文", "ab中文a") + } + proxy_pass http://127.0.0.1:$server_port/foo; + } + + location = /foo { + echo $echo_client_request_headers; + } +--- request +GET /bar +--- response_body_like chomp +\bFoo%E4%B8%AD%E6%96%87: ab中文a\r\n diff --git a/t/030-uri-args.t b/t/030-uri-args.t index b9dba2b736..d4cb6d17ef 100644 --- a/t/030-uri-args.t +++ b/t/030-uri-args.t @@ -9,7 +9,7 @@ log_level('warn'); repeat_each(2); #repeat_each(1); -plan tests => repeat_each() * (blocks() * 2 + 22); +plan tests => repeat_each() * (blocks() * 2 + 21); no_root_location(); @@ -1568,9 +1568,9 @@ args: foo=%2C%24%40%7C%60&bar=-_.!~*'() } --- request GET /t ---- error_code: 200 ---- response_body eval -qr#/foo\tbar# +--- response +/foo bar +--- no_error_log diff --git a/t/113-req-header-cookie.t b/t/113-req-header-cookie.t index 3baa899c3b..944549cd74 100644 --- a/t/113-req-header-cookie.t +++ b/t/113-req-header-cookie.t @@ -8,7 +8,7 @@ use Test::Nginx::Socket::Lua; repeat_each(2); -plan tests => repeat_each() * (4 * blocks()); +plan tests => repeat_each() * (3 * blocks() + 6); #no_diff(); no_long_string(); @@ -263,9 +263,10 @@ Cookie: boo=123; foo=bar } --- request GET /t ---- error_code: 500 ---- error_log -unsafe byte "0xa" in header value "boo=123\x0Afoo" -failed to set header +--- response_body +Cookie foo: bar%0Dbar +Cookie baz: +Cookie boo: 123%0Afoo +Cookie: boo=123%0Afoo; foo=bar%0Dbar --- no_error_log -[crit] +[error] From 7cecbec5d416d18f51b33bb820e908c5966f78be Mon Sep 17 00:00:00 2001 From: lijunlong Date: Tue, 19 May 2020 10:56:31 +0800 Subject: [PATCH 13/19] add not_component argument for ngx.escape_uri --- README.markdown | 4 +++- src/ngx_http_lua_string.c | 13 ++++++++----- src/ngx_http_lua_util.c | 2 +- t/006-escape.t | 22 +++++++++++++++++++++- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/README.markdown b/README.markdown index 9aa0e4c4e3..c091cea455 100644 --- a/README.markdown +++ b/README.markdown @@ -5581,11 +5581,13 @@ This method was introduced in the `0.5.0rc30` release. ngx.escape_uri -------------- -**syntax:** *newstr = ngx.escape_uri(str)* +**syntax:** *newstr = ngx.escape_uri(str, not_component?)* **context:** *init_by_lua*, init_worker_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua** Escape `str` as a URI component. +Since the `v0.10.16rc6` release, this function accepts an optional boolean `not_component` argument. When this argument is `true`, this function act like encodeURI. These characters bellow will not be escape. +a-zA-Z0-9-_.~!*'();:@&=+$,/?# [Back to TOC](#nginx-api-for-lua) diff --git a/src/ngx_http_lua_string.c b/src/ngx_http_lua_string.c index 1bd4f2d99c..3c8989230f 100644 --- a/src/ngx_http_lua_string.c +++ b/src/ngx_http_lua_string.c @@ -433,17 +433,20 @@ ngx_http_lua_ffi_unescape_uri(const u_char *src, size_t len, u_char *dst) size_t -ngx_http_lua_ffi_uri_escaped_length(const u_char *src, size_t len) +ngx_http_lua_ffi_uri_escaped_length(const u_char *src, size_t len, + int not_component) { - return len + 2 * ngx_http_lua_escape_uri(NULL, (u_char *) src, len, - NGX_ESCAPE_URI_COMPONENT); + int type = not_component ? NGX_ESCAPE_URI : NGX_ESCAPE_URI_COMPONENT; + return len + 2 * ngx_http_lua_escape_uri(NULL, (u_char *) src, len, type); } void -ngx_http_lua_ffi_escape_uri(const u_char *src, size_t len, u_char *dst) +ngx_http_lua_ffi_escape_uri(const u_char *src, size_t len, u_char *dst, + int not_component) { - ngx_http_lua_escape_uri(dst, (u_char *) src, len, NGX_ESCAPE_URI_COMPONENT); + int type = not_component ? NGX_ESCAPE_URI : NGX_ESCAPE_URI_COMPONENT; + ngx_http_lua_escape_uri(dst, (u_char *) src, len, type); } diff --git a/src/ngx_http_lua_util.c b/src/ngx_http_lua_util.c index 0154796b34..06463b20bc 100644 --- a/src/ngx_http_lua_util.c +++ b/src/ngx_http_lua_util.c @@ -1892,7 +1892,7 @@ ngx_http_lua_escape_uri(u_char *dst, u_char *src, size_t size, ngx_uint_t type) 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */ - 0xfc00886d, /* 1111 1100 0000 0000 1000 1000 0110 1101 */ + 0x50000051, /* 0101 0000 0000 0000 0000 0000 0010 0101 */ /* _^]\ [ZYX WVUT SRQP ONML KJIH GFED CBA@ */ 0x78000000, /* 0111 1000 0000 0000 0000 0000 0000 0000 */ diff --git a/t/006-escape.t b/t/006-escape.t index a21d3cb509..cfbb137712 100644 --- a/t/006-escape.t +++ b/t/006-escape.t @@ -4,7 +4,7 @@ use Test::Nginx::Socket::Lua; repeat_each(2); -plan tests => repeat_each() * (blocks() * 2 + 2); +plan tests => repeat_each() * (blocks() * 2 + 3); no_long_string(); @@ -197,3 +197,23 @@ GET /lua %2C%24%40%7C%60 --- no_error_log [error] + + + +=== TEST 15: not component argument +--- config + location /lua { + content_by_lua_block { + ngx.say(ngx.escape_uri("https://www.google.com", true)) + ngx.say(ngx.escape_uri("https://www.google.com/query?q=test", true)) + ngx.say(ngx.escape_uri("https://www.google.com/query?\r\nq=test", true)) + } + } +--- request +GET /lua +--- response_body +https://www.google.com +https://www.google.com/query?q=test +https://www.google.com/query?%0D%0Aq=test +--- no_error_log +[error] From 885a1ed74b84777ecb76d6854dd0db41437412e2 Mon Sep 17 00:00:00 2001 From: lijunlong Date: Tue, 19 May 2020 17:36:26 +0800 Subject: [PATCH 14/19] add not_component argument for ngx.escape_uri --- README.markdown | 13 ++++++++++--- doc/HttpLuaModule.wiki | 9 +++++++++ src/ngx_http_lua_string.c | 4 ++-- src/ngx_http_lua_util.c | 2 +- t/006-escape.t | 4 ++++ 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/README.markdown b/README.markdown index c091cea455..7a9ac5ca7f 100644 --- a/README.markdown +++ b/README.markdown @@ -5581,13 +5581,20 @@ This method was introduced in the `0.5.0rc30` release. ngx.escape_uri -------------- -**syntax:** *newstr = ngx.escape_uri(str, not_component?)* +**syntax:** *newstr = ngx.escape_uri(str)* **context:** *init_by_lua*, init_worker_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua** Escape `str` as a URI component. -Since the `v0.10.16rc6` release, this function accepts an optional boolean `not_component` argument. When this argument is `true`, this function act like encodeURI. These characters bellow will not be escape. -a-zA-Z0-9-_.~!*'();:@&=+$,/?# + +Since `v0.10.16rc6`, this function accepts an optional boolean `not_component` argument. When this argument is `true`, these characters bellow will not be escaped. + + + Alphabets: a-zA-Z + Digits: 0-9 + Reserve characters: -_.~!*'();:@&=+$,/?# + + [Back to TOC](#nginx-api-for-lua) diff --git a/doc/HttpLuaModule.wiki b/doc/HttpLuaModule.wiki index bfacf86cb0..fe75105af9 100644 --- a/doc/HttpLuaModule.wiki +++ b/doc/HttpLuaModule.wiki @@ -4692,6 +4692,15 @@ This method was introduced in the 0.5.0rc30 release. Escape str as a URI component. +Since `v0.10.16rc6`, this function accepts an optional boolean not_component argument. When this argument is true, these characters bellow will not be escaped. + + +Alphabets: a-zA-Z +Digits: 0-9 +Reserve characters: -_.~!*'();:@&=+$,/?# + + + == ngx.unescape_uri == '''syntax:''' ''newstr = ngx.unescape_uri(str)'' diff --git a/src/ngx_http_lua_string.c b/src/ngx_http_lua_string.c index 3c8989230f..f15fa3d526 100644 --- a/src/ngx_http_lua_string.c +++ b/src/ngx_http_lua_string.c @@ -433,7 +433,7 @@ ngx_http_lua_ffi_unescape_uri(const u_char *src, size_t len, u_char *dst) size_t -ngx_http_lua_ffi_uri_escaped_length(const u_char *src, size_t len, +ngx_http_lua_ffi_uri_escaped_length(const u_char *src, size_t len, int not_component) { int type = not_component ? NGX_ESCAPE_URI : NGX_ESCAPE_URI_COMPONENT; @@ -442,7 +442,7 @@ ngx_http_lua_ffi_uri_escaped_length(const u_char *src, size_t len, void -ngx_http_lua_ffi_escape_uri(const u_char *src, size_t len, u_char *dst, +ngx_http_lua_ffi_escape_uri(const u_char *src, size_t len, u_char *dst, int not_component) { int type = not_component ? NGX_ESCAPE_URI : NGX_ESCAPE_URI_COMPONENT; diff --git a/src/ngx_http_lua_util.c b/src/ngx_http_lua_util.c index 06463b20bc..2c031261c1 100644 --- a/src/ngx_http_lua_util.c +++ b/src/ngx_http_lua_util.c @@ -1892,7 +1892,7 @@ ngx_http_lua_escape_uri(u_char *dst, u_char *src, size_t size, ngx_uint_t type) 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */ - 0x50000051, /* 0101 0000 0000 0000 0000 0000 0010 0101 */ + 0x50000025, /* 0101 0000 0000 0000 0000 0000 0010 0101 */ /* _^]\ [ZYX WVUT SRQP ONML KJIH GFED CBA@ */ 0x78000000, /* 0111 1000 0000 0000 0000 0000 0000 0000 */ diff --git a/t/006-escape.t b/t/006-escape.t index cfbb137712..0b7af8070a 100644 --- a/t/006-escape.t +++ b/t/006-escape.t @@ -207,6 +207,8 @@ GET /lua ngx.say(ngx.escape_uri("https://www.google.com", true)) ngx.say(ngx.escape_uri("https://www.google.com/query?q=test", true)) ngx.say(ngx.escape_uri("https://www.google.com/query?\r\nq=test", true)) + ngx.say(ngx.escape_uri("-_.~!*'();:@&=+$,/?#", true)) + ngx.say(ngx.escape_uri("<>[]{}\\\" ", true)) } } --- request @@ -215,5 +217,7 @@ GET /lua https://www.google.com https://www.google.com/query?q=test https://www.google.com/query?%0D%0Aq=test +-_.~!*'();:@&=+$,/?# +%3C%3E%5B%5D%7B%7D%5C%22%20 --- no_error_log [error] From 7fac7ebd4a9e5db60a0c074656c10bc7b2c3587c Mon Sep 17 00:00:00 2001 From: lijunlong Date: Fri, 26 Jun 2020 20:39:52 +0800 Subject: [PATCH 15/19] add not_uri_comp argument for ngx.escape_uri --- README.markdown | 10 ++++------ doc/HttpLuaModule.wiki | 10 ++++------ src/ngx_http_lua_consts.c | 28 ++++++++++++++++++++++++++++ src/ngx_http_lua_consts.h | 1 + src/ngx_http_lua_string.c | 6 ++---- src/ngx_http_lua_util.c | 7 ++++--- t/006-escape.t | 8 ++++---- 7 files changed, 47 insertions(+), 23 deletions(-) diff --git a/README.markdown b/README.markdown index 7a9ac5ca7f..c11806ceee 100644 --- a/README.markdown +++ b/README.markdown @@ -5581,19 +5581,17 @@ This method was introduced in the `0.5.0rc30` release. ngx.escape_uri -------------- -**syntax:** *newstr = ngx.escape_uri(str)* +**syntax:** *newstr = ngx.escape_uri(str, not_uri_comp?)* **context:** *init_by_lua*, init_worker_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua** Escape `str` as a URI component. -Since `v0.10.16rc6`, this function accepts an optional boolean `not_component` argument. When this argument is `true`, these characters bellow will not be escaped. +Since `v0.10.16rc6`, this function accepts an optional boolean `not_uri_comp` argument. When this argument is `true`, these characters bellow will be escaped. - Alphabets: a-zA-Z - Digits: 0-9 - Reserve characters: -_.~!*'();:@&=+$,/?# - + ASCII code: 0~32, 127~255 + Characters: ?%#! [Back to TOC](#nginx-api-for-lua) diff --git a/doc/HttpLuaModule.wiki b/doc/HttpLuaModule.wiki index fe75105af9..90f3e3dadc 100644 --- a/doc/HttpLuaModule.wiki +++ b/doc/HttpLuaModule.wiki @@ -4686,21 +4686,19 @@ This method was introduced in the 0.5.0rc30 release. == ngx.escape_uri == -'''syntax:''' ''newstr = ngx.escape_uri(str)'' +'''syntax:''' ''newstr = ngx.escape_uri(str, not_uri_comp?)'' '''context:''' ''init_by_lua*, init_worker_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua*'' Escape str as a URI component. -Since `v0.10.16rc6`, this function accepts an optional boolean not_component argument. When this argument is true, these characters bellow will not be escaped. +Since `v0.10.16rc6`, this function accepts an optional boolean not_uri_comp argument. When this argument is true, these characters bellow will be escaped. -Alphabets: a-zA-Z -Digits: 0-9 -Reserve characters: -_.~!*'();:@&=+$,/?# + ASCII code: 0~32, 127~255 + Characters: ?%#! - == ngx.unescape_uri == '''syntax:''' ''newstr = ngx.unescape_uri(str)'' diff --git a/src/ngx_http_lua_consts.c b/src/ngx_http_lua_consts.c index 47ba998576..6318c7d943 100644 --- a/src/ngx_http_lua_consts.c +++ b/src/ngx_http_lua_consts.c @@ -13,6 +13,34 @@ #include "ngx_http_lua_consts.h" +void +ngx_http_lua_inject_escape_consts(lua_State *L) +{ + /* {{{ escape constants */ + lua_pushinteger(L, NGX_ESCAPE_URI); + lua_setfield(L, -2, "NGX_ESCAPE_URI"); + + lua_pushinteger(L, NGX_ESCAPE_ARGS); + lua_setfield(L, -2, "NGX_ESCAPE_ARGS"); + + lua_pushinteger(L, NGX_ESCAPE_URI_COMPONENT); + lua_setfield(L, -2, "NGX_ESCAPE_URI_COMPONENT"); + + lua_pushinteger(L, NGX_ESCAPE_HTML); + lua_setfield(L, -2, "NGX_ESCAPE_HTML"); + + lua_pushinteger(L, NGX_ESCAPE_REFRESH); + lua_setfield(L, -2, "NGX_ESCAPE_REFRESH"); + + lua_pushinteger(L, NGX_ESCAPE_MEMCACHED); + lua_setfield(L, -2, "NGX_ESCAPE_MEMCACHED"); + + lua_pushinteger(L, NGX_ESCAPE_MAIL_AUTH); + lua_setfield(L, -2, "ERNGX_ESCAPE_MAIL_AUTHROR"); + + /* }}} */ +} + void ngx_http_lua_inject_core_consts(lua_State *L) diff --git a/src/ngx_http_lua_consts.h b/src/ngx_http_lua_consts.h index c403f81c4a..d0c3055107 100644 --- a/src/ngx_http_lua_consts.h +++ b/src/ngx_http_lua_consts.h @@ -13,6 +13,7 @@ void ngx_http_lua_inject_http_consts(lua_State *L); void ngx_http_lua_inject_core_consts(lua_State *L); +void ngx_http_lua_inject_escape_consts(lua_State *L); #endif /* _NGX_HTTP_LUA_CONSTS_H_INCLUDED_ */ diff --git a/src/ngx_http_lua_string.c b/src/ngx_http_lua_string.c index f15fa3d526..a94e181745 100644 --- a/src/ngx_http_lua_string.c +++ b/src/ngx_http_lua_string.c @@ -434,18 +434,16 @@ ngx_http_lua_ffi_unescape_uri(const u_char *src, size_t len, u_char *dst) size_t ngx_http_lua_ffi_uri_escaped_length(const u_char *src, size_t len, - int not_component) + int type) { - int type = not_component ? NGX_ESCAPE_URI : NGX_ESCAPE_URI_COMPONENT; return len + 2 * ngx_http_lua_escape_uri(NULL, (u_char *) src, len, type); } void ngx_http_lua_ffi_escape_uri(const u_char *src, size_t len, u_char *dst, - int not_component) + int type) { - int type = not_component ? NGX_ESCAPE_URI : NGX_ESCAPE_URI_COMPONENT; ngx_http_lua_escape_uri(dst, (u_char *) src, len, type); } diff --git a/src/ngx_http_lua_util.c b/src/ngx_http_lua_util.c index 2c031261c1..43669803a6 100644 --- a/src/ngx_http_lua_util.c +++ b/src/ngx_http_lua_util.c @@ -726,6 +726,7 @@ ngx_http_lua_inject_ngx_api(lua_State *L, ngx_http_lua_main_conf_t *lmcf, ngx_http_lua_inject_arg_api(L); + ngx_http_lua_inject_escape_consts(L); ngx_http_lua_inject_http_consts(L); ngx_http_lua_inject_core_consts(L); @@ -1892,13 +1893,13 @@ ngx_http_lua_escape_uri(u_char *dst, u_char *src, size_t size, ngx_uint_t type) 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ /* ?>=< ;:98 7654 3210 /.-, +*)( '&%$ #"! */ - 0x50000025, /* 0101 0000 0000 0000 0000 0000 0010 0101 */ + 0x80000029, /* 1000 0000 0000 0000 0000 0000 0010 1001 */ /* _^]\ [ZYX WVUT SRQP ONML KJIH GFED CBA@ */ - 0x78000000, /* 0111 1000 0000 0000 0000 0000 0000 0000 */ + 0x00000000, /* 0000 0000 0000 0000 0000 0000 0000 0000 */ /* ~}| {zyx wvut srqp onml kjih gfed cba` */ - 0xa8000000, /* 1010 1000 0000 0000 0000 0000 0000 0000 */ + 0x80000000, /* 1000 0000 0000 0000 0000 0000 0000 0000 */ 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ diff --git a/t/006-escape.t b/t/006-escape.t index 0b7af8070a..df5e45b2da 100644 --- a/t/006-escape.t +++ b/t/006-escape.t @@ -215,9 +215,9 @@ GET /lua GET /lua --- response_body https://www.google.com -https://www.google.com/query?q=test -https://www.google.com/query?%0D%0Aq=test --_.~!*'();:@&=+$,/?# -%3C%3E%5B%5D%7B%7D%5C%22%20 +https://www.google.com/query%3Fq=test +https://www.google.com/query%3F%0D%0Aq=test +-_.~!*'();:@&=+$,/%3F%23 +<>[]{}\"%20 --- no_error_log [error] From 3603a5b7b16ef6c88401eb7158bc94652eb20c00 Mon Sep 17 00:00:00 2001 From: lijunlong Date: Sat, 27 Jun 2020 10:29:22 +0800 Subject: [PATCH 16/19] feature: change from not_uri_comp to tyep --- README.markdown | 32 ++++++++++---- doc/HttpLuaModule.wiki | 27 ++++++++---- src/ngx_http_lua_consts.c | 14 +++---- t/006-escape.t | 87 +++++++++++++++++++++++++++++++++++---- 4 files changed, 132 insertions(+), 28 deletions(-) diff --git a/README.markdown b/README.markdown index c11806ceee..47eae680d5 100644 --- a/README.markdown +++ b/README.markdown @@ -3192,6 +3192,7 @@ Nginx API for Lua * [HTTP method constants](#http-method-constants) * [HTTP status constants](#http-status-constants) * [Nginx log level constants](#nginx-log-level-constants) +* [Nginx escape type constants](#nginx-escape-type-constants) * [print](#print) * [ngx.ctx](#ngxctx) * [ngx.location.capture](#ngxlocationcapture) @@ -3596,6 +3597,27 @@ These constants are usually used by the [ngx.log](#ngxlog) method. [Back to TOC](#nginx-api-for-lua) +Nginx escape type constants +--------------------------- + +**context:** *init_by_lua*, init_worker_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua** + +```lua + + ngx.ESCAPE_URI + ngx.ESCAPE_ARGS + ngx.ESCAPE_URI_COMPONENT + ngx.ESCAPE_HTML + ngx.ESCAPE_REFRESH + ngx.ESCAPE_MEMCACHED + ngx.ESCAPE_MAIL_AUTH +``` + + +These constants are usually used by the [ngx.escape_uri](#ngxescape_uri) method. + +[Back to TOC](#nginx-api-for-lua) + print ----- @@ -5581,18 +5603,14 @@ This method was introduced in the `0.5.0rc30` release. ngx.escape_uri -------------- -**syntax:** *newstr = ngx.escape_uri(str, not_uri_comp?)* +**syntax:** *newstr = ngx.escape_uri(str, type?)* **context:** *init_by_lua*, init_worker_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua** Escape `str` as a URI component. -Since `v0.10.16rc6`, this function accepts an optional boolean `not_uri_comp` argument. When this argument is `true`, these characters bellow will be escaped. - - - ASCII code: 0~32, 127~255 - Characters: ?%#! - +Since `v0.10.16rc6`, this function accepts an optional `type` argument. +Note that this method accepts all [Nginx escape type constants](#nginx-escape-type-constants) as input. [Back to TOC](#nginx-api-for-lua) diff --git a/doc/HttpLuaModule.wiki b/doc/HttpLuaModule.wiki index 90f3e3dadc..c18a31a116 100644 --- a/doc/HttpLuaModule.wiki +++ b/doc/HttpLuaModule.wiki @@ -2929,6 +2929,23 @@ These constants are usually used in [[#ngx.location.capture|ngx.location.capture These constants are usually used by the [[#ngx.log|ngx.log]] method. +== Nginx escape type constants == + +'''context:''' ''init_by_lua*, init_worker_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua*'' + + + ngx.ESCAPE_URI + ngx.ESCAPE_ARGS + ngx.ESCAPE_URI_COMPONENT + ngx.ESCAPE_HTML + ngx.ESCAPE_REFRESH + ngx.ESCAPE_MEMCACHED + ngx.ESCAPE_MAIL_AUTH + + + +These constants are usually used by the [[#ngx.escape_uri|ngx.escape_uri]] method. + == print == '''syntax:''' ''print(...)'' @@ -4686,18 +4703,14 @@ This method was introduced in the 0.5.0rc30 release. == ngx.escape_uri == -'''syntax:''' ''newstr = ngx.escape_uri(str, not_uri_comp?)'' +'''syntax:''' ''newstr = ngx.escape_uri(str, type?)'' '''context:''' ''init_by_lua*, init_worker_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua*'' Escape str as a URI component. -Since `v0.10.16rc6`, this function accepts an optional boolean not_uri_comp argument. When this argument is true, these characters bellow will be escaped. - - - ASCII code: 0~32, 127~255 - Characters: ?%#! - +Since `v0.10.16rc6`, this function accepts an optional type argument. +Note that this method accepts all [[#Nginx escape type constants|Nginx escape type constants]] as input. == ngx.unescape_uri == diff --git a/src/ngx_http_lua_consts.c b/src/ngx_http_lua_consts.c index 6318c7d943..38b3d6e9a0 100644 --- a/src/ngx_http_lua_consts.c +++ b/src/ngx_http_lua_consts.c @@ -18,25 +18,25 @@ ngx_http_lua_inject_escape_consts(lua_State *L) { /* {{{ escape constants */ lua_pushinteger(L, NGX_ESCAPE_URI); - lua_setfield(L, -2, "NGX_ESCAPE_URI"); + lua_setfield(L, -2, "ESCAPE_URI"); lua_pushinteger(L, NGX_ESCAPE_ARGS); - lua_setfield(L, -2, "NGX_ESCAPE_ARGS"); + lua_setfield(L, -2, "ESCAPE_ARGS"); lua_pushinteger(L, NGX_ESCAPE_URI_COMPONENT); - lua_setfield(L, -2, "NGX_ESCAPE_URI_COMPONENT"); + lua_setfield(L, -2, "ESCAPE_URI_COMPONENT"); lua_pushinteger(L, NGX_ESCAPE_HTML); - lua_setfield(L, -2, "NGX_ESCAPE_HTML"); + lua_setfield(L, -2, "ESCAPE_HTML"); lua_pushinteger(L, NGX_ESCAPE_REFRESH); - lua_setfield(L, -2, "NGX_ESCAPE_REFRESH"); + lua_setfield(L, -2, "ESCAPE_REFRESH"); lua_pushinteger(L, NGX_ESCAPE_MEMCACHED); - lua_setfield(L, -2, "NGX_ESCAPE_MEMCACHED"); + lua_setfield(L, -2, "ESCAPE_MEMCACHED"); lua_pushinteger(L, NGX_ESCAPE_MAIL_AUTH); - lua_setfield(L, -2, "ERNGX_ESCAPE_MAIL_AUTHROR"); + lua_setfield(L, -2, "ESCAPE_MAIL_AUTH"); /* }}} */ } diff --git a/t/006-escape.t b/t/006-escape.t index df5e45b2da..49d5d3df86 100644 --- a/t/006-escape.t +++ b/t/006-escape.t @@ -4,7 +4,7 @@ use Test::Nginx::Socket::Lua; repeat_each(2); -plan tests => repeat_each() * (blocks() * 2 + 3); +plan tests => repeat_each() * (blocks() * 2 + 4); no_long_string(); @@ -200,15 +200,15 @@ GET /lua -=== TEST 15: not component argument +=== TEST 15: escape type argument --- config location /lua { content_by_lua_block { - ngx.say(ngx.escape_uri("https://www.google.com", true)) - ngx.say(ngx.escape_uri("https://www.google.com/query?q=test", true)) - ngx.say(ngx.escape_uri("https://www.google.com/query?\r\nq=test", true)) - ngx.say(ngx.escape_uri("-_.~!*'();:@&=+$,/?#", true)) - ngx.say(ngx.escape_uri("<>[]{}\\\" ", true)) + ngx.say(ngx.escape_uri("https://www.google.com", ngx.ESCAPE_URI)) + ngx.say(ngx.escape_uri("https://www.google.com/query?q=test", ngx.ESCAPE_URI)) + ngx.say(ngx.escape_uri("https://www.google.com/query?\r\nq=test", ngx.ESCAPE_URI)) + ngx.say(ngx.escape_uri("-_.~!*'();:@&=+$,/?#", ngx.ESCAPE_URI)) + ngx.say(ngx.escape_uri("<>[]{}\\\" ", ngx.ESCAPE_URI)) } } --- request @@ -221,3 +221,76 @@ https://www.google.com/query%3F%0D%0Aq=test <>[]{}\"%20 --- no_error_log [error] + + + +=== TEST 16: escape type argument +--- config + location /lua { + content_by_lua_block { + ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", ngx.ESCAPE_URI)) + ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", ngx.ESCAPE_ARGS)) + ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", ngx.ESCAPE_URI_COMPONENT)) + ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", ngx.ESCAPE_HTML)) + ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", ngx.ESCAPE_REFRESH)) + ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", ngx.ESCAPE_MEMCACHED)) + ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", ngx.ESCAPE_MAIL_AUTH)) + } + } +--- request +GET /lua +--- response_body +https://www.google.com/%3Ft=abc@%20: +https://www.google.com/%3Ft=abc@%20: +https%3A%2F%2Fwww.google.com%2F%3Ft%3Dabc%40%20%3A +https://www.google.com/?t=abc@%20: +https://www.google.com/?t=abc@%20: +https://www.google.com/?t=abc@%20: +https://www.google.com/?t=abc@%20: +--- no_error_log +[error] + + + +=== TEST 17: escape type out of range +--- config + location /lua { + content_by_lua_block { + ngx.say(ngx.escape_uri("https://www.google.com", -1)) + } + } +--- request +GET /lua +--- error_code: 500 +--- error_log eval +qr/\[error\] \d+#\d+: \*\d+ lua entry thread aborted: runtime error: "esc_type" \-1 out of range/ + + + +=== TEST 18: escape type out of range +--- config + location /lua { + content_by_lua_block { + ngx.say(ngx.escape_uri("https://www.google.com", 100)) + } + } +--- request +GET /lua +--- error_code: 500 +--- error_log eval +qr/\[error\] \d+#\d+: \*\d+ lua entry thread aborted: runtime error: "esc_type" 100 out of range/ + + + +=== TEST 19: escape type not integer +--- config + location /lua { + content_by_lua_block { + ngx.say(ngx.escape_uri("https://www.google.com", true)) + } + } +--- request +GET /lua +--- error_code: 500 +--- error_log eval +qr/\[error\] \d+#\d+: \*\d+ lua entry thread aborted: runtime error: "esc_type" is not number/ From a6d063bd3af76aa99858b8760b85c8b6886e9de1 Mon Sep 17 00:00:00 2001 From: lijunlong Date: Sun, 28 Jun 2020 15:50:59 +0800 Subject: [PATCH 17/19] feature: don't export ngx.ESC_XXX constants --- README.markdown | 30 +++++++----------------------- doc/HttpLuaModule.wiki | 24 ++++++------------------ src/ngx_http_lua_consts.c | 28 ---------------------------- src/ngx_http_lua_consts.h | 1 - src/ngx_http_lua_string.c | 6 ++++++ src/ngx_http_lua_util.c | 1 - t/006-escape.t | 32 ++++++++++++++++---------------- 7 files changed, 35 insertions(+), 87 deletions(-) diff --git a/README.markdown b/README.markdown index 47eae680d5..3341473212 100644 --- a/README.markdown +++ b/README.markdown @@ -3192,7 +3192,6 @@ Nginx API for Lua * [HTTP method constants](#http-method-constants) * [HTTP status constants](#http-status-constants) * [Nginx log level constants](#nginx-log-level-constants) -* [Nginx escape type constants](#nginx-escape-type-constants) * [print](#print) * [ngx.ctx](#ngxctx) * [ngx.location.capture](#ngxlocationcapture) @@ -3597,27 +3596,6 @@ These constants are usually used by the [ngx.log](#ngxlog) method. [Back to TOC](#nginx-api-for-lua) -Nginx escape type constants ---------------------------- - -**context:** *init_by_lua*, init_worker_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua** - -```lua - - ngx.ESCAPE_URI - ngx.ESCAPE_ARGS - ngx.ESCAPE_URI_COMPONENT - ngx.ESCAPE_HTML - ngx.ESCAPE_REFRESH - ngx.ESCAPE_MEMCACHED - ngx.ESCAPE_MAIL_AUTH -``` - - -These constants are usually used by the [ngx.escape_uri](#ngxescape_uri) method. - -[Back to TOC](#nginx-api-for-lua) - print ----- @@ -5610,7 +5588,13 @@ ngx.escape_uri Escape `str` as a URI component. Since `v0.10.16rc6`, this function accepts an optional `type` argument. -Note that this method accepts all [Nginx escape type constants](#nginx-escape-type-constants) as input. +```lua + + type = 0 (Escape str as a URI) + type = 2 (Escape str as a URI component) +``` +when type set to 0, " ", "#", "%", "?", 0x00-0x1F, 0x7F-0xFF will be escaped. +when type set to 2, all characters excepter ALPHA, DIGIT, "-", ".", "_", "~" will be escaped. [Back to TOC](#nginx-api-for-lua) diff --git a/doc/HttpLuaModule.wiki b/doc/HttpLuaModule.wiki index c18a31a116..bc01b76a0e 100644 --- a/doc/HttpLuaModule.wiki +++ b/doc/HttpLuaModule.wiki @@ -2929,23 +2929,6 @@ These constants are usually used in [[#ngx.location.capture|ngx.location.capture These constants are usually used by the [[#ngx.log|ngx.log]] method. -== Nginx escape type constants == - -'''context:''' ''init_by_lua*, init_worker_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua*'' - - - ngx.ESCAPE_URI - ngx.ESCAPE_ARGS - ngx.ESCAPE_URI_COMPONENT - ngx.ESCAPE_HTML - ngx.ESCAPE_REFRESH - ngx.ESCAPE_MEMCACHED - ngx.ESCAPE_MAIL_AUTH - - - -These constants are usually used by the [[#ngx.escape_uri|ngx.escape_uri]] method. - == print == '''syntax:''' ''print(...)'' @@ -4710,7 +4693,12 @@ This method was introduced in the 0.5.0rc30 release. Escape str as a URI component. Since `v0.10.16rc6`, this function accepts an optional type argument. -Note that this method accepts all [[#Nginx escape type constants|Nginx escape type constants]] as input. + + type = 0 (Escape str as a URI) + type = 2 (Escape str as a URI component) + +when type set to 0, " ", "#", "%", "?", 0x00-0x1F, 0x7F-0xFF will be escaped. +when type set to 2, all characters excepter ALPHA, DIGIT, "-", ".", "_", "~" will be escaped. == ngx.unescape_uri == diff --git a/src/ngx_http_lua_consts.c b/src/ngx_http_lua_consts.c index 38b3d6e9a0..47ba998576 100644 --- a/src/ngx_http_lua_consts.c +++ b/src/ngx_http_lua_consts.c @@ -13,34 +13,6 @@ #include "ngx_http_lua_consts.h" -void -ngx_http_lua_inject_escape_consts(lua_State *L) -{ - /* {{{ escape constants */ - lua_pushinteger(L, NGX_ESCAPE_URI); - lua_setfield(L, -2, "ESCAPE_URI"); - - lua_pushinteger(L, NGX_ESCAPE_ARGS); - lua_setfield(L, -2, "ESCAPE_ARGS"); - - lua_pushinteger(L, NGX_ESCAPE_URI_COMPONENT); - lua_setfield(L, -2, "ESCAPE_URI_COMPONENT"); - - lua_pushinteger(L, NGX_ESCAPE_HTML); - lua_setfield(L, -2, "ESCAPE_HTML"); - - lua_pushinteger(L, NGX_ESCAPE_REFRESH); - lua_setfield(L, -2, "ESCAPE_REFRESH"); - - lua_pushinteger(L, NGX_ESCAPE_MEMCACHED); - lua_setfield(L, -2, "ESCAPE_MEMCACHED"); - - lua_pushinteger(L, NGX_ESCAPE_MAIL_AUTH); - lua_setfield(L, -2, "ESCAPE_MAIL_AUTH"); - - /* }}} */ -} - void ngx_http_lua_inject_core_consts(lua_State *L) diff --git a/src/ngx_http_lua_consts.h b/src/ngx_http_lua_consts.h index d0c3055107..c403f81c4a 100644 --- a/src/ngx_http_lua_consts.h +++ b/src/ngx_http_lua_consts.h @@ -13,7 +13,6 @@ void ngx_http_lua_inject_http_consts(lua_State *L); void ngx_http_lua_inject_core_consts(lua_State *L); -void ngx_http_lua_inject_escape_consts(lua_State *L); #endif /* _NGX_HTTP_LUA_CONSTS_H_INCLUDED_ */ diff --git a/src/ngx_http_lua_string.c b/src/ngx_http_lua_string.c index a94e181745..556cb3e76b 100644 --- a/src/ngx_http_lua_string.c +++ b/src/ngx_http_lua_string.c @@ -436,6 +436,9 @@ size_t ngx_http_lua_ffi_uri_escaped_length(const u_char *src, size_t len, int type) { + if (type < NGX_ESCAPE_URI || type > NGX_ESCAPE_MAIL_AUTH) { + return -1; + } return len + 2 * ngx_http_lua_escape_uri(NULL, (u_char *) src, len, type); } @@ -444,6 +447,9 @@ void ngx_http_lua_ffi_escape_uri(const u_char *src, size_t len, u_char *dst, int type) { + if (type < NGX_ESCAPE_URI || type > NGX_ESCAPE_MAIL_AUTH) { + return; + } ngx_http_lua_escape_uri(dst, (u_char *) src, len, type); } diff --git a/src/ngx_http_lua_util.c b/src/ngx_http_lua_util.c index 43669803a6..272906cb96 100644 --- a/src/ngx_http_lua_util.c +++ b/src/ngx_http_lua_util.c @@ -726,7 +726,6 @@ ngx_http_lua_inject_ngx_api(lua_State *L, ngx_http_lua_main_conf_t *lmcf, ngx_http_lua_inject_arg_api(L); - ngx_http_lua_inject_escape_consts(L); ngx_http_lua_inject_http_consts(L); ngx_http_lua_inject_core_consts(L); diff --git a/t/006-escape.t b/t/006-escape.t index 49d5d3df86..7c357eefe8 100644 --- a/t/006-escape.t +++ b/t/006-escape.t @@ -204,11 +204,11 @@ GET /lua --- config location /lua { content_by_lua_block { - ngx.say(ngx.escape_uri("https://www.google.com", ngx.ESCAPE_URI)) - ngx.say(ngx.escape_uri("https://www.google.com/query?q=test", ngx.ESCAPE_URI)) - ngx.say(ngx.escape_uri("https://www.google.com/query?\r\nq=test", ngx.ESCAPE_URI)) - ngx.say(ngx.escape_uri("-_.~!*'();:@&=+$,/?#", ngx.ESCAPE_URI)) - ngx.say(ngx.escape_uri("<>[]{}\\\" ", ngx.ESCAPE_URI)) + ngx.say(ngx.escape_uri("https://www.google.com", 0)) + ngx.say(ngx.escape_uri("https://www.google.com/query?q=test", 0)) + ngx.say(ngx.escape_uri("https://www.google.com/query?\r\nq=test", 0)) + ngx.say(ngx.escape_uri("-_.~!*'();:@&=+$,/?#", 0)) + ngx.say(ngx.escape_uri("<>[]{}\\\" ", 0)) } } --- request @@ -228,13 +228,13 @@ https://www.google.com/query%3F%0D%0Aq=test --- config location /lua { content_by_lua_block { - ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", ngx.ESCAPE_URI)) - ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", ngx.ESCAPE_ARGS)) - ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", ngx.ESCAPE_URI_COMPONENT)) - ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", ngx.ESCAPE_HTML)) - ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", ngx.ESCAPE_REFRESH)) - ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", ngx.ESCAPE_MEMCACHED)) - ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", ngx.ESCAPE_MAIL_AUTH)) + ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", 0)) + ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", 1)) + ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", 2)) + ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", 3)) + ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", 4)) + ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", 5)) + ngx.say(ngx.escape_uri("https://www.google.com/?t=abc@ :", 6)) } } --- request @@ -263,7 +263,7 @@ https://www.google.com/?t=abc@%20: GET /lua --- error_code: 500 --- error_log eval -qr/\[error\] \d+#\d+: \*\d+ lua entry thread aborted: runtime error: "esc_type" \-1 out of range/ +qr/\[error\] \d+#\d+: \*\d+ lua entry thread aborted: runtime error: "type" \-1 out of range/ @@ -271,14 +271,14 @@ qr/\[error\] \d+#\d+: \*\d+ lua entry thread aborted: runtime error: "esc_type" --- config location /lua { content_by_lua_block { - ngx.say(ngx.escape_uri("https://www.google.com", 100)) + ngx.say(ngx.escape_uri("https://www.google.com", 10)) } } --- request GET /lua --- error_code: 500 --- error_log eval -qr/\[error\] \d+#\d+: \*\d+ lua entry thread aborted: runtime error: "esc_type" 100 out of range/ +qr/\[error\] \d+#\d+: \*\d+ lua entry thread aborted: runtime error: "type" 10 out of range/ @@ -293,4 +293,4 @@ qr/\[error\] \d+#\d+: \*\d+ lua entry thread aborted: runtime error: "esc_type" GET /lua --- error_code: 500 --- error_log eval -qr/\[error\] \d+#\d+: \*\d+ lua entry thread aborted: runtime error: "esc_type" is not number/ +qr/\[error\] \d+#\d+: \*\d+ lua entry thread aborted: runtime error: "type" is not number/ From c15ea74cd1d0ed38b731b5dbc3fe2f7b64632a12 Mon Sep 17 00:00:00 2001 From: lijunlong Date: Mon, 29 Jun 2020 00:50:36 +0800 Subject: [PATCH 18/19] style: remove duplicate checks and adjust doc --- README.markdown | 13 ++++--------- doc/HttpLuaModule.wiki | 12 ++++-------- src/ngx_http_lua_string.c | 6 ------ 3 files changed, 8 insertions(+), 23 deletions(-) diff --git a/README.markdown b/README.markdown index 3341473212..85ac8bb8c7 100644 --- a/README.markdown +++ b/README.markdown @@ -5585,16 +5585,11 @@ ngx.escape_uri **context:** *init_by_lua*, init_worker_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua** -Escape `str` as a URI component. - Since `v0.10.16rc6`, this function accepts an optional `type` argument. -```lua - - type = 0 (Escape str as a URI) - type = 2 (Escape str as a URI component) -``` -when type set to 0, " ", "#", "%", "?", 0x00-0x1F, 0x7F-0xFF will be escaped. -when type set to 2, all characters excepter ALPHA, DIGIT, "-", ".", "_", "~" will be escaped. +When `type` is set to 0, escape `str` as a URI. And these +characters " ", "#", "%", "?", 0x00-0x1F, 0x7F-0xFF will be escaped. +When `type` is set to 2 (which is the default), escape `str` +as a URI component. All characters excepter ALPHA, DIGIT, "-", ".", "_", "~" will be escaped. [Back to TOC](#nginx-api-for-lua) diff --git a/doc/HttpLuaModule.wiki b/doc/HttpLuaModule.wiki index bc01b76a0e..0a9ee4b734 100644 --- a/doc/HttpLuaModule.wiki +++ b/doc/HttpLuaModule.wiki @@ -4690,15 +4690,11 @@ This method was introduced in the 0.5.0rc30 release. '''context:''' ''init_by_lua*, init_worker_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua*'' -Escape str as a URI component. - Since `v0.10.16rc6`, this function accepts an optional type argument. - - type = 0 (Escape str as a URI) - type = 2 (Escape str as a URI component) - -when type set to 0, " ", "#", "%", "?", 0x00-0x1F, 0x7F-0xFF will be escaped. -when type set to 2, all characters excepter ALPHA, DIGIT, "-", ".", "_", "~" will be escaped. +When type is set to 0, escape str as a URI. And these +characters " ", "#", "%", "?", 0x00-0x1F, 0x7F-0xFF will be escaped. +When type is set to 2 (which is the default), escape str +as a URI component. All characters excepter ALPHA, DIGIT, "-", ".", "_", "~" will be escaped. == ngx.unescape_uri == diff --git a/src/ngx_http_lua_string.c b/src/ngx_http_lua_string.c index 556cb3e76b..a94e181745 100644 --- a/src/ngx_http_lua_string.c +++ b/src/ngx_http_lua_string.c @@ -436,9 +436,6 @@ size_t ngx_http_lua_ffi_uri_escaped_length(const u_char *src, size_t len, int type) { - if (type < NGX_ESCAPE_URI || type > NGX_ESCAPE_MAIL_AUTH) { - return -1; - } return len + 2 * ngx_http_lua_escape_uri(NULL, (u_char *) src, len, type); } @@ -447,9 +444,6 @@ void ngx_http_lua_ffi_escape_uri(const u_char *src, size_t len, u_char *dst, int type) { - if (type < NGX_ESCAPE_URI || type > NGX_ESCAPE_MAIL_AUTH) { - return; - } ngx_http_lua_escape_uri(dst, (u_char *) src, len, type); } From 76516a3a164f1dcaf0fd7ab18ec27f55da55da30 Mon Sep 17 00:00:00 2001 From: lijunlong Date: Mon, 29 Jun 2020 02:06:17 +0800 Subject: [PATCH 19/19] test: modify .travis.yml for testing --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index fc2e4c20bf..dd56267580 100644 --- a/.travis.yml +++ b/.travis.yml @@ -83,10 +83,10 @@ install: - git clone https://github.com/openresty/rds-json-nginx-module.git ../rds-json-nginx-module - git clone https://github.com/openresty/srcache-nginx-module.git ../srcache-nginx-module - git clone https://github.com/openresty/redis2-nginx-module.git ../redis2-nginx-module - - git clone https://github.com/openresty/lua-resty-core.git ../lua-resty-core + - git clone -b uri_not_comp https://github.com/zhuizhuhaomeng/lua-resty-core.git ../lua-resty-core - git clone https://github.com/openresty/lua-resty-lrucache.git ../lua-resty-lrucache - git clone https://github.com/openresty/lua-resty-mysql.git ../lua-resty-mysql - - git clone https://github.com/openresty/stream-lua-nginx-module.git ../stream-lua-nginx-module + - git clone -b uri_not_comp https://github.com/zhuizhuhaomeng/stream-lua-nginx-module.git ../stream-lua-nginx-module - git clone -b v2.1-agentzh https://github.com/openresty/luajit2.git luajit2 before_script: