[INCOMPATIBLE][SELF INCOMPATIBLE] main,Tmain: revise acceptable characters for kind letters and kind names

README.md

@@ -1,4 +1,3 @@
-
 # Universal Ctags
 
 [![Build Status](https://travis-ci.org/universal-ctags/ctags.svg?branch=master)](https://travis-ci.org/universal-ctags/ctags)
@@ -57,18 +56,37 @@ In addition rst2man command is needed.
 
 rst2man is part of the python-docutils package on Ubuntu.
 
-## Difference ##
+## Differences ##
 
 You may be interested in how universal-ctags is different from
 exuberant-ctags. The critical and attractive changes are explained
 in docs/\*.rst. The preformatted version is available on line,
 http://docs.ctags.io/.
 
-The most significant one is that Universal-ctags doesn't load
+The most significant incompatible changes:
+
+* Universal-ctags doesn't load
 `~/.ctags` and `./.ctags` at starting up time. Instead, it loads
 `~/.ctags.d/*.ctags` and `./.ctags.d/*.ctags`. See the above web
 site and man pages
 (man/ctags.1.rst.in and man/ctags-incompatibilities.7.in in the
 source tree).
 
+* Universal-ctags is more strict about characters that can be
+  used in kind letters and kind names than Exuberant-ctags.
+
+  - The letter must be an alphabetical character (`[a-zA-EG-Z]`).
+    `F` is reserved for `file` kind.
+
+  - The first character of the name must be alphabetic, and
+    the rest characters must be alphanumeric (`[a-zA-Z][a-zA-Z0-9]*`).
+
+  See the web site and man pages. The detailed background is explained
+  in [#1737](https://github.com/universal-ctags/ctags/pull/1737).
+
+  If you want to reuse your .ctags written for Exuberant-ctags,
+  you must review kind letters and names defined with `--regex-<LANG>=...`
+  options. When updating the definitions, using `--kind-<LANG>` option
+  is appreciated.
+
 Pull-requests are welcome!

Tmain/kinddef.d/run.sh

@@ -68,6 +68,9 @@ ${CTAGS} --kinddef-MYTEST= --list-kinds-full=MYTEST
 title '# wrong letter'
 ${CTAGS} --kinddef-MYTEST=^ --list-kinds-full=MYTEST
 
+title '# wrong letter (using number)'
+${CTAGS} --kinddef-MYTEST=7 --list-kinds-full=MYTEST
+
 title '# reusing the letter for file kind'
 ${CTAGS} --kinddef-MYTEST=F --list-kinds-full=MYTEST
 
@@ -89,6 +92,12 @@ ${CTAGS} --kinddef-MYTEST='x,kind,desc\{' --list-kinds-full=MYTEST
 title '# use a { in description (2)'
 ${CTAGS} --kinddef-MYTEST='x,kind,desc\{}' --list-kinds-full=MYTEST
 
+title '# use a number char as the initial letter'
+${CTAGS} --kinddef-MYTEST='x,0kind,desc' --list-kinds-full=MYTEST
+
+title '# use a number char within the body'
+${CTAGS} --kinddef-MYTEST='x,k0ind,desc' --list-kinds-full=MYTEST
+
 # title '# use a { and \t in description'
 # ${CTAGS} --kinddef-MYTEST='x,kind,desc\{}\t' --list-kinds-full=MYTEST
 

Tmain/kinddef.d/stderr-expected.txt

@@ -23,10 +23,10 @@ ctags: the kind name in "--kinddef-MYTEST" option is empty
 ctags: the kind name in "--kinddef-MYTEST" option is empty
 
 # wrong letter in name 1
-ctags: unacceptable char as part of kind name in "--kinddef-MYTEST" option
+ctags: non-alphanumeric char is used as part of kind name: 'a	x' in "--kinddef-MYTEST" option
 
 # wrong letter in name 2
-ctags: unacceptable char as part of kind name in "--kinddef-MYTEST" option
+ctags: non-alphanumeric char is used as part of kind name: 'a	x' in "--kinddef-MYTEST" option
 
 # wrong letter in name 3
 ctags: no kind description specified in "--kinddef-MYTEST" option
@@ -47,7 +47,10 @@ ctags: no kind letter specified in "--kinddef-MYTEST" option
 ctags: no kind definition specified in "--kinddef-MYTEST" option
 
 # wrong letter
-ctags: the kind letter given in "--kinddef-MYTEST" option is not an alphabet or a number
+ctags: the kind letter given in "--kinddef-MYTEST" option is not an alphabet
+
+# wrong letter (using number)
+ctags: the kind letter given in "--kinddef-MYTEST" option is not an alphabet
 
 # reusing the letter for file kind
 ctags: the kind letter `F' in "--kinddef-MYTEST" option is reserved for "file" kind
@@ -67,6 +70,11 @@ ctags: Warning: long flags specifier opened with `{' is not closed `}': "{"
 
 # use a { in description (2)
 
+# use a number char as the initial letter
+ctags: a kind name doesn't start with an alphabetical character: '0kind' in "--kinddef-MYTEST" option
+
+# use a number char within the body
+
 # use a \ in description
 
 # description started from {

Tmain/kinddef.d/stdout-expected.txt

@@ -35,6 +35,8 @@ a       aa   yes     no      0      NONE   aaa
 
 # wrong letter
 
+# wrong letter (using number)
+
 # reusing the letter for file kind
 
 # reusing the name for file kind
@@ -59,6 +61,12 @@ x       kind yes     no      0      NONE   desc{
 #LETTER NAME ENABLED REFONLY NROLES MASTER DESCRIPTION
 x       kind yes     no      0      NONE   desc{}
 
+# use a number char as the initial letter
+
+# use a number char within the body
+#LETTER NAME  ENABLED REFONLY NROLES MASTER DESCRIPTION
+x       k0ind yes     no      0      NONE   desc
+
 # use a \ in description
 #LETTER NAME ENABLED REFONLY NROLES MASTER DESCRIPTION
 x       kind yes     no      0      NONE   desc\backslash

Tmain/lregex-kind-letter.d/run.sh

@@ -0,0 +1,25 @@
+# Copyright: 2018 Masatake YAMATO
+# License: GPL-2
+
+CTAGS="$1 --quiet --options=NONE"
+
+. ../utils.sh
+
+echo2 param: '|(.)|\1|^ => ignoring'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|^' --list-kinds-full=x
+# In this case, ctags ignores the substring after the last '|'.
+# It is evaluated as flags.
+
+echo2 param: '|(.)|\1|^| => warning'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|^|' --list-kinds-full=x
+# In this case, ctags warns specifying a wrong kind letter '^'.
+
+echo2 param: '|(.)|\1|, => ignoring'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|,' --list-kinds-full=x
+# In this case, ctags ignores the substring after the last '|'.
+# It is evaluated as flags.
+
+echo2 param: '|(.)|\1|,| => using the default letter and name'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|,|' --list-kinds-full=x
+# In this case, ctags recognizes a kind letter and name
+# are not given; 'r' and "regex" are used as default values.

Tmain/lregex-kind-letter.d/stderr-expected.txt

@@ -0,0 +1,5 @@
+param: |(.)|\1|^ => ignoring
+param: |(.)|\1|^| => warning
+ctags: Kind letter must be an alphabetical character: "^"
+param: |(.)|\1|, => ignoring
+param: |(.)|\1|,| => using the default letter and name

Tmain/lregex-kind-letter.d/stdout-expected.txt

@@ -0,0 +1,10 @@
+param: |(.)|\1|^ => ignoring
+#LETTER NAME  ENABLED REFONLY NROLES MASTER DESCRIPTION
+r       regex yes     no      0      NONE   regex
+param: |(.)|\1|^| => warning
+param: |(.)|\1|, => ignoring
+#LETTER NAME  ENABLED REFONLY NROLES MASTER DESCRIPTION
+r       regex yes     no      0      NONE   regex
+param: |(.)|\1|,| => using the default letter and name
+#LETTER NAME  ENABLED REFONLY NROLES MASTER DESCRIPTION
+r       regex yes     no      0      NONE   regex

Tmain/lregex-kind-name.d/run.sh

@@ -0,0 +1,54 @@
+# Copyright: 2018 Masatake YAMATO
+# License: GPL-2
+
+CTAGS="$1 --quiet --options=NONE"
+
+. ../utils.sh
+
+echo2 param: '|(.)|\1|x,name| => acceptable'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|x,name|' --list-kinds-full=x
+
+echo2 param: '|(.)|\1|x,name,documents| => acceptable'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|x,name,documents|' --list-kinds-full=x
+
+echo2 param: '|(.)|\1|x,name,0documents| => acceptable'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|x,name,0documents|' --list-kinds-full=x
+
+echo2 param: '|(.)|\1|x,name,doc uments| => acceptable'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|x,name,doc uments|' --list-kinds-full=x
+
+echo2 param: '|(.)|\1|x,name0| => acceptable'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|x,name0|' --list-kinds-full=x
+
+echo2 param: '|(.)|\1|x,name0,documents| => acceptable'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|x,name0,documents|' --list-kinds-full=x
+
+echo2 param: '|(.)|\1|x,name0,0documents| => acceptable'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|x,name0,0documents|' --list-kinds-full=x
+
+echo2 param: '|(.)|\1|x,name0,doc uments| => acceptable'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|x,name0,doc uments|' --list-kinds-full=x
+
+echo2 param: '|(.)|\1|x,0name| => unacceptable'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|x,0name|' --list-kinds-full=x
+
+echo2 param: '|(.)|\1|x,0name,documents| => unacceptable'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|x,0name,documents|' --list-kinds-full=x
+
+echo2 param: '|(.)|\1|x,0name,0documents| => unacceptable'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|x,0name,0documents|' --list-kinds-full=x
+
+echo2 param: '|(.)|\1|x,0name,doc uments| => unacceptable'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|x,0name,doc uments|' --list-kinds-full=x
+
+echo2 param: '|(.)|\1|x,na me| => unacceptable'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|x,na me|' --list-kinds-full=x
+
+echo2 param: '|(.)|\1|x,na me,documents| => unacceptable'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|x,na me,documents|' --list-kinds-full=x
+
+echo2 param: '|(.)|\1|x,na me,0documents| => unacceptable'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|x,na me,0documents|' --list-kinds-full=x
+
+echo2 param: '|(.)|\1|x,na me,doc uments| => unacceptable'
+${CTAGS} --langdef=x --regex-x='|(.)|\1|x,na me,doc uments|' --list-kinds-full=x

Tmain/lregex-kind-name.d/stderr-expected.txt

@@ -0,0 +1,24 @@
+param: |(.)|\1|x,name| => acceptable
+param: |(.)|\1|x,name,documents| => acceptable
+param: |(.)|\1|x,name,0documents| => acceptable
+param: |(.)|\1|x,name,doc uments| => acceptable
+param: |(.)|\1|x,name0| => acceptable
+param: |(.)|\1|x,name0,documents| => acceptable
+param: |(.)|\1|x,name0,0documents| => acceptable
+param: |(.)|\1|x,name0,doc uments| => acceptable
+param: |(.)|\1|x,0name| => unacceptable
+ctags: A kind name doesn't start with an alphabetical character: '0name' in "--regex-x" option
+param: |(.)|\1|x,0name,documents| => unacceptable
+ctags: A kind name doesn't start with an alphabetical character: '0name' in "--regex-x" option
+param: |(.)|\1|x,0name,0documents| => unacceptable
+ctags: A kind name doesn't start with an alphabetical character: '0name' in "--regex-x" option
+param: |(.)|\1|x,0name,doc uments| => unacceptable
+ctags: A kind name doesn't start with an alphabetical character: '0name' in "--regex-x" option
+param: |(.)|\1|x,na me| => unacceptable
+ctags: Non-alphanumeric char is used in kind name: 'na me' in "--regex-x" option
+param: |(.)|\1|x,na me,documents| => unacceptable
+ctags: Non-alphanumeric char is used in kind name: 'na me' in "--regex-x" option
+param: |(.)|\1|x,na me,0documents| => unacceptable
+ctags: Non-alphanumeric char is used in kind name: 'na me' in "--regex-x" option
+param: |(.)|\1|x,na me,doc uments| => unacceptable
+ctags: Non-alphanumeric char is used in kind name: 'na me' in "--regex-x" option

Tmain/lregex-kind-name.d/stdout-expected.txt

@@ -0,0 +1,32 @@
+param: |(.)|\1|x,name| => acceptable
+#LETTER NAME ENABLED REFONLY NROLES MASTER DESCRIPTION
+x       name yes     no      0      NONE   name
+param: |(.)|\1|x,name,documents| => acceptable
+#LETTER NAME ENABLED REFONLY NROLES MASTER DESCRIPTION
+x       name yes     no      0      NONE   documents
+param: |(.)|\1|x,name,0documents| => acceptable
+#LETTER NAME ENABLED REFONLY NROLES MASTER DESCRIPTION
+x       name yes     no      0      NONE   0documents
+param: |(.)|\1|x,name,doc uments| => acceptable
+#LETTER NAME ENABLED REFONLY NROLES MASTER DESCRIPTION
+x       name yes     no      0      NONE   doc uments
+param: |(.)|\1|x,name0| => acceptable
+#LETTER NAME  ENABLED REFONLY NROLES MASTER DESCRIPTION
+x       name0 yes     no      0      NONE   name0
+param: |(.)|\1|x,name0,documents| => acceptable
+#LETTER NAME  ENABLED REFONLY NROLES MASTER DESCRIPTION
+x       name0 yes     no      0      NONE   documents
+param: |(.)|\1|x,name0,0documents| => acceptable
+#LETTER NAME  ENABLED REFONLY NROLES MASTER DESCRIPTION
+x       name0 yes     no      0      NONE   0documents
+param: |(.)|\1|x,name0,doc uments| => acceptable
+#LETTER NAME  ENABLED REFONLY NROLES MASTER DESCRIPTION
+x       name0 yes     no      0      NONE   doc uments
+param: |(.)|\1|x,0name| => unacceptable
+param: |(.)|\1|x,0name,documents| => unacceptable
+param: |(.)|\1|x,0name,0documents| => unacceptable
+param: |(.)|\1|x,0name,doc uments| => unacceptable
+param: |(.)|\1|x,na me| => unacceptable
+param: |(.)|\1|x,na me,documents| => unacceptable
+param: |(.)|\1|x,na me,0documents| => unacceptable
+param: |(.)|\1|x,na me,doc uments| => unacceptable

Tmain/lregex-list-kinds-full.d/run.sh

@@ -5,7 +5,7 @@ CTAGS=$1
 
 ${CTAGS} --quiet --options=NONE -o - \
 	 --langdef=foo \
-	 --regex-foo='/a/\0/a,x y,x y z/' --kinds-foo=-a --foo-kinds=+a \
+	 --regex-foo='/a/\0/a,xy,x y z/' --kinds-foo=-a --foo-kinds=+a \
 	 --regex-foo=/b/\0/b/ --foo-kinds=-b \
 	 --regex-foo=/c/\0/c/ \
 	 --kinds-foo=-c \

Tmain/lregex-list-kinds-full.d/stdout-expected.txt

@@ -1,5 +1,5 @@
 #LETTER	NAME	ENABLED	REFONLY	NROLES	MASTER	DESCRIPTION
-a	x y	yes	no	0	NONE	x y z
+a	xy	yes	no	0	NONE	x y z
 b	regex	no	no	0	NONE	regex
 c	regex	no	no	0	NONE	regex
 d	regex	yes	no	0	NONE	regex

Tmain/optlib-message-flag.d/args.ctags

@@ -5,13 +5,13 @@
 --kinddef-FOO=c,class,classes
 --kinddef-FOO=v,variable,variables
 
---regex-FOO=/namespace ([a-zA-Z]+) /\1/N,another_namespace/{warning="found namespace '\1'"}{exclusive}
+--regex-FOO=/namespace ([a-zA-Z]+) /\1/N,anotherNamespace/{warning="found namespace '\1'"}{exclusive}
 --regex-FOO=/namespace ([a-zA-Z]+) /\1/x,bad/{fatal="should not get this"}
 --regex-FOO=/bad regex ([a-zA-Z-]+)/\1/x,bad/{fatal="bad='\1'"}
 
 
---mline-regex-FOO=/namespace ([a-zA-Z]+) /\1/m,mline_namespace/{mgroup=1}{warning="got namespace"}
---mline-regex-FOO=/var ([a-zA-Z]+) ([a-zA-Z]+);/\2/V,mline_variable/{warning="got variable '\2' of type \1"}{mgroup=2}
+--mline-regex-FOO=/namespace ([a-zA-Z]+) /\1/m,mlineNamespace/{mgroup=1}{warning="got namespace"}
+--mline-regex-FOO=/var ([a-zA-Z]+) ([a-zA-Z]+);/\2/V,mlineVariable/{warning="got variable '\2' of type \1"}{mgroup=2}
 --mline-regex-FOO=/bad multi-line ([a-zA-Z-]+)/\1/x,bad/{fatal="bad='\1'"}
 
 

Tmain/utils.sh

@@ -126,3 +126,11 @@ filter_by_column_index()
 		done
 	done
 }
+
+echo2()
+{
+	# use a external echo command here.
+	# built-in echo suppresses \1.
+	/bin/echo "$@"
+	/bin/echo "$@" 1>&2
+}

docs/format.rst

@@ -260,6 +260,9 @@ A tagfield has a name, a colon, and a value: "name:value".
   are allowed.  Lower case is recommended.  Case matters ("kind:" and "Kind:
   are different tagfields).
 
+  EXCEPTION: Universal-ctags allows users to use a numerical character
+  in the name other than its initial letter.
+
 * The value may be empty.
   It cannot contain a <Tab>.
 
@@ -482,6 +485,12 @@ Exceptions
    exceptions. See {tagname} of Proposal section for more detail about the
    conversion.
 
+.. NOT REVIEWED YET
+
+#. "name" part of {tagfield} in a tag generated by Universal-ctags may
+   contain numeric characters, but the first character of the "name"
+   must be alphabetic.
+
 .. _compat-output:
 
 Compatible output and weakness