Change messages and Gherkin parser/pickle compiler to retain step key…

…words Proof-of-concept implementation of retention of the keyword specified in the step specification into the Pickle, for discussion.
cucumber · Sep 8, 2021 · 04f3941 · 04f3941
1 parent f948e5a
commit 04f3941
Show file tree

Hide file tree

Showing 8 changed files with 141 additions and 15 deletions.
diff --git a/gherkin/perl/lib/Gherkin/AstBuilder.pm b/gherkin/perl/lib/Gherkin/AstBuilder.pm
@@ -169,12 +169,13 @@ sub transform_node {
         my $doc_string = $node->get_single('DocString') || undef;
 
         return Cucumber::Messages::Step->new(
-            id         => $self->next_id,
-            location   => $self->get_location($step_line),
-            keyword    => $step_line->matched_keyword,
-            text       => $step_line->matched_text,
-            doc_string => $doc_string,
-            data_table => $data_table,
+            id           => $self->next_id,
+            location     => $self->get_location($step_line),
+            keyword      => $step_line->matched_keyword,
+            keyword_type => $step_line->keyword_type,
+            text         => $step_line->matched_text,
+            doc_string   => $doc_string,
+            data_table   => $data_table,
             );
     } elsif ( $node->rule_type eq 'DocString' ) {
         my $separator_token = $node->get_tokens('DocStringSeparator')->[0];

diff --git a/gherkin/perl/lib/Gherkin/Pickles/Compiler.pm b/gherkin/perl/lib/Gherkin/Pickles/Compiler.pm
@@ -50,6 +50,14 @@ sub _pickle_steps {
     return [ map { $class->_pickle_step( $_, $id_generator ) } @$steps ];
 }
 
+my %inheriting_keyword = (
+    Given   => 0,
+    When    => 0,
+    Then    => 0,
+    And     => 1,
+    But     => 1,
+    );
+
 sub _compile_scenario {
     my ( $class, $uri, $tags, $background_steps,
          $scenario, $variables, $values, $values_id,
@@ -58,6 +66,7 @@ sub _compile_scenario {
 
     my @steps;
     if ($scenario->steps and @{ $scenario->steps }) {
+        my $last_keyword;
         @steps = @{ $class->_pickle_steps($background_steps,
                                           $id_generator) };
         for my $step (@{ $scenario->steps } ) {
@@ -68,9 +77,17 @@ sub _compile_scenario {
                 $class->_create_pickle_arguments(
                     $step,
                     $variables, $values );
+
+            $last_keyword =
+                $inheriting_keyword{$step->keyword_type} ?
+                $last_keyword : $step->keyword_type;
+
+            #TODO: Throw an exception here when $last_keyword is empty
+
             push @steps,
                 Cucumber::Messages::PickleStep->new(
                     id         => $id_generator->(),
+                    keyword    => $last_keyword,
                     text       => $step_text,
                     argument   => $arguments,
                     ast_node_ids => [ $step->id,

diff --git a/gherkin/perl/lib/Gherkin/Token.pm b/gherkin/perl/lib/Gherkin/Token.pm
@@ -6,7 +6,7 @@ use warnings;
 use Class::XSAccessor
   constructor => 'new',
   accessors   => [
-    qw/line location/,
+    qw/line location keyword_type/,
     map { "matched_$_" } qw/type keyword indent items text gherkin_dialect/
   ],
   ;

diff --git a/gherkin/perl/lib/Gherkin/TokenFormatterBuilder.pm b/gherkin/perl/lib/Gherkin/TokenFormatterBuilder.pm
@@ -35,7 +35,8 @@ sub format_token {
         $token->location->{'line'},
         $token->location->{'column'},
         $token->matched_type,
-        $token->matched_keyword || '',
+        ( $token->matched_keyword ?
+          sprintf('(%s)%s',$token->keyword_type,$token->matched_keyword) : ''),
         $token->matched_text    || '',
         join( ',',
             map { $_->{'column'} . ':' . $_->{'text'} }

diff --git a/gherkin/perl/lib/Gherkin/TokenMatcher.pm b/gherkin/perl/lib/Gherkin/TokenMatcher.pm
@@ -113,6 +113,8 @@ sub _set_token_matched {
 
     $token->matched_keyword( $options->{'keyword'} )
       if defined $options->{'keyword'};
+    $token->keyword_type( $options->{'keyword_type'} )
+      if defined $options->{'keyword_type'};
 
     if ( defined $options->{'indent'} ) {
         $token->matched_indent( $options->{'indent'} );
@@ -176,14 +178,19 @@ sub _unescaped_docstring {
 
 sub match_StepLine {
     my ( $self, $token ) = @_;
-    my @keywords = map { @{ $self->dialect->$_ } } qw/Given When Then And But/;
 
-    for my $keyword (@keywords) {
-        if ( $token->line->startswith($keyword) ) {
-            my $title = $token->line->get_rest_trimmed( length($keyword) );
-            $self->_set_token_matched( $token,
-                StepLine => { text => $title, keyword => $keyword } );
-            return 1;
+    for my $keyword_type (qw/Given When Then And But/) {
+        for my $keyword (@{ $self->dialect->$keyword_type() }) {
+            if ( $token->line->startswith($keyword) ) {
+                my $title = $token->line->get_rest_trimmed( length($keyword) );
+                $self->_set_token_matched( $token,
+                                           StepLine => {
+                                               text         => $title,
+                                               keyword      => $keyword,
+                                               keyword_type => $keyword_type,
+                                           } );
+                return 1;
+            }
         }
     }
     return;

diff --git a/messages/jsonschema/GherkinDocument.json b/messages/jsonschema/GherkinDocument.json
@@ -338,6 +338,7 @@
       "required": [
         "location",
         "keyword",
+        "keyword_type",
         "text",
         "id"
       ],
@@ -347,8 +348,20 @@
           "description": "The location of the steps' `keyword`"
         },
         "keyword": {
+          "description": "The actual keyword as it appeared in the source.",
           "type": "string"
         },
+        "keywordType": {
+          "description": "The keyword as it is known in the Gherkin grammar that the value from the `keyword` field is associated with.",
+          "type": "string",
+          "enum": [
+            "Given",
+            "When",
+            "Then",
+            "But",
+            "And"
+          ]
+        },
         "text": {
           "type": "string"
         },

diff --git a/messages/jsonschema/Pickle.json b/messages/jsonschema/Pickle.json
@@ -25,6 +25,7 @@
       "required": [
         "astNodeIds",
         "id",
+        "keyword",
         "text"
       ],
       "properties": {
@@ -42,6 +43,15 @@
           "description": "A unique ID for the PickleStep",
           "type": "string"
         },
+        "keyword": {
+          "description": "The context in which the step was specified: context (Given), action (When) or outcome (Then).\n\nNote that the keywords `But` and `And` inherit their contextual meaning from prior steps and can therefore not appear in the pickle (as the picle is considered to be the execution plan).",
+          "type": "string",
+          "enum": [
+            "Given",
+            "When",
+            "Then"
+          ]
+        },
         "text": {
           "type": "string"
         }

diff --git a/messages/perl/lib/Cucumber/Messages.pm b/messages/perl/lib/Cucumber/Messages.pm
@@ -1594,6 +1594,7 @@ use Scalar::Util qw( blessed );
 my %types = (
    location => 'Cucumber::Messages::Location',
    keyword => 'string',
+   keyword_type => '',
    text => 'string',
    doc_string => 'Cucumber::Messages::DocString',
    data_table => 'Cucumber::Messages::DataTable',
@@ -1623,6 +1624,7 @@ has location =>
 
 =head4 keyword
 
+The actual keyword as it appeared in the source.
 
 =cut
 
@@ -1633,6 +1635,45 @@ has keyword =>
     );
 
 
+=head4 keyword_type
+
+The keyword as it is known in the Gherkin grammar that the value from the `keyword` field is associated with.
+
+
+Available constants for valid values of this field:
+
+=over
+
+=item * KEYWORD_TYPE_GIVEN
+
+=item * KEYWORD_TYPE_WHEN
+
+=item * KEYWORD_TYPE_THEN
+
+=item * KEYWORD_TYPE_BUT
+
+=item * KEYWORD_TYPE_AND
+
+=back
+
+=cut
+
+
+use constant
+   KEYWORD_TYPE_GIVEN => 'Given',
+   KEYWORD_TYPE_WHEN => 'When',
+   KEYWORD_TYPE_THEN => 'Then',
+   KEYWORD_TYPE_BUT => 'But',
+   KEYWORD_TYPE_AND => 'And',
+   ;
+
+has keyword_type =>
+    (is => 'ro',
+     required => 1,
+     default => sub { KEYWORD_TYPE_GIVEN },
+    );
+
+
 =head4 text
 
 
@@ -2750,6 +2791,7 @@ my %types = (
    argument => 'Cucumber::Messages::PickleStepArgument',
    ast_node_ids => '[]string',
    id => 'string',
+   keyword => '',
    text => 'string',
 );
 
@@ -2798,6 +2840,41 @@ has id =>
     );
 
 
+=head4 keyword
+
+The context in which the step was specified: context (Given), action (When) or outcome (Then).
+
+Note that the keywords `But` and `And` inherit their contextual meaning from prior steps and can therefore not appear in the pickle (as the picle is considered to be the execution plan).
+
+
+Available constants for valid values of this field:
+
+=over
+
+=item * KEYWORD_GIVEN
+
+=item * KEYWORD_WHEN
+
+=item * KEYWORD_THEN
+
+=back
+
+=cut
+
+
+use constant
+   KEYWORD_GIVEN => 'Given',
+   KEYWORD_WHEN => 'When',
+   KEYWORD_THEN => 'Then',
+   ;
+
+has keyword =>
+    (is => 'ro',
+     required => 1,
+     default => sub { KEYWORD_GIVEN },
+    );
+
+
 =head4 text