Skip to content

Commit

Permalink
[ObjC] Parsing helper and tests around unknown fields.
Browse files Browse the repository at this point in the history 
Make a new internal api for collecting up the unknown group fields
that will be used in a future change.

Add testing for the new api and for unknown field parsing of groups in
general.

PiperOrigin-RevId: 654875605
  • Loading branch information
@thomasvl
thomasvl committed Jul 24, 2024
1 parent fad7b78 commit 9b16ee4
Show file tree
Hide file tree
Showing 5 changed files with 493 additions and 1 deletion.
54 changes: 54 additions & 0 deletions objectivec/GPBCodedInputStream.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,60 @@ int32_t GPBCodedInputStreamReadTag(GPBCodedInputStreamState *state) {
return result;
}

static void SkipToEndGroupInternal(GPBCodedInputStreamState *state, uint32_t endGroupTag) {
CheckRecursionLimit(state);
++state->recursionDepth;
while (YES) {
uint32_t tag = GPBCodedInputStreamReadTag(state);
if (tag == endGroupTag || tag == 0) {
GPBCodedInputStreamCheckLastTagWas(state, endGroupTag); // Will fail for end of input.
--state->recursionDepth;
return;
}
switch (GPBWireFormatGetTagWireType(tag)) {
case GPBWireFormatVarint:
(void)ReadRawVarint64(state);
break;
case GPBWireFormatFixed64:
SkipRawData(state, sizeof(uint64_t));
break;
case GPBWireFormatLengthDelimited: {
uint64_t size = ReadRawVarint64(state);
CheckFieldSize(size);
size_t size2 = (size_t)size; // Cast safe on 32bit because of CheckFieldSize() above.
SkipRawData(state, size2);
break;
}
case GPBWireFormatStartGroup:
SkipToEndGroupInternal(state, GPBWireFormatMakeTag(GPBWireFormatGetTagFieldNumber(tag),
GPBWireFormatEndGroup));
break;
case GPBWireFormatEndGroup:
GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidTag, @"Unmatched end group");
break;
case GPBWireFormatFixed32:
SkipRawData(state, sizeof(uint32_t));
break;
}
}
}

// This doesn't include the start group, but it collects all bytes until the end group including
// the end group tag.
NSData *GPBCodedInputStreamReadRetainedBytesToEndGroupNoCopy(GPBCodedInputStreamState *state,
int32_t fieldNumber) {
// Better have just read the start of the group.
GPBCodedInputStreamCheckLastTagWas(state,
GPBWireFormatMakeTag(fieldNumber, GPBWireFormatStartGroup));
const uint8_t *start = state->bytes + state->bufferPos;
SkipToEndGroupInternal(state, GPBWireFormatMakeTag(fieldNumber, GPBWireFormatEndGroup));
// This will be after the end group tag.
const uint8_t *end = state->bytes + state->bufferPos;
return [[NSData alloc] initWithBytesNoCopy:(void *)start
length:(NSUInteger)(end - start)
freeWhenDone:NO];
}

size_t GPBCodedInputStreamPushLimit(GPBCodedInputStreamState *state, size_t byteLimit) {
byteLimit += state->bufferPos;
size_t oldLimit = state->currentLimit;
Expand Down
3 changes: 3 additions & 0 deletions objectivec/GPBCodedInputStream_PackagePrivate.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,9 @@ NSData *GPBCodedInputStreamReadRetainedBytes(GPBCodedInputStreamState *state)
__attribute((ns_returns_retained));
NSData *GPBCodedInputStreamReadRetainedBytesNoCopy(GPBCodedInputStreamState *state)
__attribute((ns_returns_retained));
NSData *GPBCodedInputStreamReadRetainedBytesToEndGroupNoCopy(GPBCodedInputStreamState *state,
int32_t fieldNumber)
__attribute((ns_returns_retained));

size_t GPBCodedInputStreamPushLimit(GPBCodedInputStreamState *state, size_t byteLimit);
void GPBCodedInputStreamPopLimit(GPBCodedInputStreamState *state, size_t oldLimit);
Expand Down
102 changes: 101 additions & 1 deletion objectivec/Tests/GPBCodedInputStreamTests.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,14 @@
// https://developers.google.com/open-source/licenses/bsd

#import <Foundation/Foundation.h>
#import "GPBTestUtilities.h"

#import "GPBCodedInputStream.h"
#import "GPBCodedInputStream_PackagePrivate.h"
#import "GPBCodedOutputStream.h"
#import "GPBTestUtilities.h"
#import "GPBUnknownFieldSet_PackagePrivate.h"
#import "GPBUtilities_PackagePrivate.h"
#import "GPBWireFormat.h"
#import "objectivec/Tests/Unittest.pbobjc.h"

@interface CodedInputStreamTests : GPBTestCase
Expand Down Expand Up @@ -444,4 +446,102 @@ - (void)testBOMWithinStrings {
}
}

- (void)assertReadByteToEndGroupFails:(NSData*)data {
GPBCodedInputStream* input = [GPBCodedInputStream streamWithData:data];
uint32_t tag = [input readTag];
XCTAssertThrows(GPBCodedInputStreamReadRetainedBytesToEndGroupNoCopy(
&input->state_, GPBWireFormatGetTagFieldNumber(tag)));
}

- (void)assertReadByteToEndGroup:(NSData*)data value:(NSData*)value {
GPBCodedInputStream* input = [GPBCodedInputStream streamWithData:data];
uint32_t tag = [input readTag];
NSData* readValue = GPBCodedInputStreamReadRetainedBytesToEndGroupNoCopy(
&input->state_, GPBWireFormatGetTagFieldNumber(tag));
XCTAssertNotNil(readValue);
XCTAssertEqualObjects(readValue, value);
[readValue release];
}

static NSData* DataForGroupsOfDepth(NSUInteger depth) {
NSMutableData* data = [NSMutableData dataWithCapacity:0];

uint32_t byte = 35; // 35 = 0b100011 -> field 4/start group
for (NSUInteger i = 0; i < depth; ++i) {
[data appendBytes:&byte length:1];
}

byte = 8; // 8 = 0b1000, -> field 1/varint
[data appendBytes:&byte length:1];
byte = 1; // 1 -> varint value of 1
[data appendBytes:&byte length:1];

byte = 36; // 36 = 0b100100 -> field 4/end group
for (NSUInteger i = 0; i < depth; ++i) {
[data appendBytes:&byte length:1];
}
return data;
}

- (void)testBytesToEndGroup {
// 35 = 0b100011 -> field 4/start group
// 36 = 0b100100 -> field 4/end group
// 43 = 0b101011 -> field 5/end group
// 44 = 0b101100 -> field 5/end group
// 8 = 0b1000, 1 -> field 1/varint, value of 1
// 21 = 0b10101, 0x78, 0x56, 0x34, 0x12 -> field 2/fixed32, value of 0x12345678
// 25 = 0b11001, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12 -> field 3/fixed64,
// value of 0x123456789abcdef0LL
// 50 = 0b110010, 0x0 -> field 6/length delimited, length 0
// 50 = 0b110010, 0x1, 42 -> field 6/length delimited, length 1, byte 42
// 0 -> field 0 which is invalid/varint
// 15 = 0b1111 -> field 1, wire type 7 which is invalid

[self assertReadByteToEndGroup:bytes(35, 36) value:bytes(36)]; // empty group
[self assertReadByteToEndGroup:bytes(35, 8, 1, 36) value:bytes(8, 1, 36)]; // varint
[self assertReadByteToEndGroup:bytes(35, 21, 0x78, 0x56, 0x34, 0x12, 36) // fixed32
value:bytes(21, 0x78, 0x56, 0x34, 0x12, 36)];
[self assertReadByteToEndGroup:bytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12,
36) // fixed64
value:bytes(25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12, 36)];
[self assertReadByteToEndGroup:bytes(35, 50, 0, 36)
value:bytes(50, 0, 36)]; // length delimited, length 0
[self assertReadByteToEndGroup:bytes(35, 50, 1, 42, 36)
value:bytes(50, 1, 42, 36)]; // length delimited, length 1, byte 42
[self assertReadByteToEndGroup:bytes(35, 43, 44, 36) value:bytes(43, 44, 36)]; // Sub group
[self assertReadByteToEndGroup:bytes(35, 8, 1, 43, 8, 1, 44,
36) // varint and sub group with varint
value:bytes(8, 1, 43, 8, 1, 44, 36)];

[self assertReadByteToEndGroupFails:bytes(35, 0, 36)]; // Invalid field number
[self assertReadByteToEndGroupFails:bytes(35, 15, 36)]; // Invalid wire type
[self assertReadByteToEndGroupFails:bytes(35, 21, 0x78, 0x56, 0x34)]; // truncated fixed32
[self assertReadByteToEndGroupFails:bytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56,
0x34)]; // truncated fixed64

// Mising end group
[self assertReadByteToEndGroupFails:bytes(35)];
[self assertReadByteToEndGroupFails:bytes(35, 8, 1)];
[self assertReadByteToEndGroupFails:bytes(35, 43)];
[self assertReadByteToEndGroupFails:bytes(35, 43, 8, 1)];

// Wrong end group
[self assertReadByteToEndGroupFails:bytes(35, 44)];
[self assertReadByteToEndGroupFails:bytes(35, 8, 1, 44)];
[self assertReadByteToEndGroupFails:bytes(35, 43, 36)];
[self assertReadByteToEndGroupFails:bytes(35, 43, 8, 1, 36)];
[self assertReadByteToEndGroupFails:bytes(35, 43, 44, 44)];
[self assertReadByteToEndGroupFails:bytes(35, 43, 8, 1, 44, 44)];

// This is the same limit as within GPBCodedInputStream.
const NSUInteger kDefaultRecursionLimit = 100;
// That depth parses.
NSData* testData = DataForGroupsOfDepth(kDefaultRecursionLimit);
[self assertReadByteToEndGroup:testData
value:[testData subdataWithRange:NSMakeRange(1, testData.length - 1)]];
// One more level deep fails.
testData = DataForGroupsOfDepth(kDefaultRecursionLimit + 1);
[self assertReadByteToEndGroupFails:testData];
}

@end
175 changes: 175 additions & 0 deletions objectivec/Tests/GPBUnknownFieldSetTest.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
// https://developers.google.com/open-source/licenses/bsd

#import "GPBTestUtilities.h"
#import "GPBUnknownFieldSet.h"

#import "GPBUnknownFieldSet_PackagePrivate.h"
#import "GPBUnknownField_PackagePrivate.h"
Expand Down Expand Up @@ -376,6 +377,180 @@ - (void)testLargeVarint {
XCTAssertEqual(0x7FFFFFFFFFFFFFFFULL, [field2.varintList valueAtIndex:0]);
}

static NSData* DataForGroupsOfDepth(NSUInteger depth) {
NSMutableData* data = [NSMutableData dataWithCapacity:0];

uint32_t byte = 35; // 35 = 0b100011 -> field 4/start group
for (NSUInteger i = 0; i < depth; ++i) {
[data appendBytes:&byte length:1];
}

byte = 8; // 8 = 0b1000, -> field 1/varint
[data appendBytes:&byte length:1];
byte = 1; // 1 -> varint value of 1
[data appendBytes:&byte length:1];

byte = 36; // 36 = 0b100100 -> field 4/end group
for (NSUInteger i = 0; i < depth; ++i) {
[data appendBytes:&byte length:1];
}
return data;
}

- (void)testParsingNestingGroupData {
// 35 = 0b100011 -> field 4/start group
// 36 = 0b100100 -> field 4/end group
// 43 = 0b101011 -> field 5/end group
// 44 = 0b101100 -> field 5/end group
// 8 = 0b1000, 1 -> field 1/varint, value of 1
// 21 = 0b10101, 0x78, 0x56, 0x34, 0x12 -> field 2/fixed32, value of 0x12345678
// 25 = 0b11001, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12 -> field 3/fixed64,
// value of 0x123456789abcdef0LL
// 50 = 0b110010, 0x0 -> field 6/length delimited, length 0
// 50 = 0b110010, 0x1, 42 -> field 6/length delimited, length 1, byte 42
// 0 -> field 0 which is invalid/varint
// 15 = 0b1111 -> field 1, wire type 7 which is invalid

TestEmptyMessage* m = [TestEmptyMessage parseFromData:DataFromBytes(35, 36)
error:NULL]; // empty group
XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1);
GPBUnknownField* field = [m.unknownFields getField:4];
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
GPBUnknownFieldSet* group = field.groupList[0];
XCTAssertEqual(group.countOfFields, (NSUInteger)0);

m = [TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1, 36) error:NULL]; // varint
XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1);
field = [m.unknownFields getField:4];
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
group = field.groupList[0];
field = [group getField:1];
XCTAssertEqual(field.varintList.count, (NSUInteger)1);
XCTAssertEqual([field.varintList valueAtIndex:0], 1);

m = [TestEmptyMessage parseFromData:DataFromBytes(35, 21, 0x78, 0x56, 0x34, 0x12, 36)
error:NULL]; // fixed32
XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1);
field = [m.unknownFields getField:4];
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
group = field.groupList[0];
field = [group getField:2];
XCTAssertEqual(field.fixed32List.count, (NSUInteger)1);
XCTAssertEqual([field.fixed32List valueAtIndex:0], 0x12345678);

m = [TestEmptyMessage
parseFromData:DataFromBytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12,
36)
error:NULL]; // fixed64
XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1);
field = [m.unknownFields getField:4];
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
group = field.groupList[0];
field = [group getField:3];
XCTAssertEqual(field.fixed64List.count, (NSUInteger)1);
XCTAssertEqual([field.fixed64List valueAtIndex:0], 0x123456789abcdef0LL);

m = [TestEmptyMessage parseFromData:DataFromBytes(35, 50, 0, 36)
error:NULL]; // length delimited, length 0
XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1);
field = [m.unknownFields getField:4];
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
group = field.groupList[0];
field = [group getField:6];
XCTAssertEqual(field.lengthDelimitedList.count, (NSUInteger)1);
XCTAssertEqualObjects(field.lengthDelimitedList[0], [NSData data]);

m = [TestEmptyMessage parseFromData:DataFromBytes(35, 50, 1, 42, 36)
error:NULL]; // length delimited, length 1, byte 42
XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1);
field = [m.unknownFields getField:4];
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
group = field.groupList[0];
field = [group getField:6];
XCTAssertEqual(field.lengthDelimitedList.count, (NSUInteger)1);
XCTAssertEqualObjects(field.lengthDelimitedList[0], DataFromBytes(42));

m = [TestEmptyMessage parseFromData:DataFromBytes(35, 43, 44, 36) error:NULL]; // Sub group
field = [m.unknownFields getField:4];
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
group = field.groupList[0];
XCTAssertEqual(group.countOfFields, (NSUInteger)1);
field = [group getField:5];
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
group = field.groupList[0];
XCTAssertEqual(group.countOfFields, (NSUInteger)0);

m = [TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1, 43, 8, 2, 44, 36)
error:NULL]; // varint and sub group with varint
XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1);
field = [m.unknownFields getField:4];
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
group = field.groupList[0];
XCTAssertEqual(group.countOfFields, (NSUInteger)2);
field = [group getField:1];
XCTAssertEqual(field.varintList.count, (NSUInteger)1);
XCTAssertEqual([field.varintList valueAtIndex:0], 1);
field = [group getField:5];
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
group = field.groupList[0];
field = [group getField:1];
XCTAssertEqual(field.varintList.count, (NSUInteger)1);
XCTAssertEqual([field.varintList valueAtIndex:0], 2);

XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 0, 36)
error:NULL]); // Invalid field number
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 15, 36)
error:NULL]); // Invalid wire type
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 21, 0x78, 0x56, 0x34)
error:NULL]); // truncated fixed32
XCTAssertNil([TestEmptyMessage
parseFromData:DataFromBytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56,
0x34)
error:NULL]); // truncated fixed64

// Mising end group
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35) error:NULL]);
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1) error:NULL]);
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43) error:NULL]);
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 8, 1) error:NULL]);

// Wrong end group
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 44) error:NULL]);
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1, 44) error:NULL]);
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 36) error:NULL]);
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 8, 1, 36) error:NULL]);
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 44, 44) error:NULL]);
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 8, 1, 44, 44) error:NULL]);

// This is the same limit as within GPBCodedInputStream.
const NSUInteger kDefaultRecursionLimit = 100;
// That depth parses.
NSData* testData = DataForGroupsOfDepth(kDefaultRecursionLimit);
m = [TestEmptyMessage parseFromData:testData error:NULL];
XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1);
field = [m.unknownFields getField:4];
for (NSUInteger i = 0; i < kDefaultRecursionLimit; ++i) {
XCTAssertEqual(field.varintList.count, (NSUInteger)0);
XCTAssertEqual(field.fixed32List.count, (NSUInteger)0);
XCTAssertEqual(field.fixed64List.count, (NSUInteger)0);
XCTAssertEqual(field.lengthDelimitedList.count, (NSUInteger)0);
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
group = field.groupList[0];
XCTAssertEqual(group.countOfFields, (NSUInteger)1);
field = [group getField:(i < (kDefaultRecursionLimit - 1) ? 4 : 1)];
}
// field is of the inner most group
XCTAssertEqual(field.varintList.count, (NSUInteger)1);
XCTAssertEqual([field.varintList valueAtIndex:0], (NSUInteger)1);
XCTAssertEqual(field.fixed32List.count, (NSUInteger)0);
XCTAssertEqual(field.fixed64List.count, (NSUInteger)0);
XCTAssertEqual(field.lengthDelimitedList.count, (NSUInteger)0);
XCTAssertEqual(field.groupList.count, (NSUInteger)0);
// One more level deep fails.
testData = DataForGroupsOfDepth(kDefaultRecursionLimit + 1);
XCTAssertNil([TestEmptyMessage parseFromData:testData error:NULL]);
}

#pragma mark - Field tests
// Some tests directly on fields since the dictionary in FieldSet can gate
// testing some of these.
Expand Down
Loading

0 comments on commit 9b16ee4

Please sign in to comment.