Merge pull request elastic#360 from Shweta-Bhandare/addInternalFields

Added static fields.
harper-carroll · Jun 23, 2015 · cbc5f93 · cbc5f93
2 parents 7044bcb + 5206026
commit cbc5f93
Show file tree

Hide file tree

Showing 5 changed files with 108 additions and 0 deletions.
diff --git a/protofiles/DpiMsgLRproto.proto b/protofiles/DpiMsgLRproto.proto
@@ -83,6 +83,7 @@ message DpiMsgLRproto
    repeated EmailAttach emailAttachments = 49;
    repeated CustomField customFields = 50;
    // REMEMBER TO UPDATE resources/DpiMsgLRproto.protoheader as well
+   // REMEMBER TO UPDATE resources/StaticFields.csv as well
 
 repeated bytes framed_ipQ_PROTO_DIAMETER = 343; // QOSMOS:Q_PROTO_DIAMETER,Q_DIAMETER_FRAMED_IP
 repeated bytes applicationQ_PROTO_BASE = 344; // QOSMOS:Q_PROTO_BASE,Q_BASE_APPLICATION

diff --git a/resources/DpiMsgLRproto.protoheader b/resources/DpiMsgLRproto.protoheader
@@ -83,4 +83,5 @@ message DpiMsgLRproto
    repeated EmailAttach emailAttachments = 49;
    repeated CustomField customFields = 50;
    // REMEMBER TO UPDATE resources/DpiMsgLRproto.protoheader as well
+   // REMEMBER TO UPDATE resources/StaticFields.csv as well
 
diff --git a/resources/LuaProtoFieldsDesc.csv b/resources/LuaProtoFieldsDesc.csv
@@ -1,4 +1,54 @@
 Protocol Name, Long Protocol Name, Attribute Name, Full Attribute Name, Attribute Type, Attribute Description
+Internal, Internal, session, Session, string, "Session UUID."
+Internal, Internal, srcmac, SrcMAC, uint64, "Source MAC address."
+Internal, Internal, destmac, DestMAC, uint64, "Destination MAC address."
+Internal, Internal, srcip, SrcIP, uint32, "Source IP address."
+Internal, Internal, destip, DestIP, uint32, "Destination IP address."
+Internal, Internal, packetpath, PacketPath, string, "Packet path."
+Internal, Internal, flowsessioncount, FlowSessionCount, uint32, "Flow Session Count."
+Internal, Internal, srcport, SrcPort, uint32, "Source Port."
+Internal, Internal, destport, DestPort, uint32, "Destination Port."
+Internal, Internal, flowcompleted, FlowCompleted, bool, "Flow Completed flag."
+Internal, Internal, delay, Delay, string, "Delay."
+Internal, Internal, protocol, Protocol, uint32, "Protocol."
+Internal, Internal, totalpackets, TotalPackets, uint32, "Total packets in the session."
+Internal, Internal, timestart, TimeStart, uint64, "Start time of the flow."
+Internal, Internal, timeupdated, TimeUpdated, uint64, "Time updated."
+Internal, Internal, destbytes, DestBytes, uint64, "Destination bytes."
+Internal, Internal, srcbytes, SrcBytes, uint64, "Source bytes."
+Internal, Internal, flowtype, FlowType, FlowType, "Flow type."
+Internal, Internal, packetsdelta, PacketsDelta, uint64, "Packets delta between update."
+Internal, Internal, timedelta, TimeDelta, uint64, "Time delta between update."
+Internal, Internal, destbytesdelta, DestBytesDelta, uint64, "Destination byte delta between update."
+Internal, Internal, srcbytesdelta, SrcBytesDelta, uint64, "Source byte delta between update."
+Internal, Internal, customapplication, CustomApplication, bytes, "Custom Application."
+Internal, Internal, flowstate, FlowState, FlowState, "Flow State type."
+Internal, Internal, captured, Captured, bool, "Captured flag."
+Internal, Internal, childflownumber, ChildFlowNumber, uint32, "Child Flow number."
+Internal, Internal, totalbytes, TotalBytes, uint64, "Total bytes of the session."
+Internal, Internal, totalbytesdelta, TotalBytesDelta, uint64, "Total bytes delta between update."
+Internal, Internal, application, Application, string, "Application."
+Internal, Internal, applicationpath, ApplicationPath, string, "Application Path."
+Internal, Internal, duration, Duration, uint64, "Duration of the flow."
+Internal, Internal, messagesize, MessageSize, uint64, "Size of the DPI message."
+Internal, Internal, threadid, ThreadID, uint32, "Thread ID."
+Internal, Internal, fieldcount, FieldCount, uint64, "Total fields in DPI message."
+Internal, Internal, debugmessage, DebugMessage, string, "Debug message."
+Internal, Internal, applicationid, ApplicationID, uint32, "Application ID."
+Internal, Internal, latestupdate, LatestUpdate, bool, "Latest update flag."
+Internal, Internal, timeprevious, TimePrevious, uint64, "Time Previous."
+Internal, Internal, written, Written, bool, "Capture written flag."
+Internal, Internal, captureremoved, CaptureRemoved, bool, "Capture removed flag."
+Internal, Internal, srcip6, SrcIP6, uint32, "Source IP6 address."
+Internal, Internal, destip6, DestIP6, uint32, "Destination IP6 address."
+Internal, Internal, normalizedsyslogdata, NormalizedSyslogData, string, "Normalized Syslog data."
+Internal, Internal, timeend, TimeEnd, uint64, "Time End."
+Internal, Internal, headerwritten, HeaderWritten, bool, "Header written flag."
+Internal, Internal, connectionestablished, ConnectionEstablished, bool, "Connection Established flag."
+Internal, Internal, maxrepeatedfieldcount, MaxRepeatedFieldCount, uint32, "Maximum number of fields indexed by ElasticSearch."
+Internal, Internal, fieldcountindexed, FieldCountIndexed, uint32, "Field count indexed by ElasticSearch."
+Internal, Internal, emailAttachments, EmailAttachments, EmailAttach, "Email attachment structure."
+Internal, Internal, customfields, CustomFields, CustomField, "Custom Fields."
 0zz0, 0zz0.com, end, endq_proto_0zz0, Void, "Indicates the end of a top level event."
 0zz0, 0zz0.com, login, loginq_proto_0zz0, bytes, "User's login string."
 0zz0, 0zz0.com, password, passwordq_proto_0zz0, bytes, "User's password string."

diff --git a/resources/StaticFields.csv b/resources/StaticFields.csv
@@ -0,0 +1,50 @@
+Internal, Internal, session, Session, string, "Session UUID."
+Internal, Internal, srcmac, SrcMAC, uint64, "Source MAC address."
+Internal, Internal, destmac, DestMAC, uint64, "Destination MAC address."
+Internal, Internal, srcip, SrcIP, uint32, "Source IP address."
+Internal, Internal, destip, DestIP, uint32, "Destination IP address."
+Internal, Internal, packetpath, PacketPath, string, "Packet path."
+Internal, Internal, flowsessioncount, FlowSessionCount, uint32, "Flow Session Count."
+Internal, Internal, srcport, SrcPort, uint32, "Source Port."
+Internal, Internal, destport, DestPort, uint32, "Destination Port."
+Internal, Internal, flowcompleted, FlowCompleted, bool, "Flow Completed flag."
+Internal, Internal, delay, Delay, string, "Delay."
+Internal, Internal, protocol, Protocol, uint32, "Protocol."
+Internal, Internal, totalpackets, TotalPackets, uint32, "Total packets in the session."
+Internal, Internal, timestart, TimeStart, uint64, "Start time of the flow."
+Internal, Internal, timeupdated, TimeUpdated, uint64, "Time updated."
+Internal, Internal, destbytes, DestBytes, uint64, "Destination bytes."
+Internal, Internal, srcbytes, SrcBytes, uint64, "Source bytes."
+Internal, Internal, flowtype, FlowType, FlowType, "Flow type."
+Internal, Internal, packetsdelta, PacketsDelta, uint64, "Packets delta between update."
+Internal, Internal, timedelta, TimeDelta, uint64, "Time delta between update."
+Internal, Internal, destbytesdelta, DestBytesDelta, uint64, "Destination byte delta between update."
+Internal, Internal, srcbytesdelta, SrcBytesDelta, uint64, "Source byte delta between update."
+Internal, Internal, customapplication, CustomApplication, bytes, "Custom Application."
+Internal, Internal, flowstate, FlowState, FlowState, "Flow State type."
+Internal, Internal, captured, Captured, bool, "Captured flag."
+Internal, Internal, childflownumber, ChildFlowNumber, uint32, "Child Flow number."
+Internal, Internal, totalbytes, TotalBytes, uint64, "Total bytes of the session."
+Internal, Internal, totalbytesdelta, TotalBytesDelta, uint64, "Total bytes delta between update."
+Internal, Internal, application, Application, string, "Application."
+Internal, Internal, applicationpath, ApplicationPath, string, "Application Path."
+Internal, Internal, duration, Duration, uint64, "Duration of the flow."
+Internal, Internal, messagesize, MessageSize, uint64, "Size of the DPI message."
+Internal, Internal, threadid, ThreadID, uint32, "Thread ID."
+Internal, Internal, fieldcount, FieldCount, uint64, "Total fields in DPI message."
+Internal, Internal, debugmessage, DebugMessage, string, "Debug message."
+Internal, Internal, applicationid, ApplicationID, uint32, "Application ID."
+Internal, Internal, latestupdate, LatestUpdate, bool, "Latest update flag."
+Internal, Internal, timeprevious, TimePrevious, uint64, "Time Previous."
+Internal, Internal, written, Written, bool, "Capture written flag."
+Internal, Internal, captureremoved, CaptureRemoved, bool, "Capture removed flag."
+Internal, Internal, srcip6, SrcIP6, uint32, "Source IP6 address."
+Internal, Internal, destip6, DestIP6, uint32, "Destination IP6 address."
+Internal, Internal, normalizedsyslogdata, NormalizedSyslogData, string, "Normalized Syslog data."
+Internal, Internal, timeend, TimeEnd, uint64, "Time End."
+Internal, Internal, headerwritten, HeaderWritten, bool, "Header written flag."
+Internal, Internal, connectionestablished, ConnectionEstablished, bool, "Connection Established flag."
+Internal, Internal, maxrepeatedfieldcount, MaxRepeatedFieldCount, uint32, "Maximum number of fields indexed by ElasticSearch."
+Internal, Internal, fieldcountindexed, FieldCountIndexed, uint32, "Field count indexed by ElasticSearch."
+Internal, Internal, emailAttachments, EmailAttachments, EmailAttach, "Email attachment structure."
+Internal, Internal, customfields, CustomFields, CustomField, "Custom Fields."
diff --git a/scripts/buildDpiMsgLRProto.pl b/scripts/buildDpiMsgLRProto.pl
@@ -282,6 +282,12 @@ sub CreateRemappingFile {
 seek luaProtoFieldsCsv, 0, 0;
 print luaProtoFieldsCsv "Protocol Name, Long Protocol Name, Attribute Name, Full Attribute Name, Attribute Type, Attribute Description\n";
 
+# Add static fields to the Lua Protocol description file.
+open (staticFieldsCsv, "<", "resources/StaticFields.csv") or die 'Could not open resources/StaticFields.csv';
+while ( my $line = <staticFieldsCsv> ) {
+   print luaProtoFieldsCsv $line;
+}
+
 seek qosmosWorkbook, 0, 0;
 while (<qosmosWorkbook>) {
    # Include all attributes matching the includeFilter, but exclude the 19 attributes at the beginning