Enable AV1, HEVC via RTMP to YouTube

UI/data/locale/en-US.ini

@@ -402,6 +402,7 @@ Output.ConnectFail.Title="Failed to connect"
 Output.ConnectFail.BadPath="Invalid Path or Connection URL. Please check your settings to confirm that they are valid."
 Output.ConnectFail.ConnectFailed="Failed to connect to server"
 Output.ConnectFail.InvalidStream="Could not access the specified channel or stream key, please double-check your stream key. If it is correct, there may be a problem connecting to the server."
+Output.ConnectFail.HdrDisabled="HDR output is currently disabled for this output."
 Output.ConnectFail.Error="An unexpected error occurred when trying to connect to the server. More information in the log file."
 Output.ConnectFail.Disconnected="Disconnected from server."
 

UI/window-basic-main.cpp

@@ -7282,6 +7282,10 @@ void OBSBasic::StreamingStop(int code, QString last_error)
 		encode_error = true;
 		break;
 
+	case OBS_OUTPUT_HDR_DISABLED:
+		errorDescription = Str("Output.ConnectFail.HdrDisabled");
+		break;
+
 	default:
 	case OBS_OUTPUT_ERROR:
 		use_last_error = true;

libobs/obs-defs.h

@@ -42,6 +42,7 @@
 #define OBS_OUTPUT_UNSUPPORTED -6
 #define OBS_OUTPUT_NO_SPACE -7
 #define OBS_OUTPUT_ENCODE_ERROR -8
+#define OBS_OUTPUT_HDR_DISABLED -9
 
 #define OBS_VIDEO_SUCCESS 0
 #define OBS_VIDEO_FAIL -1

libobs/obs-hevc.c

@@ -21,50 +21,6 @@
 #include "obs-nal.h"
 #include "util/array-serializer.h"
 
-enum {
-	OBS_HEVC_NAL_TRAIL_N = 0,
-	OBS_HEVC_NAL_TRAIL_R = 1,
-	OBS_HEVC_NAL_TSA_N = 2,
-	OBS_HEVC_NAL_TSA_R = 3,
-	OBS_HEVC_NAL_STSA_N = 4,
-	OBS_HEVC_NAL_STSA_R = 5,
-	OBS_HEVC_NAL_RADL_N = 6,
-	OBS_HEVC_NAL_RADL_R = 7,
-	OBS_HEVC_NAL_RASL_N = 8,
-	OBS_HEVC_NAL_RASL_R = 9,
-	OBS_HEVC_NAL_VCL_N10 = 10,
-	OBS_HEVC_NAL_VCL_R11 = 11,
-	OBS_HEVC_NAL_VCL_N12 = 12,
-	OBS_HEVC_NAL_VCL_R13 = 13,
-	OBS_HEVC_NAL_VCL_N14 = 14,
-	OBS_HEVC_NAL_VCL_R15 = 15,
-	OBS_HEVC_NAL_BLA_W_LP = 16,
-	OBS_HEVC_NAL_BLA_W_RADL = 17,
-	OBS_HEVC_NAL_BLA_N_LP = 18,
-	OBS_HEVC_NAL_IDR_W_RADL = 19,
-	OBS_HEVC_NAL_IDR_N_LP = 20,
-	OBS_HEVC_NAL_CRA_NUT = 21,
-	OBS_HEVC_NAL_RSV_IRAP_VCL22 = 22,
-	OBS_HEVC_NAL_RSV_IRAP_VCL23 = 23,
-	OBS_HEVC_NAL_RSV_VCL24 = 24,
-	OBS_HEVC_NAL_RSV_VCL25 = 25,
-	OBS_HEVC_NAL_RSV_VCL26 = 26,
-	OBS_HEVC_NAL_RSV_VCL27 = 27,
-	OBS_HEVC_NAL_RSV_VCL28 = 28,
-	OBS_HEVC_NAL_RSV_VCL29 = 29,
-	OBS_HEVC_NAL_RSV_VCL30 = 30,
-	OBS_HEVC_NAL_RSV_VCL31 = 31,
-	OBS_HEVC_NAL_VPS = 32,
-	OBS_HEVC_NAL_SPS = 33,
-	OBS_HEVC_NAL_PPS = 34,
-	OBS_HEVC_NAL_AUD = 35,
-	OBS_HEVC_NAL_EOS_NUT = 36,
-	OBS_HEVC_NAL_EOB_NUT = 37,
-	OBS_HEVC_NAL_FD_NUT = 38,
-	OBS_HEVC_NAL_SEI_PREFIX = 39,
-	OBS_HEVC_NAL_SEI_SUFFIX = 40,
-};
-
 bool obs_hevc_keyframe(const uint8_t *data, size_t size)
 {
 	const uint8_t *nal_start, *nal_end;

libobs/obs-hevc.h

@@ -25,6 +25,50 @@ extern "C" {
 
 struct encoder_packet;
 
+enum {
+	OBS_HEVC_NAL_TRAIL_N = 0,
+	OBS_HEVC_NAL_TRAIL_R = 1,
+	OBS_HEVC_NAL_TSA_N = 2,
+	OBS_HEVC_NAL_TSA_R = 3,
+	OBS_HEVC_NAL_STSA_N = 4,
+	OBS_HEVC_NAL_STSA_R = 5,
+	OBS_HEVC_NAL_RADL_N = 6,
+	OBS_HEVC_NAL_RADL_R = 7,
+	OBS_HEVC_NAL_RASL_N = 8,
+	OBS_HEVC_NAL_RASL_R = 9,
+	OBS_HEVC_NAL_VCL_N10 = 10,
+	OBS_HEVC_NAL_VCL_R11 = 11,
+	OBS_HEVC_NAL_VCL_N12 = 12,
+	OBS_HEVC_NAL_VCL_R13 = 13,
+	OBS_HEVC_NAL_VCL_N14 = 14,
+	OBS_HEVC_NAL_VCL_R15 = 15,
+	OBS_HEVC_NAL_BLA_W_LP = 16,
+	OBS_HEVC_NAL_BLA_W_RADL = 17,
+	OBS_HEVC_NAL_BLA_N_LP = 18,
+	OBS_HEVC_NAL_IDR_W_RADL = 19,
+	OBS_HEVC_NAL_IDR_N_LP = 20,
+	OBS_HEVC_NAL_CRA_NUT = 21,
+	OBS_HEVC_NAL_RSV_IRAP_VCL22 = 22,
+	OBS_HEVC_NAL_RSV_IRAP_VCL23 = 23,
+	OBS_HEVC_NAL_RSV_VCL24 = 24,
+	OBS_HEVC_NAL_RSV_VCL25 = 25,
+	OBS_HEVC_NAL_RSV_VCL26 = 26,
+	OBS_HEVC_NAL_RSV_VCL27 = 27,
+	OBS_HEVC_NAL_RSV_VCL28 = 28,
+	OBS_HEVC_NAL_RSV_VCL29 = 29,
+	OBS_HEVC_NAL_RSV_VCL30 = 30,
+	OBS_HEVC_NAL_RSV_VCL31 = 31,
+	OBS_HEVC_NAL_VPS = 32,
+	OBS_HEVC_NAL_SPS = 33,
+	OBS_HEVC_NAL_PPS = 34,
+	OBS_HEVC_NAL_AUD = 35,
+	OBS_HEVC_NAL_EOS_NUT = 36,
+	OBS_HEVC_NAL_EOB_NUT = 37,
+	OBS_HEVC_NAL_FD_NUT = 38,
+	OBS_HEVC_NAL_SEI_PREFIX = 39,
+	OBS_HEVC_NAL_SEI_SUFFIX = 40,
+};
+
 EXPORT bool obs_hevc_keyframe(const uint8_t *data, size_t size);
 EXPORT void obs_parse_hevc_packet(struct encoder_packet *hevc_packet,
 				  const struct encoder_packet *src);

plugins/obs-outputs/CMakeLists.txt

@@ -28,6 +28,9 @@ target_sources(
           rtmp-stream.c
           rtmp-stream.h
           rtmp-windows.c
+          rtmp-av1.c
+          rtmp-av1.h
+          utils.h
           librtmp/amf.c
           librtmp/amf.h
           librtmp/bytes.h
@@ -45,6 +48,10 @@ target_sources(
           librtmp/rtmp.h
           librtmp/rtmp_sys.h)
 
+if(ENABLE_HEVC)
+  target_sources(obs-outputs PRIVATE rtmp-hevc.c rtmp-hevc.h)
+endif()
+
 target_link_libraries(obs-outputs PRIVATE OBS::libobs)
 
 set_target_properties(obs-outputs PROPERTIES FOLDER "plugins" PREFIX "")

plugins/obs-outputs/flv-mux.c

@@ -32,6 +32,66 @@
 #define VIDEODATA_AVCVIDEOPACKET 7.0
 #define AUDIODATA_AAC 10.0
 
+#define VIDEO_FRAMETYPE_OFFSET 4
+enum video_frametype_t {
+	FT_KEY = 1 << VIDEO_FRAMETYPE_OFFSET,
+	FT_INTER = 2 << VIDEO_FRAMETYPE_OFFSET,
+};
+
+// Y2023 spec
+const uint8_t FRAME_HEADER_EX = 8 << VIDEO_FRAMETYPE_OFFSET;
+enum packet_type_t {
+	PACKETTYPE_SEQ_START = 0,
+	PACKETTYPE_FRAMES = 1,
+	PACKETTYPE_SEQ_END = 2,
+#ifdef ENABLE_HEVC
+	PACKETTYPE_FRAMESX = 3,
+#endif
+	PACKETTYPE_METADATA = 4
+};
+
+enum datatype_t {
+	DATA_TYPE_NUMBER = 0,
+	DATA_TYPE_STRING = 2,
+	DATA_TYPE_OBJECT = 3,
+	DATA_TYPE_OBJECT_END = 9,
+};
+
+static void s_w4cc(struct serializer *s, enum video_id_t id)
+{
+	switch (id) {
+	case CODEC_AV1:
+		s_w8(s, 'a');
+		s_w8(s, 'v');
+		s_w8(s, '0');
+		s_w8(s, '1');
+		break;
+#ifdef ENABLE_HEVC
+	case CODEC_HEVC:
+		s_w8(s, 'h');
+		s_w8(s, 'v');
+		s_w8(s, 'c');
+		s_w8(s, '1');
+		break;
+#endif
+	case CODEC_H264:
+		assert(0);
+	}
+}
+
+static void s_wstring(struct serializer *s, const char *str)
+{
+	size_t len = strlen(str);
+	s_wb16(s, (uint16_t)len);
+	s_write(s, str, len);
+}
+
+static inline void s_wtimestamp(struct serializer *s, int32_t i32)
+{
+	s_wb24(s, (uint32_t)(i32 & 0xFFFFFF));
+	s_w8(s, (uint32_t)(i32 >> 24) & 0x7F);
+}
+
 static inline double encoder_bitrate(obs_encoder_t *encoder)
 {
 	obs_data_t *settings = obs_encoder_get_settings(encoder);
@@ -189,7 +249,7 @@ static void flv_video(struct serializer *s, int32_t dts_offset,
 #endif
 
 	s_wb24(s, (uint32_t)packet->size + 5);
-	s_wb24(s, time_ms);
+	s_wb24(s, (uint32_t)time_ms);
 	s_w8(s, (time_ms >> 24) & 0x7F);
 	s_wb24(s, 0);
 
@@ -223,7 +283,7 @@ static void flv_audio(struct serializer *s, int32_t dts_offset,
 #endif
 
 	s_wb24(s, (uint32_t)packet->size + 2);
-	s_wb24(s, time_ms);
+	s_wb24(s, (uint32_t)time_ms);
 	s_w8(s, (time_ms >> 24) & 0x7F);
 	s_wb24(s, 0);
 
@@ -253,6 +313,151 @@ void flv_packet_mux(struct encoder_packet *packet, int32_t dts_offset,
 	*size = data.bytes.num;
 }
 
+// Y2023 spec
+void flv_packet_ex(struct encoder_packet *packet, enum video_id_t codec_id,
+		   int32_t dts_offset, uint8_t **output, size_t *size, int type)
+{
+	struct array_output_data data;
+	struct serializer s;
+	array_output_serializer_init(&s, &data);
+
+	assert(packet->type == OBS_ENCODER_VIDEO);
+
+	int32_t time_ms = get_ms_time(packet, packet->dts) - dts_offset;
+
+	// packet head
+	s_w8(&s, RTMP_PACKET_TYPE_VIDEO);
+	s_wb24(&s, (uint32_t)packet->size + 5); // 5 = (w8+w4cc)
+	s_wtimestamp(&s, time_ms);
+	s_wb24(&s, 0); // always 0
+
+	// packet ext header
+	s_w8(&s, FRAME_HEADER_EX | type | (packet->keyframe ? FT_KEY : 0));
+	s_w4cc(&s, codec_id);
+	// packet data
+	s_write(&s, packet->data, packet->size);
+
+	// packet tail
+	s_wb32(&s, (uint32_t)serializer_get_pos(&s) - 1);
+
+	*output = data.bytes.array;
+	*size = data.bytes.num;
+}
+
+void flv_packet_start(struct encoder_packet *packet, enum video_id_t codec,
+		      int32_t dts_offset, uint8_t **output, size_t *size)
+{
+	flv_packet_ex(packet, codec, dts_offset, output, size,
+		      PACKETTYPE_SEQ_START);
+}
+
+void flv_packet_frames(struct encoder_packet *packet, enum video_id_t codec,
+		       int32_t dts_offset, uint8_t **output, size_t *size)
+{
+#ifdef ENABLE_HEVC
+	flv_packet_ex(packet, codec, dts_offset, output, size,
+		      (codec == CODEC_HEVC) ? PACKETTYPE_FRAMESX
+					    : PACKETTYPE_FRAMES);
+#else
+	flv_packet_ex(packet, dts_offset, output, size, PACKETTYPE_FRAMES);
+#endif
+}
+
+void flv_packet_end(struct encoder_packet *packet, enum video_id_t codec,
+		    int32_t dts_offset, uint8_t **output, size_t *size)
+{
+	flv_packet_ex(packet, codec, dts_offset, output, size,
+		      PACKETTYPE_SEQ_END);
+}
+
+void flv_packet_metadata(enum video_id_t codec_id, uint8_t **output,
+			 size_t *size, int bits_per_raw_sample,
+			 uint8_t color_primaries, int color_trc,
+			 int color_space, int min_luminance, int max_luminance)
+{
+	// metadata array
+	struct array_output_data data;
+	struct array_output_data metadata;
+	struct serializer s;
+	array_output_serializer_init(&s, &data);
+
+	// metadata data array
+	{
+		struct serializer s;
+		array_output_serializer_init(&s, &metadata);
+
+		s_w8(&s, DATA_TYPE_STRING);
+		s_wstring(&s, "colorInfo");
+		s_w8(&s, DATA_TYPE_OBJECT);
+		{
+			// colorConfig:
+			s_wstring(&s, "colorConfig");
+			s_w8(&s, DATA_TYPE_OBJECT);
+			{
+				s_wstring(&s, "bitDepth");
+				s_w8(&s, DATA_TYPE_NUMBER);
+				s_wbd(&s, bits_per_raw_sample);
+
+				s_wstring(&s, "colorPrimaries");
+				s_w8(&s, DATA_TYPE_NUMBER);
+				s_wbd(&s, color_primaries);
+
+				s_wstring(&s, "transferCharacteristics");
+				s_w8(&s, DATA_TYPE_NUMBER);
+				s_wbd(&s, color_trc);
+
+				s_wstring(&s, "matrixCoefficients");
+				s_w8(&s, DATA_TYPE_NUMBER);
+				s_wbd(&s, color_space);
+			}
+			s_w8(&s, 0);
+			s_w8(&s, 0);
+			s_w8(&s, DATA_TYPE_OBJECT_END);
+
+			if (max_luminance != 0) {
+				// hdrMdcv
+				s_wstring(&s, "hdrMdcv");
+				s_w8(&s, DATA_TYPE_OBJECT);
+				{
+					s_wstring(&s, "maxLuminance");
+					s_w8(&s, DATA_TYPE_NUMBER);
+					s_wbd(&s, max_luminance);
+
+					s_wstring(&s, "minLuminance");
+					s_w8(&s, DATA_TYPE_NUMBER);
+					s_wbd(&s, min_luminance);
+				}
+				s_w8(&s, 0);
+				s_w8(&s, 0);
+				s_w8(&s, DATA_TYPE_OBJECT_END);
+			}
+		}
+		s_w8(&s, 0);
+		s_w8(&s, 0);
+		s_w8(&s, DATA_TYPE_OBJECT_END);
+	}
+
+	// packet head
+	s_w8(&s, RTMP_PACKET_TYPE_VIDEO);
+	s_wb24(&s, (uint32_t)metadata.bytes.num + 5); // 5 = (w8+w4cc)
+	s_wtimestamp(&s, 0);
+	s_wb24(&s, 0); // always 0
+
+	// packet ext header
+	// these are the 5 extra bytes mentioned above
+	s_w8(&s, FRAME_HEADER_EX | PACKETTYPE_METADATA);
+	s_w4cc(&s, codec_id);
+	// packet data
+	s_write(&s, metadata.bytes.array, metadata.bytes.num);
+	array_output_serializer_free(&metadata); // must be freed
+
+	// packet tail
+	s_wb32(&s, (uint32_t)serializer_get_pos(&s) - 1);
+
+	*output = data.bytes.array;
+	*size = data.bytes.num;
+}
+
 /* ------------------------------------------------------------------------- */
 /* stuff for additional media streams                                        */
 
@@ -471,7 +676,7 @@ static void flv_additional_audio(struct serializer *s, int32_t dts_offset,
 #endif
 
 	s_wb24(s, (uint32_t)size);
-	s_wb24(s, time_ms);
+	s_wb24(s, (uint32_t)time_ms);
 	s_w8(s, (time_ms >> 24) & 0x7F);
 	s_wb24(s, 0);