Fix an issue with H.264 format attribute comparison and fix an issue with...

Fix an issue with H.264 format attribute comparison and fix an issue with improper SDP being produced.

The H.264 format attribute module compares two format attribute structures to determine if they are
compatible or not. In some instances it was possible for this check to determine that both structures
were incompatible when they actually should be considered compatible. This check has now been made even
more permissive by assuming that if no attribute information is available the two structures are compatible.
If both structures contain attribute information a base level comparison of the H.264 IDC value is done to
see if they are compatible or not.

The above issue uncovered a secondary issue in chan_sip where the SDP being produced would be incorrect if
the formats were considered incompatible. This has now been fixed by checking that all information required
to produce the SDP is available instead of assuming it is.

(closes issue ASTERISK-20464)
Reported by: Leif Madsen
........

Merged revisions 373413 from http://svn.asterisk.org/svn/asterisk/branches/11


git-svn-id: https://origsvn.digium.com/svn/asterisk/trunk@373414 65c4cc65-6c06-0410-ace0-fbb531ad65f3

channels/chan_sip.c

@@ -12228,11 +12228,17 @@ static void add_codec_to_sdp(const struct sip_pvt *p,
 {
 	int rtp_code;
 	struct ast_format_list fmt;
+	const char *mime;
+	unsigned int rate;
 	if (debug)
 		ast_verbose("Adding codec %d (%s) to SDP\n", format->id, ast_getformatname(format));
-	if ((rtp_code = ast_rtp_codecs_payload_code(ast_rtp_instance_get_codecs(p->rtp), 1, format, 0)) == -1)
+
+	if (((rtp_code = ast_rtp_codecs_payload_code(ast_rtp_instance_get_codecs(p->rtp), 1, format, 0)) == -1) ||
+	    !(mime = ast_rtp_lookup_mime_subtype2(1, format, 0, ast_test_flag(&p->flags[0], SIP_G726_NONSTANDARD) ? AST_RTP_OPT_G726_NONSTANDARD : 0)) ||
+	    !(rate = ast_rtp_lookup_sample_rate2(1, format, 0))) {
 		return;
+	}
 	if (p->rtp) {
 		struct ast_codec_pref *pref = &ast_rtp_instance_get_codecs(p->rtp)->pref;
@@ -12240,10 +12246,7 @@ static void add_codec_to_sdp(const struct sip_pvt *p,
 	} else /* I don't see how you couldn't have p->rtp, but good to check for and error out if not there like earlier code */
 		return;
 	ast_str_append(m_buf, 0, " %d", rtp_code);
-	ast_str_append(a_buf, 0, "a=rtpmap:%d %s/%d\r\n",
-		rtp_code,
-		ast_rtp_lookup_mime_subtype2(1, format, 0, ast_test_flag(&p->flags[0], SIP_G726_NONSTANDARD) ? AST_RTP_OPT_G726_NONSTANDARD : 0),
-		ast_rtp_lookup_sample_rate2(1, format, 0));
+	ast_str_append(a_buf, 0, "a=rtpmap:%d %s/%d\r\n", rtp_code, mime, rate);
 	ast_format_sdp_generate(format, rtp_code, a_buf);
@@ -12289,6 +12292,8 @@ static void add_vcodec_to_sdp(const struct sip_pvt *p, struct ast_format *format
 			     int debug, int *min_packet_size)
 {
 	int rtp_code;
+	const char *subtype;
+	unsigned int rate;
 	if (!p->vrtp)
 		return;
@@ -12296,13 +12301,14 @@ static void add_vcodec_to_sdp(const struct sip_pvt *p, struct ast_format *format
 	if (debug)
 		ast_verbose("Adding video codec %d (%s) to SDP\n", format->id, ast_getformatname(format));
-	if ((rtp_code = ast_rtp_codecs_payload_code(ast_rtp_instance_get_codecs(p->vrtp), 1, format, 0)) == -1)
+	if (((rtp_code = ast_rtp_codecs_payload_code(ast_rtp_instance_get_codecs(p->vrtp), 1, format, 0)) == -1) ||
+	    !(subtype = ast_rtp_lookup_mime_subtype2(1, format, 0, 0)) ||
+	    !(rate = ast_rtp_lookup_sample_rate2(1, format, 0))) {
 		return;
+	}
 	ast_str_append(m_buf, 0, " %d", rtp_code);
-	ast_str_append(a_buf, 0, "a=rtpmap:%d %s/%d\r\n", rtp_code,
-		       ast_rtp_lookup_mime_subtype2(1, format, 0, 0),
-		       ast_rtp_lookup_sample_rate2(1, format, 0));
+	ast_str_append(a_buf, 0, "a=rtpmap:%d %s/%d\r\n", rtp_code, subtype, rate);
 	ast_format_sdp_generate(format, rtp_code, a_buf);
 }

res/res_format_attr_h264.c

@@ -74,11 +74,12 @@ enum h264_attr_keys {
 
 static enum ast_format_cmp_res h264_format_attr_cmp(const struct ast_format_attr *fattr1, const struct ast_format_attr *fattr2)
 {
-	unsigned int idc1 = fattr1->format_attr[H264_ATTR_KEY_PROFILE_IDC] ? fattr1->format_attr[H264_ATTR_KEY_PROFILE_IDC] : 0x42;
-	unsigned int idc2 = fattr2->format_attr[H264_ATTR_KEY_PROFILE_IDC] ? fattr2->format_attr[H264_ATTR_KEY_PROFILE_IDC] : 0x42;
+	if (!fattr1->format_attr[H264_ATTR_KEY_PROFILE_IDC] || !fattr2->format_attr[H264_ATTR_KEY_PROFILE_IDC] ||
+	    (fattr1->format_attr[H264_ATTR_KEY_PROFILE_IDC] == fattr2->format_attr[H264_ATTR_KEY_PROFILE_IDC])) {
+		return AST_FORMAT_CMP_EQUAL;
+	}
 
-	/* We are as permissive as possible to ensure the maximum number of calls succeed */
-	return (idc1 == idc2) ? AST_FORMAT_CMP_EQUAL : AST_FORMAT_CMP_NOT_EQUAL;
+	return AST_FORMAT_CMP_NOT_EQUAL;
 }
 
 static int h264_format_attr_get_joint(const struct ast_format_attr *fattr1, const struct ast_format_attr *fattr2, struct ast_format_attr *result)