drm/amd/display: Try RGB before YCbCr 4:4:4 in stream validationgrafted

amdgpu_dm_create_validate_stream_for_sink() walks encoding_order[] and
uses the first encoding that validates. YCbCr 4:4:4 is listed before
RGB, so an HDMI sink that advertises 4:4:4 gets YCbCr 4:4:4 whenever the
"color format" property is left at AUTO, even though RGB fits the same
link.

That contradicts the documented AUTO behaviour for HDMI in enum
drm_connector_color_format (RGB, falling back to YCbCr 4:2:0 only when
the bandwidth is not available or the mode is 4:2:0-only), which the
amdgpu implementation of the property also describes. It also leaves
the "Broadcast RGB" property without effect on such sinks, since the
quantization range it selects only applies to RGB output.

Try RGB first. The mask still holds every encoding the sink supports,
so a mode that cannot carry RGB falls back exactly as before.

For reference, v7.2 picked RGB here unless YCbCr 4:4:4 was forced
through debugfs, while earlier kernels picked YCbCr 4:4:4 for any HDMI
sink that advertised it.

Fixes: 0b0ff65d3ca1 ("drm/amd/display: Refactor stream validation")
Suggested-by: Adolfo Rodrigues <adolfotregosa@gmail.com>
Assisted-by: Claude Code:claude-fable-5-1
Signed-off-by: Adrian Betschart <adrian.betschart@cinemaone.ch>
Reviewed-by: Fangzhi Zuo <jerry.zuo@amd.com>
Tested-by: Adolfo Rodrigues <adolfotregosa@gmail.com>
Tested-by: Daniel Wheeler <daniel.wheeler@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
(cherry picked from commit 7de432bc753133764dc40d37f9388da56de251e7)

1 files changed, 315 insertions, 0 deletions

diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh
new file mode 100755
index 000000000..ab0b8125c
--- /dev/null
+++ b/scripts/link-vmlinux.sh
@@ -0,0 +1,315 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0
+#
+# link vmlinux
+#
+# vmlinux is linked from the objects in vmlinux.a and $(KBUILD_VMLINUX_LIBS).
+# vmlinux.a contains objects that are linked unconditionally.
+# $(KBUILD_VMLINUX_LIBS) are archives which are linked conditionally
+# (not within --whole-archive), and do not require symbol indexes added.
+#
+# vmlinux
+#   ^
+#   |
+#   +--< vmlinux.a
+#   |
+#   +--< $(KBUILD_VMLINUX_LIBS)
+#   |    +--< lib/lib.a + more
+#   |
+#   +-< ${kallsymso} (see description in KALLSYMS section)
+#
+# vmlinux version (uname -v) cannot be updated during normal
+# descending-into-subdirs phase since we do not yet know if we need to
+# update vmlinux.
+# Therefore this step is delayed until just before final link of vmlinux.
+#
+# System.map is generated to document addresses of all kernel symbols
+
+# Error out on error
+set -e
+
+LD="$1"
+KBUILD_LDFLAGS="$2"
+LDFLAGS_vmlinux="$3"
+VMLINUX="$4"
+
+is_enabled() {
+	grep -q "^$1=y" include/config/auto.conf
+}
+
+# Nice output in kbuild format
+# Will be suppressed by "make -s"
+info()
+{
+	printf "  %-7s %s\n" "${1}" "${2}"
+}
+
+# Link of vmlinux
+# ${1} - output file
+vmlinux_link()
+{
+	local output=${1}
+	local objs
+	local libs
+	local ld
+	local ldflags
+	local ldlibs
+
+	info LD ${output}
+
+	# skip output file argument
+	shift
+
+	if is_enabled CONFIG_LTO_CLANG || is_enabled CONFIG_X86_KERNEL_IBT ||
+	   is_enabled CONFIG_KLP_BUILD; then
+		# Use vmlinux.o instead of performing the slow LTO link again.
+		objs=vmlinux.o
+		libs=
+	else
+		objs=vmlinux.a
+		libs="${KBUILD_VMLINUX_LIBS}"
+	fi
+
+	if is_enabled CONFIG_GENERIC_BUILTIN_DTB; then
+		objs="${objs} .builtin-dtbs.o"
+	fi
+
+	objs="${objs} .vmlinux.export.o"
+	objs="${objs} init/version-timestamp.o"
+
+	if [ "${SRCARCH}" = "um" ]; then
+		wl=-Wl,
+		ld="${CC}"
+		ldflags="${CFLAGS_vmlinux}"
+		ldlibs="-lutil -lrt -lpthread"
+	else
+		wl=
+		ld="${LD}"
+		ldflags="${KBUILD_LDFLAGS} ${LDFLAGS_vmlinux}"
+		ldlibs=
+	fi
+
+	ldflags="${ldflags} ${wl}--script=${objtree}/${KBUILD_LDS}"
+
+	# The kallsyms linking does not need debug symbols included.
+	if [ -n "${strip_debug}" ] ; then
+		ldflags="${ldflags} ${wl}--strip-debug"
+	fi
+
+	if [ -n "${generate_map}" ];  then
+		ldflags="${ldflags} ${wl}-Map=vmlinux.map"
+	fi
+
+	${ld} ${ldflags} -o ${output}					\
+		${wl}--whole-archive ${objs} ${wl}--no-whole-archive	\
+		${wl}--start-group ${libs} ${wl}--end-group		\
+		${kallsymso} ${btf_vmlinux_bin_o} ${arch_vmlinux_o} ${ldlibs}
+}
+
+# Check if kallsymso_prev and kallsymso differ
+# If symbol sizes within ${kallsymso} change, any symbols within vmlinux are
+# likely to shift, invalidating ${kallsymso}.
+# Since file size can remain unchanged even if symbol sizes change, compare the
+# actual symbols instead of relying on file size only.
+kallsymso_changed()
+{
+	${NM} -n "${kallsymso_prev}" > "${kallsymso_prev}.sym"
+	${NM} -n "${kallsymso}" > "${kallsymso}.sym"
+	! cmp -s "${kallsymso_prev}.sym" "${kallsymso}.sym"
+}
+
+# Create ${2}.o file with all symbols from the ${1} object file
+kallsyms()
+{
+	local kallsymopt;
+
+	if is_enabled CONFIG_KALLSYMS_ALL; then
+		kallsymopt="${kallsymopt} --all-symbols"
+	fi
+
+	if is_enabled CONFIG_64BIT || is_enabled CONFIG_RELOCATABLE; then
+		kallsymopt="${kallsymopt} --pc-relative"
+	fi
+
+	info KSYMS "${2}.S"
+	scripts/kallsyms ${kallsymopt} "${1}" > "${2}.S"
+
+	info AS "${2}.o"
+	${CC} ${NOSTDINC_FLAGS} ${LINUXINCLUDE} ${KBUILD_CPPFLAGS} \
+	      ${KBUILD_AFLAGS} ${KBUILD_AFLAGS_KERNEL} -c -o "${2}.o" "${2}.S"
+
+	kallsymso_prev="${kallsymso:-}"
+	kallsymso=${2}.o
+}
+
+# Perform kallsyms for the given temporary vmlinux.
+sysmap_and_kallsyms()
+{
+	mksysmap "${1}" "${1}.syms"
+	kallsyms "${1}.syms" "${1}.kallsyms"
+
+	kallsyms_sysmap=${1}.syms
+}
+
+# Create map file with all symbols from ${1}
+# See mksymap for additional details
+mksysmap()
+{
+	info NM ${2}
+	${NM} -n "${1}" | sed -f "${srctree}/scripts/mksysmap" > "${2}"
+}
+
+sorttable()
+{
+	${NM} -S ${1} > .tmp_vmlinux.nm-sort
+	${objtree}/scripts/sorttable -s .tmp_vmlinux.nm-sort ${1}
+}
+
+cleanup()
+{
+	rm -f .btf.*
+	rm -f .tmp_vmlinux.nm-sort
+	rm -f System.map
+	rm -f vmlinux
+	rm -f vmlinux.map
+}
+
+# Use "make V=1" to debug this script
+case "${KBUILD_VERBOSE}" in
+*1*)
+	set -x
+	;;
+esac
+
+if [ "$1" = "clean" ]; then
+	cleanup
+	exit 0
+fi
+
+${MAKE} -f "${srctree}/scripts/Makefile.build" obj=init init/version-timestamp.o
+
+arch_vmlinux_o=
+if is_enabled CONFIG_ARCH_WANTS_PRE_LINK_VMLINUX; then
+	arch_vmlinux_o=arch/${SRCARCH}/tools/vmlinux.arch.o
+fi
+
+btf_vmlinux_bin_o=
+btfids_vmlinux=
+kallsymso=
+strip_debug=
+generate_map=
+
+# Use "make UT=1" to trigger warnings on unused tracepoints
+case "${WARN_ON_UNUSED_TRACEPOINTS}" in
+*1*)
+	${objtree}/scripts/tracepoint-update vmlinux.o
+	;;
+esac
+
+if is_enabled CONFIG_KALLSYMS; then
+	true > .tmp_vmlinux0.syms
+	kallsyms .tmp_vmlinux0.syms .tmp_vmlinux0.kallsyms
+fi
+
+if is_enabled CONFIG_KALLSYMS || is_enabled CONFIG_DEBUG_INFO_BTF; then
+
+	# The kallsyms linking does not need debug symbols, but the BTF does.
+	if ! is_enabled CONFIG_DEBUG_INFO_BTF; then
+		strip_debug=1
+	fi
+
+	vmlinux_link .tmp_vmlinux1
+fi
+
+if is_enabled CONFIG_DEBUG_INFO_BTF; then
+	info BTF .tmp_vmlinux1
+	if ! ${CONFIG_SHELL} ${srctree}/scripts/gen-btf.sh .tmp_vmlinux1; then
+		echo >&2 "Failed to generate BTF for vmlinux"
+		echo >&2 "Try to disable CONFIG_DEBUG_INFO_BTF"
+		exit 1
+	fi
+	btf_vmlinux_bin_o=.tmp_vmlinux1.btf.o
+	btfids_vmlinux=.tmp_vmlinux1.BTF_ids
+fi
+
+if is_enabled CONFIG_KALLSYMS; then
+
+	# kallsyms support
+	# Generate section listing all symbols and add it into vmlinux
+	# It's a four step process:
+	# 0)  Generate a dummy __kallsyms with empty symbol list.
+	# 1)  Link .tmp_vmlinux1.kallsyms so it has all symbols and sections,
+	#     with a dummy __kallsyms.
+	#     Running kallsyms on that gives us .tmp_vmlinux1.kallsyms.o with
+	#     the right size
+	# 2)  Link .tmp_vmlinux2.kallsyms so it now has a __kallsyms section of
+	#     the right size, but due to the added section, some
+	#     addresses have shifted.
+	#     From here, we generate a correct .tmp_vmlinux2.kallsyms.o
+	# 3)  That link may have expanded the kernel image enough that
+	#     more linker branch stubs / trampolines had to be added, which
+	#     introduces new names, which further expands kallsyms. Do another
+	#     pass if that is the case. In theory it's possible this results
+	#     in even more stubs, but unlikely.
+	#     KALLSYMS_EXTRA_PASS=1 may also used to debug or work around
+	#     other bugs.
+	# 4)  The correct ${kallsymso} is linked into the final vmlinux.
+	#
+	# a)  Verify that the System.map from vmlinux matches the map from
+	#     ${kallsymso}.
+
+	# The kallsyms linking does not need debug symbols included.
+	strip_debug=1
+
+	sysmap_and_kallsyms .tmp_vmlinux1
+	size1=$(${CONFIG_SHELL} "${srctree}/scripts/file-size.sh" ${kallsymso})
+
+	vmlinux_link .tmp_vmlinux2
+	sysmap_and_kallsyms .tmp_vmlinux2
+	size2=$(${CONFIG_SHELL} "${srctree}/scripts/file-size.sh" ${kallsymso})
+
+	# Due to alignment, file size of the kallsymso object file might remain
+	# unchanged even if individual symbols within change size. Changed
+	# symbol sizes can still shift other symbols, though. Therefore, don't
+	# rely on file size alone.
+	if [ $size1 -ne $size2 ] || kallsymso_changed || \
+			[ -n "${KALLSYMS_EXTRA_PASS}" ]; then
+		vmlinux_link .tmp_vmlinux3
+		sysmap_and_kallsyms .tmp_vmlinux3
+	fi
+fi
+
+strip_debug=
+
+if is_enabled CONFIG_VMLINUX_MAP; then
+	generate_map=1
+fi
+
+vmlinux_link "${VMLINUX}"
+
+if is_enabled CONFIG_DEBUG_INFO_BTF; then
+	info BTFIDS ${VMLINUX}
+	${RESOLVE_BTFIDS} --patch_btfids ${btfids_vmlinux} ${VMLINUX}
+fi
+
+mksysmap "${VMLINUX}" System.map
+
+if is_enabled CONFIG_BUILDTIME_TABLE_SORT; then
+	info SORTTAB "${VMLINUX}"
+	if ! sorttable "${VMLINUX}"; then
+		echo >&2 Failed to sort kernel tables
+		exit 1
+	fi
+fi
+
+# step a (see comment above)
+if is_enabled CONFIG_KALLSYMS; then
+	if ! cmp -s System.map "${kallsyms_sysmap}"; then
+		echo >&2 Inconsistent kallsyms data
+		echo >&2 'Try "make KALLSYMS_EXTRA_PASS=1" as a workaround'
+		exit 1
+	fi
+fi
+
+# For fixdep
+echo "${VMLINUX}: $0" > ".${VMLINUX}.d"