[Cocoa] AV1 codec strings specifying 4:2:0 Colocated subsampling inco…

…rrectly fail

https://bugs.webkit.org/show_bug.cgi?id=278676
rdar://134375956

Reviewed by Eric Carlson.

Correct a mismatch between CoreMedia's definition of the ChromaSubsampling parameter
and the definition used by the AV1 ISO-BMFF bindings specification. CoreMedia uses
a definition whose low-order digit specifies monochrome support, and AV1 uses chroma
sample position in that same digit.

Construct a query paratemer which matches CoreMedia's definition from the codecs
string provided. Also, do more validation of combinations of monochrome and subsampling
coordinates.

* LayoutTests/media/av1-codec-validate-configuration-record-expected.txt: Added.
* LayoutTests/media/av1-codec-validate-configuration-record.html: Added.
* Source/WebCore/platform/graphics/AV1Utilities.cpp:
(WebCore::validateAV1ConfigurationRecord):
* Source/WebCore/platform/graphics/AV1Utilities.h:
* Source/WebCore/platform/graphics/cocoa/AV1UtilitiesCocoa.mm:
(WebCore::validateAV1Parameters):
* Source/WebCore/testing/Internals.cpp:
(WebCore::Internals::validateAV1ConfigurationRecord):

Canonical link: https://commits.webkit.org/282806@main

LayoutTests/media/av1-codec-validate-configuration-record-expected.txt

@@ -0,0 +1,52 @@
+* Main Profile
+Only 8 or 10 bit color is supported:
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.0.00M.08") == 'true') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.0.00M.10") == 'true') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.0.00M.12") == 'false') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.0.00M.09") == 'false') OK
+Only 4:2:0 subsampling is supported:
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.0.00M.08.0.000.1.1.1.0") == 'false') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.0.00M.08.0.100.1.1.1.0") == 'false') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.0.00M.08.0.110.1.1.1.0") == 'true') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.0.00M.08.0.111.1.1.1.0") == 'true') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.0.00M.08.0.112.1.1.1.0") == 'true') OK
+
+* High Profile
+Only 8 or 10 bit color is supported:
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.1.00M.08.0.000.1.1.1.0") == 'true') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.1.00M.10.0.000.1.1.1.0") == 'true') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.1.00M.12.0.000.1.1.1.0") == 'false') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.1.00M.09.0.000.1.1.1.0") == 'false') OK
+Only 4:4:4 subsampling is supported:
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.1.00M.08.0.000.1.1.1.0") == 'true') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.1.00M.08.0.100.1.1.1.0") == 'false') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.1.00M.08.0.110.1.1.1.0") == 'false') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.1.00M.08.0.111.1.1.1.0") == 'false') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.1.00M.08.0.112.1.1.1.0") == 'false') OK
+Monochrome is not supported:
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.1.00M.08.0.000.1.1.1.0") == 'true') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.1.00M.08.1.000.1.1.1.0") == 'false') OK
+
+* Professional Profile
+Only 8, 10, or 12 bit color is supported:
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.2.00M.08.0.000.1.1.1.0") == 'true') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.2.00M.10.0.000.1.1.1.0") == 'true') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.2.00M.12.0.000.1.1.1.0") == 'true') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.2.00M.09.0.000.1.1.1.0") == 'false') OK
+For 8 and 10 bit color, only 4:4:4 subsampling is supported:
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.2.00M.08.0.000.1.1.1.0") == 'true') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.2.00M.08.0.100.1.1.1.0") == 'false') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.2.00M.08.0.110.1.1.1.0") == 'false') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.2.00M.08.0.111.1.1.1.0") == 'false') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.2.00M.08.0.112.1.1.1.0") == 'false') OK
+For 12 bit color, only all subsampling values are supported:
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.2.00M.12.0.000.1.1.1.0") == 'true') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.2.00M.12.0.100.1.1.1.0") == 'true') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.2.00M.12.0.110.1.1.1.0") == 'true') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.2.00M.12.0.111.1.1.1.0") == 'true') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.2.00M.12.0.112.1.1.1.0") == 'true') OK
+Monochrome is supported for 12 bit color:
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.2.00M.12.0.000.1.1.1.0") == 'true') OK
+EXPECTED (internals.validateAV1ConfigurationRecord("av01.2.00M.12.1.110.1.1.1.0") == 'true') OK
+END OF TEST
+

LayoutTests/media/av1-codec-validate-configuration-record.html

@@ -0,0 +1,65 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <script src="video-test.js"></script>
+    <script>
+    window.addEventListener('load', event => {
+        consoleWrite('* Main Profile');
+        consoleWrite('Only 8 or 10 bit color is supported:')
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.0.00M.08")`, true);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.0.00M.10")`, true);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.0.00M.12")`, false);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.0.00M.09")`, false);
+        consoleWrite('Only 4:2:0 subsampling is supported:')
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.0.00M.08.0.000.1.1.1.0")`, false);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.0.00M.08.0.100.1.1.1.0")`, false);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.0.00M.08.0.110.1.1.1.0")`, true);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.0.00M.08.0.111.1.1.1.0")`, true);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.0.00M.08.0.112.1.1.1.0")`, true);
+
+        consoleWrite('');
+        consoleWrite('* High Profile');
+        consoleWrite('Only 8 or 10 bit color is supported:')
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.1.00M.08.0.000.1.1.1.0")`, true);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.1.00M.10.0.000.1.1.1.0")`, true);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.1.00M.12.0.000.1.1.1.0")`, false);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.1.00M.09.0.000.1.1.1.0")`, false);
+        consoleWrite('Only 4:4:4 subsampling is supported:')
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.1.00M.08.0.000.1.1.1.0")`, true);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.1.00M.08.0.100.1.1.1.0")`, false);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.1.00M.08.0.110.1.1.1.0")`, false);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.1.00M.08.0.111.1.1.1.0")`, false);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.1.00M.08.0.112.1.1.1.0")`, false);
+        consoleWrite('Monochrome is not supported:');
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.1.00M.08.0.000.1.1.1.0")`, true);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.1.00M.08.1.000.1.1.1.0")`, false);
+
+        consoleWrite('');
+        consoleWrite('* Professional Profile');
+        consoleWrite('Only 8, 10, or 12 bit color is supported:')
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.2.00M.08.0.000.1.1.1.0")`, true);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.2.00M.10.0.000.1.1.1.0")`, true);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.2.00M.12.0.000.1.1.1.0")`, true);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.2.00M.09.0.000.1.1.1.0")`, false);
+        consoleWrite('For 8 and 10 bit color, only 4:4:4 subsampling is supported:')
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.2.00M.08.0.000.1.1.1.0")`, true);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.2.00M.08.0.100.1.1.1.0")`, false);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.2.00M.08.0.110.1.1.1.0")`, false);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.2.00M.08.0.111.1.1.1.0")`, false);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.2.00M.08.0.112.1.1.1.0")`, false);
+        consoleWrite('For 12 bit color, only all subsampling values are supported:')
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.2.00M.12.0.000.1.1.1.0")`, true);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.2.00M.12.0.100.1.1.1.0")`, true);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.2.00M.12.0.110.1.1.1.0")`, true);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.2.00M.12.0.111.1.1.1.0")`, true);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.2.00M.12.0.112.1.1.1.0")`, true);
+        consoleWrite('Monochrome is supported for 12 bit color:');
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.2.00M.12.0.000.1.1.1.0")`, true);
+        testExpected(`internals.validateAV1ConfigurationRecord("av01.2.00M.12.1.110.1.1.1.0")`, true);
+        endTest();
+    }, { once: true });
+    </script>
+</head>
+<body>
+</body>
+</html>

Source/WebCore/platform/graphics/AV1Utilities.cpp

@@ -326,6 +326,52 @@ static const AV1PerLevelConstraintsMap& perLevelConstraints()
     return perLevelConstraints;
 }
 
+bool validateAV1ConfigurationRecord(const AV1CodecConfigurationRecord& record)
+{
+    // Ref: https://aomediacodec.github.io/av1-spec/av1-spec.pdf
+
+    // 6.4.1.General sequence header OBU semantics
+
+    if (!isValidEnum<AV1ConfigurationChromaSubsampling>(record.chromaSubsampling))
+        return false;
+    auto chromaSubsampling = static_cast<AV1ConfigurationChromaSubsampling>(record.chromaSubsampling);
+
+    switch (record.profile) {
+    case AV1ConfigurationProfile::Main:
+        if (record.bitDepth != 8 && record.bitDepth != 10)
+            return false;
+        if (chromaSubsampling != AV1ConfigurationChromaSubsampling::Subsampling_420_Unknown
+            && chromaSubsampling != AV1ConfigurationChromaSubsampling::Subsampling_420_Vertical
+            && chromaSubsampling != AV1ConfigurationChromaSubsampling::Subsampling_420_Colocated)
+            return false;
+        break;
+    case AV1ConfigurationProfile::High:
+        if (record.bitDepth != 8 && record.bitDepth != 10)
+            return false;
+        if (record.monochrome)
+            return false;
+        if (chromaSubsampling != AV1ConfigurationChromaSubsampling::Subsampling_444)
+            return false;
+        break;
+    case AV1ConfigurationProfile::Professional:
+        if (record.bitDepth == 8 || record.bitDepth == 10) {
+            if (chromaSubsampling != AV1ConfigurationChromaSubsampling::Subsampling_444)
+                return false;
+        } else if (record.bitDepth != 12)
+            return false;
+        break;
+    }
+
+    // 6.4.2. Color config semantics
+    // When monochrome is set to 1, the only valid setting for subsampling_x and subsampling_y
+    // is 1 and 1. Additionally, when monochrome is set to 1 in the color_config of the Sequence OBU,
+    // the only valid setting for chroma_sample_position is CSP_UNKNOWN (0).
+    if (record.monochrome && chromaSubsampling != AV1ConfigurationChromaSubsampling::Subsampling_420_Unknown)
+        return false;
+
+    return true;
+}
+
 bool validateAV1PerLevelConstraints(const AV1CodecConfigurationRecord& record, const VideoConfiguration& configuration)
 {
     // Check that VideoConfiguration is within the specified profile and level from the configuration record:

Source/WebCore/platform/graphics/AV1Utilities.h

@@ -172,6 +172,7 @@ struct VideoInfo;
 WEBCORE_EXPORT std::optional<AV1CodecConfigurationRecord> parseAV1CodecParameters(StringView codecString);
 WEBCORE_EXPORT String createAV1CodecParametersString(const AV1CodecConfigurationRecord&);
 
+WEBCORE_EXPORT bool validateAV1ConfigurationRecord(const AV1CodecConfigurationRecord&);
 WEBCORE_EXPORT bool validateAV1PerLevelConstraints(const AV1CodecConfigurationRecord&, const VideoConfiguration&);
 
 std::optional<AV1CodecConfigurationRecord> parseAV1DecoderConfigurationRecord(const SharedBuffer&);

Source/WebCore/platform/graphics/cocoa/AV1UtilitiesCocoa.mm

@@ -63,6 +63,9 @@ static bool isConfigurationRecordHDR(const AV1CodecConfigurationRecord& record)
 
 std::optional<MediaCapabilitiesInfo> validateAV1Parameters(const AV1CodecConfigurationRecord& record, const VideoConfiguration& configuration)
 {
+    if (!validateAV1ConfigurationRecord(record))
+        return std::nullopt;
+
     if (!validateAV1PerLevelConstraints(record, configuration))
         return std::nullopt;
 
@@ -118,7 +121,25 @@ static bool isConfigurationRecordHDR(const AV1CodecConfigurationRecord& record)
         auto supportedChromaSubsampling = makeVector(cfSupportedChromaSubsampling, [](CFStringRef chromaSubsamplingString) {
             return parseInteger<uint8_t>(String(chromaSubsamplingString));
         });
-        if (!supportedChromaSubsampling.contains(static_cast<uint8_t>(record.chromaSubsampling)))
+
+        // CoreMedia defines the kVTDecoderCapability_ChromaSubsampling value as
+        // three decimal digits consisting of, in order from highest digit to lowest:
+        // [subsampling_x, subsampling_y, mono_chrome]. This conflicts with AV1's
+        // definition of chromaSubsampling in the Codecs Parameter String:
+        // "The chromaSubsampling parameter value, represented by a three-digit decimal,
+        // SHALL have its first digit equal to subsampling_x and its second digit equal to
+        // subsampling_y. If both subsampling_x and subsampling_y are set to 1, then the third
+        // digit SHALL be equal to chroma_sample_position, otherwise it SHALL be set to 0."
+
+        // CoreMedia supports all values of chroma_sample_position, so to reconcile this
+        // discrepency, construct a "chroma subsampling" query out of the high-order digits
+        // of the AV1CodecConfigurationRecord.chromaSubsampling field, and use the
+        // AV1CodecConfigurationRecord.monochrome field as the low-order digit.
+
+        uint8_t subsamplingXandY = record.chromaSubsampling - (record.chromaSubsampling % 10);
+        uint8_t subsamplingQuery = subsamplingXandY + (record.monochrome ? 1 : 0);
+
+        if (!supportedChromaSubsampling.contains(subsamplingQuery))
             return std::nullopt;
     }
 

Source/WebCore/testing/Internals.cpp

@@ -6736,6 +6736,13 @@ bool Internals::validateAV1PerLevelConstraints(const String& parameters, const V
     return false;
 }
 
+bool Internals::validateAV1ConfigurationRecord(const String& parameters)
+{
+    if (auto record = WebCore::parseAV1CodecParameters(parameters))
+        return WebCore::validateAV1ConfigurationRecord(*record);
+    return false;
+}
+
 auto Internals::getCookies() const -> Vector<CookieData>
 {
     auto* document = contextDocument();

Source/WebCore/testing/Internals.h

@@ -1247,6 +1247,7 @@ class Internals final
     using AV1CodecConfigurationRecord = WebCore::AV1CodecConfigurationRecord;
     std::optional<AV1CodecConfigurationRecord> parseAV1CodecParameters(const String&);
     String createAV1CodecParametersString(const AV1CodecConfigurationRecord&);
+    bool validateAV1ConfigurationRecord(const String&);
     bool validateAV1PerLevelConstraints(const String&, const VideoConfiguration&);
 
     struct CookieData {

Source/WebCore/testing/Internals.idl

@@ -1270,6 +1270,7 @@ enum RenderingMode {
     VPCodecConfigurationRecord? parseVPCodecParameters(DOMString codecParameters);
     AV1CodecConfigurationRecord? parseAV1CodecParameters(DOMString codecParameters);
     DOMString createAV1CodecParametersString(AV1CodecConfigurationRecord parameters);
+    boolean validateAV1ConfigurationRecord(DOMString codecParameters);
     boolean validateAV1PerLevelConstraints(DOMString codecParameters, VideoConfiguration configuration);
 
     sequence<CookieData> getCookies();