Add support for codified VUs

The following is implemented in this change: - Codified VU parsing - Codified VU reformatting in source (reflow.py) - Codified VU formatting for build - Codified VU type checking - Unit tests There is no mention of codified VUs in the spec text however at this point. That will be added as VUs are converted.
KhronosGroup · Feb 10, 2023 · 151318e · 151318e
1 parent a8e92b6
commit 151318e
Show file tree

Hide file tree

Showing 14 changed files with 3,707 additions and 70 deletions.
diff --git a/Makefile b/Makefile
@@ -136,7 +136,7 @@ SPECREMARK = from git branch: $(shell echo `git symbolic-ref --short HEAD 2> /de
 	     commit: $(shell echo `git log -1 --format="%H" 2> /dev/null || echo Git commit not available`)
 
 # Some of the attributes used in building all spec documents:
-#   chapters - absolute path to chapter sources
+#   chapters - absolute path to generated (preprocessed) chapter sources
 #   appendices - absolute path to appendix sources
 #   proposals - absolute path to proposal sources
 #   images - absolute path to images
@@ -153,7 +153,7 @@ ATTRIBOPTS   = -a revnumber="$(SPECREVISION)" \
 	       -a config=$(CURDIR)/config \
 	       -a appendices=$(CURDIR)/appendices \
 	       -a proposals=$(CURDIR)/proposals \
-	       -a chapters=$(CURDIR)/chapters \
+	       -a chapters=$(GENERATED)/chapters \
 	       -a images=$(IMAGEPATH) \
 	       -a generated=$(GENERATED) \
 	       -a refprefix \
@@ -212,11 +212,16 @@ SVGFILES  = $(wildcard $(IMAGEPATH)/*.svg)
 # Top-level spec source file
 SPECSRC        = vkspec.adoc
 # Static files making up sections of the API spec.
-SPECFILES = $(wildcard chapters/[A-Za-z]*.adoc chapters/*/[A-Za-z]*.adoc appendices/[A-Za-z]*.adoc)
+SPECALLCHAPTERFILES = $(wildcard chapters/[A-Za-z]*.adoc chapters/*/[A-Za-z]*.adoc)
+SPECCOMMONVALIDITYFILES = $(wildcard chapters/commonvalidity/[A-Za-z]*.adoc)
+SPECCHAPTERFILES = $(filter-out chapters/commonvalidity/%.adoc,$(SPECALLCHAPTERFILES))
+SPECAPPENDIXFILES = $(wildcard appendices/[A-Za-z]*.adoc)
+SPECFILES = $(SPECALLCHAPTERFILES) $(SPECAPPENDIXFILES)
 # Shorthand for where different types generated files go.
 # All can be relocated by overriding GENERATED in the make invocation.
 GENERATED      = $(CURDIR)/gen
 REFPATH        = $(GENERATED)/refpage
+CHAPTERSPATH   = $(GENERATED)/chapters
 APIPATH        = $(GENERATED)/api
 VALIDITYPATH   = $(GENERATED)/validity
 HOSTSYNCPATH   = $(GENERATED)/hostsynctable
@@ -227,6 +232,7 @@ FORMATSPATH    = $(GENERATED)/formats
 PROPOSALPATH   = $(CURDIR)/proposals
 # timeMarker is a proxy target created when many generated files are
 # made at once
+CHAPTERSDEPEND = $(CHAPTERSPATH)/timeMarker
 APIDEPEND      = $(APIPATH)/timeMarker
 VALIDITYDEPEND = $(VALIDITYPATH)/timeMarker
 HOSTSYNCDEPEND = $(HOSTSYNCPATH)/timeMarker
@@ -237,7 +243,7 @@ FORMATSDEPEND = $(FORMATSPATH)/timeMarker
 RUBYDEPEND     = $(RBAPIMAP)
 ATTRIBFILE     = $(GENERATED)/specattribs.adoc
 # All generated dependencies
-GENDEPENDS     = $(APIDEPEND) $(VALIDITYDEPEND) $(HOSTSYNCDEPEND) $(METADEPEND) $(INTERFACEDEPEND) $(SPIRVCAPDEPEND) $(FORMATSDEPEND) $(RUBYDEPEND) $(ATTRIBFILE)
+GENDEPENDS     = $(CHAPTERSDEPEND) $(APIDEPEND) $(VALIDITYDEPEND) $(HOSTSYNCDEPEND) $(METADEPEND) $(INTERFACEDEPEND) $(SPIRVCAPDEPEND) $(FORMATSDEPEND) $(RUBYDEPEND) $(ATTRIBFILE)
 # All non-format-specific dependencies
 COMMONDOCS     = $(SPECFILES) $(GENDEPENDS)
 
@@ -485,12 +491,13 @@ MANSOURCES   = $(filter-out $(REFPATH)/apispec.adoc, $(wildcard $(REFPATH)/*.ado
 # For now, all core and extension refpages are extracted by genRef.py.
 GENREF = $(SCRIPTS)/genRef.py
 LOGFILE = $(REFPATH)/refpage.log
+SPECPREPROCESSEDFILES = $(SPECCHAPTERFILES:chapters/%.adoc=$(CHAPTERSPATH)/%.adoc)
 refpages: $(REFPATH)/apispec.adoc
-$(REFPATH)/apispec.adoc: $(SPECFILES) $(GENREF) $(SCRIPTS)/reflib.py $(PYAPIMAP)
+$(REFPATH)/apispec.adoc: $(CHAPTERSDEPEND) $(SPECAPPENDIXFILES) $(GENREF) $(SCRIPTS)/reflib.py $(PYAPIMAP)
 	$(QUIET)$(MKDIR) $(REFPATH)
 	$(PYTHON) $(GENREF) -genpath $(GENERATED) -basedir $(REFPATH) \
 	    -log $(LOGFILE) -extpath $(CURDIR)/appendices \
-	    $(EXTOPTIONS) $(SPECFILES)
+	    $(EXTOPTIONS) $(SPECPREPROCESSEDFILES)
 
 # These targets are HTML5 refpages
 #
@@ -607,6 +614,13 @@ rubyapi $(RBAPIMAP): $(VKXML) $(GENVK)
 	$(QUIET)$(MKDIR) $(GENERATED)
 	$(QUIET)$(PYTHON) $(GENVK) $(GENVKOPTS) -o $(GENERATED) apimap.rb
 
+# Preprocess chapters so codified VUs can be formatted + links generated.
+chapters: $(CHAPTERSDEPEND)
+
+$(CHAPTERSDEPEND): $(SPECALLCHAPTERFILES) $(SCRIPTS)/doctransformer.py $(SCRIPTS)/vuAST.py $(SCRIPTS)/vupreprocessor.py
+	$(QUIET)$(MKDIR) $(CHAPTERSPATH)
+	$(QUIET)$(PYTHON) $(SCRIPTS)/vupreprocessor.py -o $(CHAPTERSPATH) -stamp $@
+
 apiinc: $(APIDEPEND)
 
 $(APIDEPEND): $(VKXML) $(GENVK) $(PYAPIMAP)

diff --git a/config/khronos.css b/config/khronos.css
@@ -730,3 +730,10 @@ li > p > a[id^="VUID-"].link { color: black; text-decoration: none; }
 li > p > a[id^="VUID-"].link:hover { color: black; }
 
 .vuid { color: #4d4d4d; font-family: monospace; }
+
+/* VU styling */
+.vu { font-family: monospace; }
+.vu-keyword { font-weight: bold; }
+.vu-operator { font-weight: bold; }
+.vu-number { font-weight: bold; }
+.vu-builtin { }
diff --git a/scripts/doctransformer.py b/scripts/doctransformer.py
@@ -24,17 +24,23 @@
 #   ifndef::
 conditionalStart = re.compile(r'^(ifdef|ifndef)::')
 
+# Attribute setting markup in the following form:
+#
+#   :attribute-setting
+#
+# This always ends the paragraph
+attributeSetting = re.compile(r'^:.*$')
+
 # Markup that always ends a paragraph
 #   empty line or whitespace
 #   [block options]
 #   [[anchor]]
 #   //                  comment
 #   <<<<                page break
-#   :attribute-setting
 #   macro-directive::terms
 #   +                   standalone list item continuation
 #   label::             labelled list - label must be standalone
-endPara = re.compile(r'^( *|\[.*\]|//.*|<<<<|:.*|[a-z]+::.*|\+|.*::)$')
+endPara = re.compile(r'^( *|\[.*\]|//.*|<<<<|[a-z]+::.*|\+|.*::)$')
 
 # Special case of markup ending a paragraph, used to track the current
 # command/structure. This allows for either OpenXR or Vulkan API path
@@ -182,7 +188,7 @@ def addLine(self, line, indent):
 class TransformCallbackState:
     """State given to the transformer callback object, derived from
     TransformState."""
-    def __init__(self, state):
+    def __init__(self, state, paragraphLineCount):
         self.isVU = state.vuStack[-1] if len(state.vuStack) > 0 else False
         """Whether this paragraph is a VU."""
 
@@ -196,7 +202,9 @@ def __init__(self, state):
         self.hangIndent = state.hangIndent
         """indent level of the remaining lines of a paragraph."""
 
-        self.lineNumber = state.lineNumber
+        # Make sure line number points to the beginning of the
+        # paragraph, not the end of it.
+        self.lineNumber = state.lineNumber - paragraphLineCount
         """line number being read from the input file."""
 
 
@@ -212,6 +220,10 @@ class DocTransformer:
     - transformParagraph: Called when a paragraph is parsed.  The paragraph
       along with some information (such as whether it is a VU) is passed.  The
       function may transform the paragraph as necessary.
+    - transformInclude: Called when an include directive is encountered.  The
+      include line and the state is passed.  The function may transform the
+      line as necessary.
+    - onMacro: Called when a macro is encountered.
     - onEmbeddedVUConditional: Called when an embedded VU conditional is
       encountered.
     """
@@ -249,7 +261,8 @@ def emitPara(self):
             transformedPara = self.state.para
 
             if self.state.transformStack[-1]:
-                callbackState = TransformCallbackState(self.state)
+                callbackState = TransformCallbackState(self.state,
+                                                       len(self.state.para))
 
                 transformedPara = self.callback.transformParagraph(
                         self.state.para,
@@ -362,32 +375,46 @@ def transformFile(self, lines):
 
                 self.endParaBlockTransform(line, vuBlock)
 
-            elif endPara.match(line):
-                # Ending a paragraph. Emit the current paragraph, if any, and
-                # prepare to begin a new paragraph.
+            elif attributeSetting.match(line):
+                # Notify that a macro was encountered
 
+                callbackState = TransformCallbackState(self.state, 1)
+                self.callback.onMacro(line, callbackState)
+
+                # End the paragraph.
                 self.endPara(line)
 
+            elif endPara.match(line):
                 # If this is an include:: line starting the definition of a
                 # structure or command, track that for use in VUID generation.
 
-                matches = includePat.search(line)
-                if matches is not None:
-                    generated_type = matches.group('generated_type')
-                    include_type = matches.group('category')
-                    if generated_type == 'api' and include_type in ('protos', 'structs', 'funcpointers'):
-                        apiName = matches.group('entity_name')
-                        if self.state.apiName != self.state.defaultApiName:
-                            # This happens when there are multiple API include
-                            # lines in a single block. The style guideline is to
-                            # always place the API which others are promoted to
-                            # first. In virtually all cases, the promoted API
-                            # will differ solely in the vendor suffix (or
-                            # absence of it), which is benign.
-                            if not self.apiMatch(self.state.apiName, apiName):
-                                logDiag(f'Promoted API name mismatch at line {self.state.lineNumber}: {apiName} does not match self.state.apiName (this is OK if it is just a spelling alias)')
-                        else:
-                            self.state.apiName = apiName
+                if line.startswith('include::'):
+                    matches = includePat.search(line)
+                    if matches is not None:
+                        generated_type = matches.group('generated_type')
+                        include_type = matches.group('category')
+                        if generated_type == 'api' and include_type in ('protos', 'structs', 'funcpointers'):
+                            apiName = matches.group('entity_name')
+                            if self.state.apiName != self.state.defaultApiName:
+                                # This happens when there are multiple API include
+                                # lines in a single block. The style guideline is to
+                                # always place the API which others are promoted to
+                                # first. In virtually all cases, the promoted API
+                                # will differ solely in the vendor suffix (or
+                                # absence of it), which is benign.
+                                if not self.apiMatch(self.state.apiName, apiName):
+                                    logDiag(f'Promoted API name mismatch at line {self.state.lineNumber}: {apiName} does not match self.state.apiName (this is OK if it is just a spelling alias)')
+                            else:
+                                self.state.apiName = apiName
+
+                    callbackState = TransformCallbackState(self.state, 1)
+                    line = self.callback.transformInclude(line, callbackState)
+
+                # Ending a paragraph. Emit the current paragraph, if any, and
+                # prepare to begin a new paragraph.
+
+                self.endPara(line)
+
 
             elif endParaContinue.match(line):
                 # For now, always just end the paragraph.
@@ -422,7 +449,8 @@ def transformFile(self, lines):
                 if (self.state.vuStack[-1]
                     and not beginBullet.match(line)
                     and conditionalStart.match(lines[self.state.lineNumber-2])):
-                       self.callback.onEmbeddedVUConditional(self.state)
+                        callbackState = TransformCallbackState(self.state, 1)
+                        self.callback.onEmbeddedVUConditional(callbackState)
 
             self.state.lastTitle = thisTitle
 

diff --git a/scripts/reflib.py b/scripts/reflib.py
@@ -7,6 +7,7 @@
 # Utility functions for automatic ref page generation and other script stuff
 
 import io
+from pathlib import Path
 import re
 import sys
 import subprocess
@@ -252,6 +253,16 @@ def loadFile(filename):
 
     return lines, newline_string
 
+def resolveAndMkdir(path):
+    path = Path(path).resolve()
+    # TOCTOU-safe directory creation
+    try:
+        path.mkdir()
+    except FileExistsError:
+        pass
+
+    return path
+
 def clampToBlock(line, minline, maxline):
     """Clamp a line number to be in the range [minline,maxline].
 

diff --git a/scripts/reflow-tests/expect-new-vuid-codified.adoc b/scripts/reflow-tests/expect-new-vuid-codified.adoc
@@ -0,0 +1,92 @@
+// Copyright 2015-2022 The Khronos Group Inc.
+//
+// SPDX-License-Identifier: CC-BY-4.0
+
+[[clears]]
+= Clear Commands
+
+
+[[clears-outside]]
+== Clearing Images Outside A Render Pass Instance
+
+[open,refpage='vkCmdClearColorImage',desc='Clear regions of a color image',type='protos']
+--
+To clear one or more subranges of a color image, call:
+
+include::{generated}/api/protos/vkCmdClearColorImage.adoc[]
+
+.Valid Usage
+****
+ifdef::VK_VERSION_1_1,VK_KHR_maintenance1[]
+  * [[VUID-vkCmdClearColorImage-image-01993]]
+    The <<resources-image-format-features,format features>> of pname:image
+    must: contain ename:VK_FORMAT_FEATURE_TRANSFER_DST_BIT
+endif::VK_VERSION_1_1,VK_KHR_maintenance1[]
+  * [[VUID-vkCmdClearColorImage-image-10000]]
+    require(image.create_info().usage.has_bit(VK_IMAGE_USAGE_TRANSFER_DST_BIT))
+ifdef::VK_VERSION_1_1,VK_KHR_sampler_ycbcr_conversion[]
+  * [[VUID-vkCmdClearColorImage-image-01545]]
+    pname:image must: not use any of the
+    <<formats-requiring-sampler-ycbcr-conversion, formats that require a
+    sampler {YCbCr} conversion>>
+endif::VK_VERSION_1_1,VK_KHR_sampler_ycbcr_conversion[]
+  * [[VUID-vkCmdClearColorImage-image-10001]]
+    If pname:image is non-sparse then it must: be bound completely and
+    contiguously to a single sname:VkDeviceMemory object
+  * [[VUID-vkCmdClearColorImage-imageLayout-00004]]
+    pname:imageLayout must: specify the layout of the image subresource
+    ranges of pname:image specified in pname:pRanges at the time this
+    command is executed on a sname:VkDevice
+ifndef::VK_KHR_shared_presentable_image[]
+  * [[VUID-vkCmdClearColorImage-imageLayout-00005]]
+    pname:imageLayout must: be ename:VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL or
+    ename:VK_IMAGE_LAYOUT_GENERAL
+endif::VK_KHR_shared_presentable_image[]
+ifdef::VK_KHR_shared_presentable_image[]
+  * [[VUID-vkCmdClearColorImage-imageLayout-10002]]
+    require(imageLayout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL or
+        imageLayout == VK_IMAGE_LAYOUT_GENERAL or
+        imageLayout == VK_IMAGE_LAYOUT_SHARED_PRESENT_KHR)
+endif::VK_KHR_shared_presentable_image[]
+  * [[VUID-vkCmdClearColorImage-aspectMask-02498]]
+    The slink:VkImageSubresourceRange::pname:aspectMask members of the
+    elements of the pname:pRanges array must: each only include
+    ename:VK_IMAGE_ASPECT_COLOR_BIT
+  * [[VUID-vkCmdClearColorImage-image-10003]]
+    mipLevels = image.create_info().mipLevels
+    for range in pRanges:
+     require(range.baseMipLevel < mipLevels)
+  * [[VUID-vkCmdClearColorImage-image-10004]]
+    mipLevels = image.create_info().mipLevels
+    for range in pRanges:
+     if range.levelCount != VK_REMAINING_MIP_LEVELS:
+      require(range.baseMipLevel + range.levelCount <= mipLevels)
+  * [[VUID-vkCmdClearColorImage-baseArrayLayer-01472]]
+    The slink:VkImageSubresourceRange::pname:baseArrayLayer members of the
+    elements of the pname:pRanges array must: each be less than the
+    pname:arrayLayers specified in slink:VkImageCreateInfo when pname:image
+    was created
+  * [[VUID-vkCmdClearColorImage-pRanges-01693]]
+    For each slink:VkImageSubresourceRange element of pname:pRanges, if the
+    pname:layerCount member is not ename:VK_REMAINING_ARRAY_LAYERS, then
+    [eq]#pname:baseArrayLayer {plus} pname:layerCount# must: be less than or
+    equal to the pname:arrayLayers specified in slink:VkImageCreateInfo when
+    pname:image was created
+  * [[VUID-vkCmdClearColorImage-image-10005]]
+    pname:image must: not have a compressed or depth/stencil format
+  * [[VUID-vkCmdClearColorImage-pColor-10006]]
+    pname:pColor must: be a valid pointer to a slink:VkClearColorValue union
+ifdef::VK_VERSION_1_1[]
+  * [[VUID-vkCmdClearColorImage-commandBuffer-01805]]
+    If pname:commandBuffer is an unprotected command buffer and
+    <<limits-protectedNoFault, pname:protectedNoFault>> is not supported,
+    pname:image must: not be a protected image
+  * [[VUID-vkCmdClearColorImage-commandBuffer-01806]]
+    If pname:commandBuffer is a protected command buffer and
+    <<limits-protectedNoFault, pname:protectedNoFault>> is not supported,
+    must: not be an unprotected image
+endif::VK_VERSION_1_1[]
+****
+
+include::{generated}/validity/protos/vkCmdClearColorImage.adoc[]
+--