Conformance Reporting to reflect v1.1.0 release (kubernetes-sigs#3088)

* Reflect v1.1.0, new naming format

* Add line to RELEASE.md

* Update release, generated implementation table path file names

* Chanch file structure per feedback

* Check if extended features are present

* Run against implementations

* Update hack/mkdocs-generate-conformance.py

Update so GRPCRoute is listed before TLS Route

Co-authored-by: Rob Scott <rob.scott87@gmail.com>

---------

Co-authored-by: Rob Scott <rob.scott87@gmail.com>

RELEASE.md

@@ -70,7 +70,7 @@ For a **MAJOR** or **MINOR** release:
 - Run the `make build-install-yaml` command which will generate install files in the `release/` directory.
   Attach these files to the GitHub release.
 - Update the `README.md` and `site-src/guides/index.md` files to point links and examples to the new release.
-
+- Update the implementation table path (`nav.Implementations.Comparison`) in the nav of `mkdocs.yml` to point to the latest release file (for example Implementation Comparison points to `implmenetation-table-v1.1.0.md`). Add the now past version under `Past Version Comparisons`, and edit the text blurb in `mkdocs-generate-conformance.py` to also reflect the added past version.  
 For an **RC** release:
 - Update `pkg/generator/main.go` with the new semver tag and any updates to the API review URL.
 - Run the following command `BASE_REF=vmajor.minor.patch make generate` which

hack/mkdocs-generate-conformance.py

@@ -18,6 +18,7 @@
 import pandas
 from fnmatch import fnmatch
 import glob
+import os
 
 log = logging.getLogger('mkdocs')
 
@@ -26,9 +27,12 @@
 def on_pre_build(config, **kwargs):
     log.info("generating conformance")
 
-    yamlReports = getYaml()
+    vers = getConformancePaths()
+    for v in vers[3:]:
 
-    generate_conformance_tables(yamlReports)
+        confYamls = getYaml(v)
+        releaseVersion = v.split(os.sep)[-2]
+        generate_conformance_tables(confYamls, releaseVersion)
 
 
 desc = """
@@ -45,24 +49,34 @@ def on_pre_build(config, **kwargs):
     However, as it is based on submitted conformance reports, the information is correct.
 """
 
-# NOTE: will have to be updated if new (extended) features are added
-httproute_extended_conformance_features_list = ['HTTPRouteBackendRequestHeaderModification', 'HTTPRouteQueryParamMatching', 'HTTPRouteMethodMatching', 'HTTPRouteResponseHeaderModification', 'HTTPRoutePortRedirect', 'HTTPRouteSchemeRedirect',
-                                                'HTTPRoutePathRedirect', 'HTTPRouteHostRewrite', 'HTTPRoutePathRewrite', 'HTTPRouteRequestMirror', 'HTTPRouteRequestMultipleMirrors', 'HTTPRouteRequestTimeout', 'HTTPRouteBackendTimeout', 'HTTPRouteParentRefPort']
 
 
-def generate_conformance_tables(reports):
+def generate_conformance_tables(reports, currVersion):
 
-    gateway_http_table = generate_profiles_report(reports, 'HTTP')
-    gateway_http_table = gateway_http_table.rename_axis('Organization')
+    gateway_tls_table = pandas.DataFrame()
+    gateway_grpc_table = pandas.DataFrame()
+
+    if currVersion == allVersions[-1]:
+        gateway_http_table = generate_profiles_report(reports, 'GATEWAY-HTTP')
+
+        gateway_grpc_table = generate_profiles_report(reports, 'GATEWAY-GRPC')
+        gateway_grpc_table = gateway_grpc_table.rename_axis('Organization')
 
-    # Currently no implementation has extended supported features listed.
-    # Can uncomment once a list is needed to keep track
-    # gateway_tls_table = generate_profiles_report(reprots,'TLS')
+        gateway_tls_table = generate_profiles_report(reports, 'GATEWAY-TLS')
+        gateway_tls_table = gateway_tls_table.rename_axis('Organization')
 
-    mesh_http_table = generate_profiles_report(reports, 'MESH')
+        mesh_http_table = generate_profiles_report(reports, 'MESH-HTTP')
+    else:
+        gateway_http_table = generate_profiles_report(reports, "HTTP")
+        mesh_http_table = generate_profiles_report(reports, "MESH")
+
+    gateway_http_table = gateway_http_table.rename_axis('Organization')
     mesh_http_table = mesh_http_table.rename_axis('Organization')
 
-    with open('site-src/implementation-table.md', 'w') as f:
+    versionFile = ".".join(currVersion.split(".")[:2])
+
+    with open('site-src/implementations/'+versionFile+'.md', 'w') as f:
+
         f.write(desc)
         f.write("\n\n")
 
@@ -72,6 +86,11 @@ def generate_conformance_tables(reports):
         f.write("## Gateway Profile\n\n")
         f.write("### HTTPRoute\n\n")
         f.write(gateway_http_table.to_markdown()+'\n\n')
+        if currVersion == allVersions[-1]:
+            f.write('### GRPCRoute\n\n')
+            f.write(gateway_grpc_table.to_markdown()+'\n\n')
+            f.write('### TLSRoute\n\n')
+            f.write(gateway_tls_table.to_markdown()+'\n\n')
 
         f.write("## Mesh Profile\n\n")
         f.write("### HTTPRoute\n\n")
@@ -90,10 +109,11 @@ def generate_profiles_report(reports, route):
                                'version', 'extended.supportedFeatures']].T
     http_table.columns = http_table.iloc[0]
     http_table = http_table[1:].T
-
+    
     for row in http_table.itertuples():
-        for feat in row._3:
-            http_table.loc[row.Index, feat] = ':white_check_mark:'
+        if type(row._3) is list:
+            for feat in row._3:
+                http_table.loc[row.Index, feat] = ':white_check_mark:'
     http_table = http_table.fillna(':x:')
     http_table = http_table.drop(['extended.supportedFeatures'], axis=1)
 
@@ -103,15 +123,27 @@ def generate_profiles_report(reports, route):
     return http_table
 
 
-# the path should be changed when there is a new version
-conformance_path = "conformance/reports/v1.0.0/**"
+pathTemp = "conformance/reports/*/"
+allVersions = []
+reportedImplementationsPath = []
+
+# returns v1.0.0 and greater, since that's when reports started being generated in the comparison table
+
+
+def getConformancePaths():
+    versions = sorted(glob.glob(pathTemp, recursive=True))
+    report_path = versions[-1]+"**"
+    for v in versions:
+        vers = v.split(os.sep)[-2]
+        allVersions.append(vers)
+        reportedImplementationsPath.append(v+"**")
+    return reportedImplementationsPath
 
 
-def getYaml():
-    log.info("parsing conformance reports ============================")
+def getYaml(conf_path):
     yamls = []
 
-    for p in glob.glob(conformance_path, recursive=True):
+    for p in glob.glob(conf_path, recursive=True):
 
         if fnmatch(p, "*.yaml"):
 

mkdocs.yml

@@ -80,8 +80,10 @@ nav:
         GAMMA Initiative: mesh/gamma.md
         Service Facets: mesh/service-facets.md
     - Implementations:
-        List: implementations.md
-        Comparison: implementation-table.md
+        - List: implementations.md
+        - Comparisons:
+          - v1.1: implementations/v1.1.md
+          - v1.0: implementations/v1.0.md
     - FAQ: faq.md
     - Glossary: concepts/glossary.md
   - Guides:

site-src/implementations.md

@@ -10,7 +10,7 @@ cover, and documentation to help users get started.
 
 !!! info "Compare extended supported features across implementations"
 
-    [View a table to quickly compare supported features of projects](/implementation-table). These outline Gateway controller implementations that have passed core conformance tests, and focus on extended conformance features that they have implemented.
+    [View a table to quickly compare supported features of projects](/implementations/v1.1). These outline Gateway controller implementations that have passed core conformance tests, and focus on extended conformance features that they have implemented.
 
 ## Gateway Controller Implementation Status <a name="gateways"></a>