Merge branch 'main' into issue13609

.ci/containers/gcb-terraform-vcr-tester/test_terraform_vcr.sh

@@ -161,11 +161,12 @@ comment="#### Tests analytics ${NEWLINE}"
 comment+="Total tests: \`$(($FAILED_TESTS_COUNT+$PASSED_TESTS_COUNT+$SKIPPED_TESTS_COUNT))\` ${NEWLINE}"
 comment+="Passed tests \`$PASSED_TESTS_COUNT\` ${NEWLINE}"
 comment+="Skipped tests: \`$SKIPPED_TESTS_COUNT\` ${NEWLINE}"
-comment+="Failed tests: \`$FAILED_TESTS_COUNT\` ${NEWLINE}${NEWLINE}"
+comment+="Affected tests: \`$FAILED_TESTS_COUNT\` ${NEWLINE}${NEWLINE}"
 
 if [[ -n $FAILED_TESTS_PATTERN ]]; then
   comment+="#### Action taken ${NEWLINE}"
-  comment+="<details> <summary>Triggering VCR tests in RECORDING mode for the tests that failed during VCR. Click here to see the failed tests</summary><blockquote>$FAILED_TESTS_PATTERN </blockquote></details>"
+  comment+="<details> <summary>Found $FAILED_TESTS_COUNT affected test(s) by replaying old test recordings. Starting RECORDING based on the most recent commit. Click here to see the affected tests</summary><blockquote>$FAILED_TESTS_PATTERN </blockquote></details> ${NEWLINE}${NEWLINE}"
+  comment+="[Get to know how VCR tests work](https://googlecloudplatform.github.io/magic-modules/docs/getting-started/contributing/#general-contributing-steps)"
   add_comment "${comment}"
   # Clear fixtures folder
   rm $VCR_PATH/*

.ci/containers/github-differ/Dockerfile

@@ -5,4 +5,5 @@ RUN apt-get update
 RUN apt-get install -y curl jq
 ADD generate_comment.sh /generate_comment.sh
 ADD compare_breaking_changes.sh /compare_breaking_changes.sh
+ADD test_tools.sh /test_tools.sh
 ENTRYPOINT ["/generate_comment.sh"]

.ci/containers/github-differ/generate_comment.sh

@@ -43,6 +43,11 @@ fi
 git checkout origin/$NEW_BRANCH
 popd
 
+if ! git diff --exit-code origin/main tools; then
+    ## Run unit tests for breaking change and missing test detector.
+    /test_tools.sh $MM_LOCAL_PATH $TPG_LOCAL_PATH $COMMIT_SHA $BUILD_ID $BUILD_STEP $PROJECT_ID
+fi
+
 ## Breaking change setup and execution
 TPG_LOCAL_PATH_OLD="${TPG_LOCAL_PATH}old"
 mkdir -p $TPG_LOCAL_PATH_OLD

.ci/containers/github-differ/test_tools.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+MM_LOCAL_PATH=$1
+TPG_LOCAL_PATH=$2
+mm_commit_sha=$3
+build_id=$4
+build_step=$5
+project_id=$6
+
+github_username=modular-magician
+
+set +e
+pushd $MM_LOCAL_PATH/tools/breaking-change-detector
+go test ./...
+exit_code=$?
+popd
+set -e
+
+
+if [ $exit_code -ne 0 ]; then
+	state="failure"
+else
+	state="success"
+fi
+
+post_body=$( jq -n \
+	--arg context "unit-tests-breaking-change-detector" \
+	--arg target_url "https://console.cloud.google.com/cloud-build/builds;region=global/${build_id};step=${build_step}?project=${project_id}" \
+	--arg state "${state}" \
+	'{context: $context, target_url: $target_url, state: $state}')
+
+curl \
+  -X POST \
+  -u "$github_username:$GITHUB_TOKEN" \
+  -H "Accept: application/vnd.github.v3+json" \
+  "https://api.github.com/repos/GoogleCloudPlatform/magic-modules/statuses/$mm_commit_sha" \
+  -d "$post_body"
+
+set +e
+pushd $MM_LOCAL_PATH/tools/missing-test-detector
+PROVIDER_DIR=$TPG_LOCAL_PATH go test
+exit_code=$?
+popd
+set -e
+
+
+if [ $exit_code -ne 0 ]; then
+	state="failure"
+else
+	state="success"
+fi
+
+post_body=$( jq -n \
+	--arg context "unit-tests-missing-test-detector" \
+	--arg target_url "https://console.cloud.google.com/cloud-build/builds;region=global/${build_id};step=${build_step}?project=${project_id}" \
+	--arg state "${state}" \
+	'{context: $context, target_url: $target_url, state: $state}')
+
+curl \
+  -X POST \
+  -u "$github_username:$GITHUB_TOKEN" \
+  -H "Accept: application/vnd.github.v3+json" \
+  "https://api.github.com/repos/GoogleCloudPlatform/magic-modules/statuses/$mm_commit_sha" \
+  -d "$post_body"
+

docs/content/docs/how-to/add-handwritten-test.md

@@ -60,12 +60,12 @@ For example, the following test case is a good reference:
 func TestAccComputeFirewall_noSource(t *testing.T) {
     t.Parallel()
 
-    networkName := fmt.Sprintf("tf-test-firewall-%s", randString(t, 10))
-    firewallName := fmt.Sprintf("tf-test-firewall-%s", randString(t, 10))
+    networkName := fmt.Sprintf("tf-test-firewall-%s", RandString(t, 10))
+    firewallName := fmt.Sprintf("tf-test-firewall-%s", RandString(t, 10))
 
-    vcrTest(t, resource.TestCase{
+    VcrTest(t, resource.TestCase{
         PreCheck:     func() { testAccPreCheck(t) },
-        Providers:    testAccProviders,
+        Providers:    TestAccProviders,
         CheckDestroy: testAccCheckComputeFirewallDestroyProducer(t),
         Steps: []resource.TestStep{
             {
@@ -124,12 +124,12 @@ For example:
 func TestAccComputeFirewall_disabled(t *testing.T) {
     t.Parallel()
 
-    networkName := fmt.Sprintf("tf-test-firewall-%s", randString(t, 10))
-    firewallName := fmt.Sprintf("tf-test-firewall-%s", randString(t, 10))
+    networkName := fmt.Sprintf("tf-test-firewall-%s", RandString(t, 10))
+    firewallName := fmt.Sprintf("tf-test-firewall-%s", RandString(t, 10))
 
-    vcrTest(t, resource.TestCase{
+    VcrTest(t, resource.TestCase{
         PreCheck:     func() { testAccPreCheck(t) },
-        Providers:    testAccProviders,
+        Providers:    TestAccProviders,
         CheckDestroy: testAccCheckComputeFirewallDestroyProducer(t),
         Steps: []resource.TestStep{
             {

docs/content/docs/how-to/add-mmv1-test.md

@@ -104,12 +104,12 @@ func TestAccPubsubSubscription_pubsubSubscriptionDeadLetterExample(t *testing.T)
     t.Parallel()
 
     context := map[string]interface{}{
-        "random_suffix": randString(t, 10),
+        "random_suffix": RandString(t, 10),
     }
 
-    vcrTest(t, resource.TestCase{
+    VcrTest(t, resource.TestCase{
         PreCheck:     func() { testAccPreCheck(t) },
-        Providers:    testAccProviders,
+        Providers:    TestAccProviders,
         CheckDestroy: testAccCheckPubsubSubscriptionDestroyProducer(t),
         Steps: []resource.TestStep{
             {

mmv1/api/resource.rb

@@ -488,6 +488,24 @@ def delete_uri
       end
     end
 
+    def merge(other)
+      result = self.class.new
+      instance_variables.each do |v|
+        result.instance_variable_set(v, instance_variable_get(v))
+      end
+
+      other.instance_variables.each do |v|
+        if other.instance_variable_get(v).instance_of?(Array)
+          result.instance_variable_set(v, deep_merge(result.instance_variable_get(v),
+                                                     other.instance_variable_get(v)))
+        else
+          result.instance_variable_set(v, other.instance_variable_get(v))
+        end
+      end
+
+      result
+    end
+
     # ====================
     # Debugging Methods
     # ====================

mmv1/api/resource/iam_policy.rb

@@ -103,7 +103,7 @@ class IamPolicy < Api::Object
       attr_reader :self_link
 
       # [Optional] Version number in the request payload.
-      # if set, it overrides the default iamPolicyVersion
+      # if set, it overrides the default IamPolicyVersion
       attr_reader :iam_policy_version
 
       # [Optional] Min version to make IAM resources available at

mmv1/compiler.rb

@@ -212,8 +212,20 @@
       next if File.basename(file_path) == 'product.yaml' \
        || File.basename(file_path) == 'terraform.yaml'
 
-      resource_yaml = File.read(file_path)
-      resource = Api::Compiler.new(resource_yaml).run
+      if override_dir
+        resource_override_path = File.join(override_dir, file_path)
+        res_yaml = if File.exist?(resource_override_path)
+                     YAML.load_file(file_path, permitted_classes: allowed_classes) \
+                         .merge(YAML \
+                           .load_file(resource_override_path, permitted_classes: allowed_classes)) \
+                         .to_yaml
+                   else
+                     File.read(file_path)
+                   end
+      else
+        res_yaml = File.read(file_path)
+      end
+      resource = Api::Compiler.new(res_yaml).run
       resource.validate
       resources.push(resource)
     end

mmv1/products/accesscontextmanager/AccessPolicy.yaml

@@ -55,7 +55,7 @@
       primary_resource_id: "access-policy"
       test_env_vars:
         org_id: :ORG_ID
-        project: '"acm-tf-test-" + randString(t, 10)'
+        project: '"acm-tf-test-" + RandString(t, 10)'
   # Skipping the sweeper due to the non-standard base_url
   skip_sweeper: true
   custom_code: !ruby/object:Provider::Terraform::CustomCode

mmv1/products/artifactregistry/Repository.yaml

@@ -116,3 +116,140 @@
         - :RELEASE
         - :SNAPSHOT
         default_value: :VERSION_POLICY_UNSPECIFIED
+  - !ruby/object:Api::Type::Enum
+    name: 'mode'
+    min_version: beta
+    description: |-
+      The mode configures the repository to serve artifacts from different sources.
+    input: true
+    values:
+      - :STANDARD_REPOSITORY
+      - :VIRTUAL_REPOSITORY
+      - :REMOTE_REPOSITORY
+    default_value: :STANDARD_REPOSITORY
+  - !ruby/object:Api::Type::NestedObject
+    name: 'virtualRepositoryConfig'
+    min_version: beta
+    conflicts:
+      - remote_repository_config
+    description: |-
+      Configuration specific for a Virtual Repository.
+    properties:
+      - !ruby/object:Api::Type::Array
+        name: 'upstreamPolicies'
+        description: |-
+          Policies that configure the upstream artifacts distributed by the Virtual
+          Repository. Upstream policies cannot be set on a standard repository.
+        item_type: !ruby/object:Api::Type::NestedObject
+          properties:
+            - !ruby/object:Api::Type::String
+              name: 'id'
+              description: |-
+                The user-provided ID of the upstream policy.
+            - !ruby/object:Api::Type::String
+              name: 'repository'
+              description: |+
+                A reference to the repository resource, for example:
+                "projects/p1/locations/us-central1/repository/repo1".
+            - !ruby/object:Api::Type::Integer
+              name: 'priority'
+              description: |-
+                Entries with a greater priority value take precedence in the pull order.
+  - !ruby/object:Api::Type::NestedObject
+    name: 'remoteRepositoryConfig'
+    min_version: beta
+    conflicts:
+      - virtual_repository_config
+    description: |-
+      Configuration specific for a Remote Repository.
+    input: true
+    properties:
+      - !ruby/object:Api::Type::String
+        name: 'description'
+        description: |-
+          The description of the remote source.
+        input: true
+      - !ruby/object:Api::Type::NestedObject
+        name: 'dockerRepository'
+        exactly_one_of:
+          - remoteRepositoryConfig.0.docker_repository
+          - remoteRepositoryConfig.0.maven_repository
+          - remoteRepositoryConfig.0.npm_repository
+          - remoteRepositoryConfig.0.python_repository
+        description: |-
+          Specific settings for a Docker remote repository.
+        input: true
+        properties:
+          - !ruby/object:Api::Type::Enum
+            name: 'publicRepository'
+            exactly_one_of:
+              - remoteRepositoryConfig.0.docker_repository.0.public_repository
+            description: |-
+              Address of the remote repository.
+            input: true
+            values:
+              - :DOCKER_HUB
+            default_value: :DOCKER_HUB
+      - !ruby/object:Api::Type::NestedObject
+        name: 'mavenRepository'
+        exactly_one_of:
+          - remoteRepositoryConfig.0.docker_repository
+          - remoteRepositoryConfig.0.maven_repository
+          - remoteRepositoryConfig.0.npm_repository
+          - remoteRepositoryConfig.0.python_repository
+        description: |-
+          Specific settings for a Maven remote repository.
+        input: true
+        properties:
+          - !ruby/object:Api::Type::Enum
+            name: 'publicRepository'
+            exactly_one_of:
+              - remoteRepositoryConfig.0.maven_repository.0.public_repository
+            description: |-
+              Address of the remote repository.
+            input: true
+            values:
+              - :MAVEN_CENTRAL
+            default_value: :MAVEN_CENTRAL
+      - !ruby/object:Api::Type::NestedObject
+        name: 'npmRepository'
+        exactly_one_of:
+          - remoteRepositoryConfig.0.docker_repository
+          - remoteRepositoryConfig.0.maven_repository
+          - remoteRepositoryConfig.0.npm_repository
+          - remoteRepositoryConfig.0.python_repository
+        description: |-
+          Specific settings for an Npm remote repository.
+        input: true
+        properties:
+          - !ruby/object:Api::Type::Enum
+            name: 'publicRepository'
+            exactly_one_of:
+              - remoteRepositoryConfig.0.npm_repository.0.public_repository
+            description: |-
+              Address of the remote repository.
+            input: true
+            values:
+              - :NPMJS
+            default_value: :NPMJS
+      - !ruby/object:Api::Type::NestedObject
+        name: 'pythonRepository'
+        exactly_one_of:
+          - remoteRepositoryConfig.0.docker_repository
+          - remoteRepositoryConfig.0.maven_repository
+          - remoteRepositoryConfig.0.npm_repository
+          - remoteRepositoryConfig.0.python_repository
+        description: |-
+          Specific settings for a Python remote repository.
+        input: true
+        properties:
+          - !ruby/object:Api::Type::Enum
+            name: 'publicRepository'
+            exactly_one_of:
+              - remoteRepositoryConfig.0.python_repository.0.public_repository
+            description: |-
+              Address of the remote repository.
+            input: true
+            values:
+              - :PYPI
+            default_value: :PYPI

mmv1/products/artifactregistry/product.yaml

@@ -17,9 +17,12 @@ display_name: Artifact Registry
 scopes:
   - https://www.googleapis.com/auth/cloud-platform
 versions:
+  - !ruby/object:Api::Product::Version
+    name: beta
+    base_url: https://artifactregistry.googleapis.com/v1/
   - !ruby/object:Api::Product::Version
     name: ga
-    base_url: https://artifactregistry.googleapis.com/v1/    
+    base_url: https://artifactregistry.googleapis.com/v1/
 apis_required:
   - !ruby/object:Api::Product::ApiReference
     name: Artifact Registry API

mmv1/products/artifactregistry/terraform.yaml

@@ -33,6 +33,23 @@ overrides: !ruby/object:Overrides::ResourceOverrides
           kms_key_name: "kms-key"
         test_vars_overrides:
           kms_key_name: 'BootstrapKMSKeyInLocation(t, "us-central1").CryptoKey.Name'
+      - !ruby/object:Provider::Terraform::Examples
+        name: "artifact_registry_repository_virtual"
+        min_version: beta
+        primary_resource_id: "my-repo"
+        vars:
+          repository_id: "my-repository"
+          description: "example virtual docker repository"
+          upstream_repository_id: "my-repository-upstream"
+          upstream_description: "example docker repository (upstream source)"
+          upstream_policy_id: "my-repository-upstream"
+      - !ruby/object:Provider::Terraform::Examples
+        name: "artifact_registry_repository_remote"
+        min_version: beta
+        primary_resource_id: "my-repo"
+        vars:
+          repository_id: "my-repository"
+          description: "example remote docker repository"
     properties:
       location: !ruby/object:Overrides::Terraform::PropertyOverride
         required: false