Revert "[EventHub] Raise error when primary key is rotated"

.github/workflows/azure-sdk-tools.yml

@@ -0,0 +1,34 @@
+name: Test Azure SDK Tools
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [ main ]
+    paths:
+      - "tools/azure-sdk-tools/**"
+
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+
+      # todo before checkin: multiplex to include running tests on py38 -> 312
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: Install azure-sdk-tools
+        run: |
+          python -m pip install -e tools/azure-sdk-tools[build,ghtools]
+          python -m pip freeze
+        shell: bash
+
+      - name: Run tests
+        run: |
+          pytest ./tests
+        shell: bash
+        working-directory: tools/azure-sdk-tools

.vscode/cspell.json

@@ -135,7 +135,8 @@
     "conda/conda-recipes/msal-extensions/meta.yaml",
     "conda/conda-recipes/msrest/meta.yaml",
     "conda/conda-recipes/uamqp/meta.yaml",
-    "conda/conda-releaselogs/azure-mgmt.md"
+    "conda/conda-releaselogs/azure-mgmt.md",
+    ".github/workflows/azure-sdk-tools.yml"
   ],
   "words": [
     "msedge",
@@ -469,7 +470,8 @@
     "dicom",
     "WINDOWSVMIMAGE",
     "LINUXVMIMAGE",
-    "MACVMIMAGE"
+    "MACVMIMAGE",
+    "myuseragent"
   ],
   "overrides": [
     {

eng/common/pipelines/codeowners-linter.yml

@@ -31,7 +31,7 @@ stages:
       vmImage: ubuntu-22.04
 
     variables:
-      CodeownersLinterVersion: '1.0.0-dev.20240917.2'
+      CodeownersLinterVersion: '1.0.0-dev.20240926.2'
       DotNetDevOpsFeed: "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json"
       RepoLabelUri: "https://azuresdkartifacts.blob.core.windows.net/azure-sdk-write-teams/repository-labels-blob"
       TeamUserUri: "https://azuresdkartifacts.blob.core.windows.net/azure-sdk-write-teams/azure-sdk-write-teams-blob"

eng/common/scripts/Update-DocsMsPackages.ps1

@@ -54,7 +54,7 @@ function GetMetadata($moniker) {
   return $metadata
 }
 
-function ValidatePackageForOnboarding2($package) {
+function PackageIsValidForDocsOnboarding($package) {
   if (!(Test-Path "Function:$ValidateDocsMsPackagesFn")) {
     return $true
   }
@@ -88,7 +88,7 @@ foreach ($moniker in $MONIKERS) {
         if ($package.ContainsKey('_SkipDocsValidation') -and $true -eq $package['_SkipDocsValidation']) {
           Write-Host "Skip validation for package: $($packageIdentity)"
         }
-        elseif (!(ValidatePackageForOnboarding2 $package)) {
+        elseif (!(PackageIsValidForDocsOnboarding $package)) {
           LogWarning "Skip adding package that did not pass validation: $($packageIdentity)"
           continue
         }
@@ -101,7 +101,7 @@ foreach ($moniker in $MONIKERS) {
       $oldPackage = $alreadyOnboardedPackages[$packageIdentity]
 
       if ($oldPackage.Version -ne $package.Version) {
-        if (!(ValidatePackageForOnboarding2 $package)) {
+        if (!(PackageIsValidForDocsOnboarding $package)) {
           LogWarning "Omitting package that failed validation: $($packageIdentity)@$($package.Version)"
           continue
         }

eng/pipelines/templates/steps/resolve-package-targeting.yml

@@ -21,17 +21,25 @@ steps:
           $setting = "$(BuildTargetingString)"
       }
       else {
-        if ("${{ parameters.IncludeIndirect }}" -eq "true") {
-          $packageProperties = Get-ChildItem -Recurse -Force "${{ parameters.PackagePropertiesFolder }}/*.json" `
+        if (Test-Path "${{ parameters.PackagePropertiesFolder }}") {
+          if ("${{ parameters.IncludeIndirect }}" -eq "true") {
+            $packageProperties = Get-ChildItem -Recurse -Force "${{ parameters.PackagePropertiesFolder }}/*.json" `
+                | ForEach-Object { $_.Name.Replace(".json", "") }
+          }
+          else {
+            $packageProperties = Get-ChildItem -Recurse -Force "${{ parameters.PackagePropertiesFolder }}/*.json" `
+              | Where-Object { (Get-Content -Raw $_ | ConvertFrom-Json).IncludedForValidation -eq $false } `
               | ForEach-Object { $_.Name.Replace(".json", "") }
-        }
-        else {
-          $packageProperties = Get-ChildItem -Recurse -Force "${{ parameters.PackagePropertiesFolder }}/*.json" `
-            | Where-Object { (Get-Content -Raw $_ | ConvertFrom-Json).IncludedForValidation -eq $false } `
-            | ForEach-Object { $_.Name.Replace(".json", "") }
-        }
+          }
 
-        $setting = $packageProperties -join ","
+          $setting = $packageProperties -join ","
+
+          # in case we don't expect any packages, we should set the variable to null, which will match NO packages and cause whatever the check
+          # is to skip with exit 0 (which is what we want!)
+          if (-not $setting) {
+            $setting = "null"
+          }
+        }
       }
 
       Write-Host "##vso[task.setvariable variable=TargetingString;]$setting"

eng/pipelines/templates/variables/globals.yml

@@ -11,3 +11,4 @@ variables:
   REPOROOT: $(Build.SourcesDirectory)
   WINDOWS_OUTPUTROOT: $(REPOROOT)\out
   WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest'
+  GDN_SUPPRESS_FORKED_BUILD_WARNING: true

eng/scripts/Language-Settings.ps1

@@ -9,7 +9,6 @@ $PackageRepositoryUri = "https://pypi.org/project"
 ."$PSScriptRoot/docs/Docs-ToC.ps1"
 ."$PSScriptRoot/docs/Docs-Onboarding.ps1"
 
-
 function Get-python-AdditionalValidationPackagesFromPackageSet {
   param(
     [Parameter(Mandatory=$true)]
@@ -21,9 +20,22 @@ function Get-python-AdditionalValidationPackagesFromPackageSet {
   )
   $additionalValidationPackages = @()
 
+  function isOther($fileName) {
+    $startsWithPrefixes = @(".config", ".devcontainer", ".github", ".vscode", "common", "conda", "doc", "eng", "scripts")
+
+    $startsWith = $false
+    foreach ($prefix in $startsWithPrefixes) {
+      if ($fileName.StartsWith($prefix)) {
+        $startsWith = $true
+      }
+    }
+
+    return $startsWith
+  }
+
   $toolChanged = $diffObj.ChangedFiles | Where-Object { $_.StartsWith("tool")}
   $engChanged = $diffObj.ChangedFiles | Where-Object { $_.StartsWith("eng")}
-  $othersChanged = $diffObj.ChangedFiles | Where-Object { $_.StartsWith("scripts") -or $_.StartsWith("doc") -or $_.StartsWith("common") -or $_.StartsWith("conda") }
+  $othersChanged = $diffObj.ChangedFiles | Where-Object { isOther($_) }
 
   if ($toolChanged) {
     $additionalPackages = @(
@@ -391,11 +403,8 @@ function Get-python-DocsMsMetadataForPackage($PackageInfo) {
 
 function Import-Dev-Cert-python
 {
-  Write-Host "Python Trust Methodology"
-
-  $pathToScript = Resolve-Path (Join-Path -Path $PSScriptRoot -ChildPath "../../scripts/devops_tasks/trust_proxy_cert.py")
-  python -m pip install requests
-  python $pathToScript
+  Write-Host "Python no longer requires an out of proc trust methodology." `
+    "The variables SSL_CERT_DIR, SSL_CERT_FILE, and REQUESTS_CA_BUNDLE are now dynamically set in proxy_startup.py"
 }
 
 # Defined in common.ps1 as:

eng/scripts/distribute-packages-to-matrix.ps1

@@ -151,7 +151,7 @@ function Update-Matrix {
 
     # we need to ensure the presence of TargetingString in the matrix object
     if ($matrixUpdate) {
-        if ($directBatches) {
+        if ($directBatches -or $indirectBatches) {
             if (-not $Matrix.matrix.PSObject.Properties["TargetingString"]) {
                 $Matrix.matrix | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value @()
             }
@@ -217,7 +217,11 @@ $indirectIncludedPackages = $packageProperties | Where-Object { $_.IncludedForVa
 
 # I will assign all the direct included packages first. our goal is to get full coverage of the direct included packages
 # then, for the indirect packages, we will add them as sparse TargetingString bundles to the matrix
-$directBatches = Split-ArrayIntoBatches -InputArray $directIncludedPackages -BatchSize $BATCHSIZE
+$directBatches = @()
+if ($directIncludedPackages) {
+    $directBatches = Split-ArrayIntoBatches -InputArray $directIncludedPackages -BatchSize $BATCHSIZE
+}
+
 $indirectBatches = @()
 if ($indirectIncludedPackages) {
     $indirectBatches = Split-ArrayIntoBatches -InputArray $indirectIncludedPackages -BatchSize $BATCHSIZE

scripts/devops_tasks/tox_harness.py

@@ -318,7 +318,7 @@ def prep_and_run_tox(targeted_packages: List[str], parsed_args: Namespace) -> No
                         if check not in skipped_tox_checks:
                             skipped_tox_checks[check] = []
 
-                    skipped_tox_checks[check].append(parsed_package)
+                        skipped_tox_checks[check].append(parsed_package)
 
             if not filtered_tox_environment_set:
                 logging.info(
@@ -330,7 +330,6 @@ def prep_and_run_tox(targeted_packages: List[str], parsed_args: Namespace) -> No
 
             tox_execution_array.extend(["-e", filtered_tox_environment_set])
 
-
         if parsed_args.tox_env == "apistub":
             local_options_array = []
             if parsed_args.dest_dir:
@@ -355,9 +354,9 @@ def prep_and_run_tox(targeted_packages: List[str], parsed_args: Namespace) -> No
                     "setup_execute_tests.py -> tox_harness.py::prep_and_run_tox",
             )
 
-    return_code = execute_tox_serial(tox_command_tuples)
+    return_result = execute_tox_serial(tox_command_tuples)
 
     if not parsed_args.disablecov:
         collect_tox_coverage_files(targeted_packages)
 
-    sys.exit(return_code)
+    sys.exit(return_result) #type: ignore

scripts/release_sdk_status/main.py

@@ -13,8 +13,6 @@
 from packaging.version import parse
 from pathlib import Path
 
-from util import add_certificate
-
 MAIN_REPO_SWAGGER = 'https://github.com/Azure/azure-rest-api-specs/tree/main'
 PR_URL = 'https://github.com/Azure/azure-rest-api-specs/pull/'
 FAILED_RESULT = []
@@ -54,10 +52,6 @@ def print_call(cmd):
     sp.call(cmd, shell=True)
 
 
-def start_test_proxy():
-    print_check('pwsh {}/eng/common/testproxy/docker-start-proxy.ps1 \"start\"'.format(os.getenv('SDK_REPO')))
-
-
 def version_sort(versions: List[str]) -> List[str]:
     versions_package = [parse(version) for version in versions]
     versions_package.sort()
@@ -231,67 +225,8 @@ def bot_analysis(self):
             self.bot_warning += 'Need to add track2 config.'
 
 
-def sdk_code_path(service_name, sdk_name) -> str:
-    return str(Path(os.getenv('SDK_REPO') + f'/sdk/{service_name}/{sdk_name}'))
-
-
-@return_origin_path
-def install_package_locally(service_name, sdk_name):
-    os.chdir(sdk_code_path(service_name, sdk_name))
-    print_check('pip install -e .')
-    print_exec('pip install -r dev_requirements.txt')
-
-
-@return_origin_path
-def run_test_proc(sdk_name, service_name, sdk_folder):
-    # run test
-    if os.getenv('SKIP_COVERAGE') in ('true', 'yes'):
-        return SKIP_TEXT
-
-    coverage_path = ''.join([os.getenv('SDK_REPO'), '/sdk/', sdk_folder])
-    service_path = coverage_path.split('/azure/mgmt')[0]
-    os.chdir(sdk_code_path(service_name, sdk_name))
-    try:
-        print_check(f'pytest  --collect-only')
-    except:
-        print('live test run done, do not find any test !!!')
-        return '-, -, -, -\n'
-    if os.path.exists(coverage_path + '/operations') and os.path.exists(coverage_path + '/models'):
-        operations_path = coverage_path + '/operations'
-        models_path = coverage_path + '/models'
-        try:
-            start_time = int(time.time())
-            print_check(f'pytest tests -s --cov={operations_path} --cov={models_path} >result.txt', path=service_path)
-            cost_time = int(time.time()) - start_time
-            my_print(f'{service_name} play_back cost {cost_time} seconds({cost_time // 60} minutes)')
-        except Exception as e:
-            print(f'{service_name} test ERROR')
-            return '-, 0, 0, 0\n'
-    else:
-        try:
-            print_check(f'pytest tests -s >result.txt', path=service_path)
-        except Exception as e:
-            return '-, 0, 0, 0\n'
-    if os.path.exists(service_path + '/result.txt'):
-        return get_test_result(service_path + '/result.txt')
-
-
-def run_test(sdk_name, service_name, sdk_folder):
-    install_package_locally(service_name, sdk_name)
-    test_result = run_test_proc(sdk_name, service_name, sdk_folder)
-    return test_result
-
-
-def clean_test_env():
-    for item in ("SSL_CERT_DIR", "REQUESTS_CA_BUNDLE"):
-        if os.getenv(item):
-            os.environ.pop(item)
-
-
 def sdk_info_from_pypi(sdk_info: List[Dict[str, str]], cli_dependency):
     all_sdk_status = []
-    add_certificate()
-    start_test_proxy()
     for package in sdk_info:
         sdk_name = package['package_name']
         if sdk_name in cli_dependency.keys():
@@ -306,41 +241,13 @@ def sdk_info_from_pypi(sdk_info: List[Dict[str, str]], cli_dependency):
                               readme_link=readme_link, rm_link=rm_link, cli_version=cli_version, multi_api=multi_api)
         sdk_folder = package['sdk_folder']
         text_to_write = pypi_ins.write_to_list(sdk_folder)
-        service_name = package['service_name']
         if pypi_ins.pypi_link != 'NA':
-            test_result = SKIP_TEXT
-            try:
-                test_result = run_test(sdk_name, service_name, sdk_folder)
-            except:
-                print(f'[Error] fail to play back test recordings: {sdk_name}')
-            text_to_write += test_result
-            all_sdk_status.append(text_to_write)
-
-    clean_test_env()
+            all_sdk_status.append(text_to_write + SKIP_TEXT)
+
     my_print(f'total pypi package kinds: {len(all_sdk_status)}')
     return all_sdk_status
 
 
-def get_test_result(txt_path):
-    with open(txt_path, 'r+') as f:
-        coverage = ' - '
-        for line in f.readlines():
-            if 'TOTAL' in line:
-                coverage = line.split()[3]
-            if '=====' in line and ('passed' in line or 'failed' in line or 'skipped' in line):
-                # print(line)
-                passed, failed, skipped = 0, 0, 0
-                if 'passed' in line:
-                    passed = re.findall('(\d{1,2}) passed', line)[0]
-                if 'failed' in line:
-                    failed = re.findall('(\d{1,2}) failed', line)[0]
-                if 'skipped' in line:
-                    skipped = re.findall('(\d{1,2}) skipped', line)[0]
-                # print(f'{passed} {failed} {skipped}')
-
-    return f'{coverage}, {passed}, {failed}, {skipped}\n'
-
-
 def write_to_csv(sdk_status_list, csv_name):
     with open(csv_name, 'w') as file_out:
         file_out.write('foler/package name,'

scripts/release_sdk_status/release_sdk_status.yml

@@ -52,15 +52,10 @@ jobs:
         git clone ${FILE_REPO:0:8}$(USR_NAME):$(Yuchao-GitToken)@${FILE_REPO:8} $(pwd)/file-storage
         mkdir azure-rest-api-specs
         git clone https://github.com/Azure/azure-rest-api-specs.git $(pwd)/azure-rest-api-specs
-        mkdir sdk-repo
-        git clone https://github.com/Azure/azure-sdk-for-python.git $(pwd)/sdk-repo
 
         # import env variable
         export TOKEN=$(Yuchao-GitToken)
         export SWAGGER_REPO=$(pwd)/azure-rest-api-specs
-        export SDK_REPO=$(pwd)/sdk-repo
-        export SKIP_COVERAGE=$(SKIP_COVERAGE)
-        export AZURE_TEST_RUN_LIVE=false
 
         # create virtual env
         python -m venv venv-sdk

sdk/communication/azure-communication-identity/setup.py

@@ -69,7 +69,7 @@
         "pytyped": ["py.typed"],
     },
     python_requires=">=3.8",
-    install_requires=["msrest>=0.7.1", "azure-core>=1.24.0"],
+    install_requires=["msrest>=0.7.1", "azure-core>=1.27.0"],
     extras_require={":python_version<'3.8'": ["typing-extensions"]},
     project_urls={
         "Bug Reports": "https://github.com/Azure/azure-sdk-for-python/issues",