Merge branch 'bluefield' into secure_boot_fix

.azure-pipelines/azure-pipelines-build-ubuntu-2004.yml

@@ -0,0 +1,55 @@
+# C/C++ with GCC
+# Build your C/C++ project with GCC using make.
+# Add steps that publish test results, save build artifacts, deploy, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/apps/c-cpp/gcc
+pr: none
+
+trigger:
+  batch: true
+  branches:
+    include:
+    - master
+
+stages:
+- stage: Build
+
+  jobs:
+  - job:
+    displayName: "amd64/ubuntu-20.04"
+    pool:
+      vmImage: 'ubuntu-20.04'
+
+    steps:
+    - checkout: self
+      submodules: true
+    - script: |
+        sudo apt-get update
+        sudo apt-get install -y make wget libtool m4 autoconf dh-exec libdebhelper-perl=13.6ubuntu1~bpo20.04.1 debhelper=13.6ubuntu1~bpo20.04.1 \
+                         cmake pkg-config python3-pip python cmake libgtest-dev libgmock-dev libyang-dev \
+                         debhelper-compat dh-elpa dh-sequence-python3 python3-all \
+                         libpython3-all-dev python3-six xmlto unzip rake-compiler gem2deb pkg-php-tools \
+                         ant default-jdk maven-repo-helper libguava-java
+        wget http://ftp.us.debian.org/debian/pool/main/libg/libgoogle-gson-java/libgoogle-gson-java_2.8.6-1+deb11u1_all.deb
+        sudo dpkg -i libgoogle-gson-java_2.8.6-1+deb11u1_all.deb
+        mkdir -p /tmp/artifacts
+      displayName: "Install dependencies"
+    - script: |
+        SONIC_CONFIG_MAKE_JOBS=$(nproc) CONFIGURED_ARCH=amd64 DEST=/tmp/artifacts make -f ../rules/protobuf.mk -f protobuf/Makefile
+      workingDirectory: src
+      displayName: "Build protobuf"
+    - script: |
+        sudo dpkg -i protobuf-compiler_3.21.12-3_amd64.deb libprotoc32_3.21.12-3_amd64.deb \
+                      libprotobuf32_3.21.12-3_amd64.deb libprotobuf-dev_3.21.12-3_amd64.deb \
+                      libprotobuf-lite32_3.21.12-3_amd64.deb
+      workingDirectory: /tmp/artifacts
+      displayName: "Install protobuf"
+    - script: |
+        dpkg-buildpackage -rfakeroot -b -us -uc
+      workingDirectory: src/sonic-dash-api
+      displayName: "Build sonic-dash-api"
+    - script: |
+        cp *.deb /tmp/artifacts
+      workingDirectory: src
+    - publish: /tmp/artifacts
+      artifact: sonic-buildimage.amd64.ubuntu20_04
+      displayName: "Archive sonic-buildimage debian packages for ubuntu20.04"

.azure-pipelines/azure-pipelines-build.yml

@@ -35,7 +35,6 @@ jobs:
       dbg_image: no
       asan_image: no
       swi_image: no
-      raw_image: no
       docker_syncd_rpc_image: no
       syncd_rpc_image: no
       platform_rpc: no
@@ -62,7 +61,6 @@ jobs:
           variables:
             dbg_image: yes
             swi_image: yes
-            raw_image: yes
             docker_syncd_rpc_image: yes
             platform_rpc: brcm
 
@@ -140,6 +138,10 @@ jobs:
               make $BUILD_OPTIONS ENABLE_ASAN=y target/docker-sonic-vs.gz
               mv target/docker-sonic-vs.gz target/docker-sonic-vs-asan.gz
             fi
+            if [ $(K8S_OPTIONS) == 'INCLUDE_KUBERNETES_MASTER=y' ]; then
+              make $BUILD_OPTIONS $(K8S_OPTIONS) target/sonic-vs.img.gz
+              mv target/sonic-vs.img.gz target/sonic-vs-k8s.img.gz
+            fi
             make $BUILD_OPTIONS target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
             make $BUILD_OPTIONS target/docker-ptf-sai.gz
             if [ $(gnmi_test_container) == yes ]; then
@@ -158,9 +160,6 @@ jobs:
             if [ $(swi_image) == yes ]; then
               make $BUILD_OPTIONS ENABLE_IMAGE_SIGNATURE=y target/sonic-aboot-$(GROUP_NAME).swi
             fi
-            if [ $(raw_image) == yes ]; then
-              make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).raw
-            fi
             if [ $(docker_syncd_rpc_image) == yes ]; then
               # workaround for issue in rules/sairedis.dep, git ls-files will list un-exist files for cache
               make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-rpc.gz

.azure-pipelines/build-template.yml

@@ -103,9 +103,6 @@ jobs:
         if [ ${{ parameters.swi_image }} == true ]; then
           make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y target/sonic-aboot-${{ parameters.platform }}.swi
         fi
-        if [ ${{ parameters.raw_image }} == true ]; then
-          make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.raw
-        fi
         if [ ${{ parameters.sync_rpc_image }} == true ]; then
           make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y target/docker-syncd-${{ parameters.platform_short }}-rpc.gz
           # workaround for issue in rules/sairedis.dep,  git ls-files will list un-exist files for cache

.azure-pipelines/dpkg-cache-cleanup.yml

@@ -15,13 +15,34 @@ trigger: none
 pr: none
 
 jobs:
-- job: Build
+- job: Amd
   pool: sonicbld
   timeoutInMinutes: 5
   steps:
   - checkout: none
   - script: |
-      set -xe
+      df -h
       sudo find /nfs/dpkg_cache/ -name *.tgz -mtime +30 -type f -delete
+      df -h
     displayName: clean dpkg cache
-    
+- job: Armhf
+  pool: sonicbld-armhf
+  timeoutInMinutes: 5
+  steps:
+  - checkout: none
+  - script: |
+      df -h
+      sudo find /nfs/dpkg_cache/ -name *.tgz -mtime +30 -type f -delete
+      df -h
+    displayName: clean dpkg cache
+- job: Arm64
+  pool: sonicbld-arm64
+  timeoutInMinutes: 5
+  steps:
+  - checkout: none
+  - script: |
+      df -h
+      sudo find /nfs/dpkg_cache/ -name *.tgz -mtime +30 -type f -delete
+      df -h
+    displayName: clean dpkg cache
+

.azure-pipelines/official-build-cache.yml

@@ -39,7 +39,6 @@ stages:
         - name: broadcom
           variables:
             swi_image: yes
-            raw_image: yes
             docker_syncd_rpc_image: yes
             platform_rpc: brcm
         - name: mellanox

.azure-pipelines/run-test-scheduler-template.yml

@@ -18,6 +18,10 @@ parameters:
   type: string
   default: 1
 
+- name: NUM_ASIC
+  type: number
+  default: 1
+
 - name: TEST_SET
   type: string
   default: ""
@@ -34,17 +38,73 @@ parameters:
   type: string
   default: "ceos"
 
-- name: SPECIFIED_PARAMS
+- name: TESTBED_NAME
+  type: string
+  default: ""
+
+- name: IMAGE_URL
+  type: string
+  default: ""
+
+- name: HWSKU
+  type: string
+  default: ""
+
+- name: TEST_PLAN_TYPE
+  type: string
+  default: ""
+
+- name: PLATFORM
+  type: string
+  default: ""
+
+- name: SCRIPTS
+  type: string
+  default: ""
+
+- name: FEATURES
+  type: string
+  default: ""
+
+- name: SCRIPTS_EXCLUDE
+  type: string
+  default: ""
+
+- name: FEATURES_EXCLUDE
   type: string
-  default: "{}"
+  default: ""
+
+- name: REPO_NAME
+  type: string
+  default: ""
 
 - name: MGMT_BRANCH
   type: string
-  default: master
+  default: ""
 
-- name: NUM_ASIC
+- name: STOP_ON_FAILURE
+  type: string
+  default: ""
+
+- name: RETRY_TIMES
+  type: string
+  default: ""
+
+- name: DUMP_KVM_IF_FAIL
+  type: string
+  default: "True"
+  values:
+  - "True"
+  - "False"
+
+- name: REQUESTER
+  type: string
+  default: ""
+
+- name: MAX_RUN_TEST_MINUTES
   type: number
-  default: 1
+  default: 480
+
 
 steps:
   - script: |
@@ -57,12 +117,20 @@ steps:
       set -e
       pip install PyYAML
       rm -f new_test_plan_id.txt
-      python ./.azure-pipelines/test_plan.py create -t ${{ parameters.TOPOLOGY }} -o new_test_plan_id.txt \
-      --min-worker ${{ parameters.MIN_WORKER }} --max-worker ${{ parameters.MAX_WORKER }} \
-      --test-set ${{ parameters.TEST_SET }} --kvm-build-id $(KVM_BUILD_ID) \
-      --deploy-mg-extra-params "${{ parameters.DEPLOY_MG_EXTRA_PARAMS }}" --common-extra-params "${{ parameters.COMMON_EXTRA_PARAMS }}" \
-      --mgmt-branch ${{ parameters.MGMT_BRANCH }} --vm-type ${{ parameters.VM_TYPE }} --specified-params "${{ parameters.SPECIFIED_PARAMS }}" \
+      
+      python ./.azure-pipelines/test_plan.py create \
+      -t ${{ parameters.TOPOLOGY }} \
+      -o new_test_plan_id.txt \
+      --min-worker ${{ parameters.MIN_WORKER }} \
+      --max-worker ${{ parameters.MAX_WORKER }} \
+      --test-set ${{ parameters.TEST_SET }} \
+      --kvm-build-id $(KVM_BUILD_ID) \
+      --deploy-mg-extra-params "${{ parameters.DEPLOY_MG_EXTRA_PARAMS }}" \
+      --common-extra-params "${{ parameters.COMMON_EXTRA_PARAMS }}" \
+      --mgmt-branch ${{ parameters.MGMT_BRANCH }} \
+      --vm-type ${{ parameters.VM_TYPE }} \
       --num-asic ${{ parameters.NUM_ASIC }}
+      
       TEST_PLAN_ID=`cat new_test_plan_id.txt`
 
       echo "Created test plan $TEST_PLAN_ID"
@@ -108,18 +176,20 @@ steps:
     env:
       TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
     displayName: Run test
-
-  - script: |
-      set -e
-      echo "KVM dump"
-      echo "SONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com"
-      echo "Runtime detailed progress at $(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID "
-      # When "KVMDUMP" finish, it changes into "FAILED", "CANCELLED" or "FINISHED"
-      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state KVMDUMP
-    condition: succeededOrFailed()
-    env:
-      TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
-    displayName: KVM dump
+    timeoutInMinutes: ${{ parameters.MAX_RUN_TEST_MINUTES }}
+
+  - ${{ if eq(parameters.DUMP_KVM_IF_FAIL, 'True') }}:
+    - script: |
+        set -e
+        echo "KVM dump"
+        echo "SONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com"
+        echo "Runtime detailed progress at $(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID "
+        # When "KVMDUMP" finish, it changes into "FAILED", "CANCELLED" or "FINISHED"
+        python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state KVMDUMP
+      condition: succeededOrFailed()
+      env:
+        TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
+      displayName: KVM dump
 
   - script: |
       set -e

.azure-pipelines/template-skipvstest.yml

@@ -2,7 +2,13 @@ steps:
 - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
   - script: |
       set -ex
-      tar_branch=origin/$(System.PullRequest.TargetBranch)
+      tar_branch=origin/$(System.PullRequest.TargetBranchName)
+      # Check if k8s master entrance script is changed
+      if ! git diff $tar_branch..HEAD --name-only | grep -F files/image_config/kubernetes/kubernetes_master_entrance.sh; then
+        echo "##vso[task.setvariable variable=K8S_OPTIONS;]INCLUDE_KUBERNETES_MASTER=n"
+      else
+        echo "##vso[task.setvariable variable=K8S_OPTIONS;]INCLUDE_KUBERNETES_MASTER=y"
+      fi
       git diff $tar_branch..HEAD --name-only | grep -v -f .azure-pipelines/vstest-exclude && exit 0
       git diff $tar_branch..HEAD --name-only | grep -f .azure-pipelines/vstest-include && exit 0
       set +x