From 6f5d0a25790cf9cf36eb27605f2ddb391e164c83 Mon Sep 17 00:00:00 2001 From: ganglv <88995770+ganglyu@users.noreply.github.com> Date: Tue, 12 Jul 2022 14:53:46 +0800 Subject: [PATCH 01/16] Create lgtm.yml --- lgtm.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 lgtm.yml diff --git a/lgtm.yml b/lgtm.yml new file mode 100644 index 000000000000..7d275c44858c --- /dev/null +++ b/lgtm.yml @@ -0,0 +1,2 @@ +path_classifiers: + test: "tests" From 1b1c25483cb4467da514fed054268872bf0b0a13 Mon Sep 17 00:00:00 2001 From: ganglv <88995770+ganglyu@users.noreply.github.com> Date: Tue, 12 Jul 2022 15:18:55 +0800 Subject: [PATCH 02/16] Update lgtm.yml --- lgtm.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lgtm.yml b/lgtm.yml index 7d275c44858c..0200260cf76b 100644 --- a/lgtm.yml +++ b/lgtm.yml @@ -1,2 +1,9 @@ path_classifiers: test: "tests" + +extraction: + python: + index: + filters: + - include: "host_modules/*.py" + - include: "scripts/*" From 6228fef9f92cce3791220edbe9b9b65f8f89b058 Mon Sep 17 00:00:00 2001 From: ganglv <88995770+ganglyu@users.noreply.github.com> Date: Tue, 12 Jul 2022 15:41:02 +0800 Subject: [PATCH 03/16] Create azure-pipelines.yml --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000000..8b137891791f --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1 @@ + From a34d58134964b22208b1e2947e14a4ee39063efe Mon Sep 17 00:00:00 2001 From: ganglv <88995770+ganglyu@users.noreply.github.com> Date: Tue, 12 Jul 2022 16:05:03 +0800 Subject: [PATCH 04/16] Update pytest.ini --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index c4b03d4c3455..548dc83b224b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,2 @@ [pytest] -addopts = --cov=scripts --cov-report html --cov-report term --cov-report xml --ignore=tests/*/test*_vectors.py +addopts = --cov=scripts --cov-report html --cov-report term --cov-report xml --ignore=tests/*/test*_vectors.py --junitxml=test-results.xml -vv From 291570cdee99beba349b6d3184db2f825e4f9ca7 Mon Sep 17 00:00:00 2001 From: ganglv <88995770+ganglyu@users.noreply.github.com> Date: Tue, 12 Jul 2022 16:05:35 +0800 Subject: [PATCH 05/16] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 114 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8b137891791f..99352bd55f04 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1 +1,115 @@ +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +trigger: +- master + +resources: + repositories: + - repository: sonic-swss + type: github + name: Azure/sonic-swss + endpoint: build + +stages: +- stage: Build + + jobs: + - job: + displayName: "Python3" + variables: + DIFF_COVER_CHECK_THRESHOLD: 50 + DIFF_COVER_ENABLE: 'true' + pool: + vmImage: ubuntu-20.04 + + container: + image: sonicdev-microsoft.azurecr.io:443/sonic-slave-bullseye:latest + + steps: + - script: | + sourceBranch=$(Build.SourceBranchName) + if [[ "$(Build.Reason)" == "PullRequest" ]];then + sourceBranch=$(System.PullRequest.TargetBranch) + fi + echo "Download artifact branch: $sourceBranch" + echo "##vso[task.setvariable variable=sourceBranch]$sourceBranch" + displayName: "Get correct artifact downloading branch" + - task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: 142 + artifact: sonic-buildimage.vs + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/$(sourceBranch)' + displayName: "Download artifacts from latest sonic-buildimage build" + + - script: | + set -xe + workingDirectory: $(Pipeline.Workspace)/target/debs/bullseye/ + displayName: 'Install Debian dependencies' + + - task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: 9 + artifact: sonic-swss-common + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/master' + displayName: "Download sonic swss common deb packages" + + - script: | + set -xe + sudo dpkg -i libswsscommon_1.0.0_amd64.deb + sudo dpkg -i python3-swsscommon_1.0.0_amd64.deb + workingDirectory: $(Pipeline.Workspace)/ + displayName: 'Install swss-common dependencies' + + - script: | + set -xe + sudo pip3 install swsssdk-2.0.1-py3-none-any.whl + sudo pip3 install sonic_py_common-1.0-py3-none-any.whl + workingDirectory: $(Pipeline.Workspace)/target/python-wheels/bullseye/ + displayName: 'Install Python dependencies' + + - script: | + set -ex + # Install .NET CORE + curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - + sudo apt-add-repository https://packages.microsoft.com/debian/11/prod + sudo apt-get update + sudo apt-get install -y dotnet-sdk-5.0 + displayName: "Install .NET CORE" + + - script: | + python3 setup.py test + displayName: 'Test Python 3' + + - task: PublishTestResults@2 + inputs: + testResultsFiles: '$(System.DefaultWorkingDirectory)/test-results.xml' + testRunTitle: Python 3 + failTaskOnFailedTests: true + condition: succeededOrFailed() + displayName: 'Publish Python 3 test results' + + - task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml' + reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov/' + displayName: 'Publish Python 3 test coverage' + + - script: | + set -e + python3 setup.py bdist_wheel + displayName: 'Build Python 3 wheel' + + - publish: '$(System.DefaultWorkingDirectory)/dist/' + artifact: wheels + displayName: "Publish Python wheels" From 0b1fefea07c2c37590c1e8f5b89229033ee93bd8 Mon Sep 17 00:00:00 2001 From: ganglv <88995770+ganglyu@users.noreply.github.com> Date: Tue, 12 Jul 2022 16:12:02 +0800 Subject: [PATCH 06/16] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 99352bd55f04..90a4d52661a5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -6,13 +6,6 @@ trigger: - master -resources: - repositories: - - repository: sonic-swss - type: github - name: Azure/sonic-swss - endpoint: build - stages: - stage: Build @@ -49,6 +42,12 @@ stages: - script: | set -xe + sudo apt-get -y purge libhiredis-dev libnl-3-dev libnl-route-3-dev + sudo dpkg -i libnl-3-200_*.deb + sudo dpkg -i libnl-genl-3-200_*.deb + sudo dpkg -i libnl-route-3-200_*.deb + sudo dpkg -i libnl-nf-3-200_*.deb + sudo dpkg -i libhiredis0.14_*.deb workingDirectory: $(Pipeline.Workspace)/target/debs/bullseye/ displayName: 'Install Debian dependencies' From 2834a75187543b2fe5cb86a3a959ec069b25ea2d Mon Sep 17 00:00:00 2001 From: ganglv <88995770+ganglyu@users.noreply.github.com> Date: Tue, 12 Jul 2022 16:27:34 +0800 Subject: [PATCH 07/16] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 90a4d52661a5..39614b5db49b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,7 +19,7 @@ stages: vmImage: ubuntu-20.04 container: - image: sonicdev-microsoft.azurecr.io:443/sonic-slave-bullseye:latest + image: sonicdev-microsoft.azurecr.io:443/sonic-slave-buster:latest steps: - script: | @@ -48,7 +48,7 @@ stages: sudo dpkg -i libnl-route-3-200_*.deb sudo dpkg -i libnl-nf-3-200_*.deb sudo dpkg -i libhiredis0.14_*.deb - workingDirectory: $(Pipeline.Workspace)/target/debs/bullseye/ + workingDirectory: $(Pipeline.Workspace)/target/debs/buster/ displayName: 'Install Debian dependencies' - task: DownloadPipelineArtifact@2 @@ -72,14 +72,14 @@ stages: set -xe sudo pip3 install swsssdk-2.0.1-py3-none-any.whl sudo pip3 install sonic_py_common-1.0-py3-none-any.whl - workingDirectory: $(Pipeline.Workspace)/target/python-wheels/bullseye/ + workingDirectory: $(Pipeline.Workspace)/target/python-wheels/buster/ displayName: 'Install Python dependencies' - script: | set -ex # Install .NET CORE curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - sudo apt-add-repository https://packages.microsoft.com/debian/11/prod + sudo apt-add-repository https://packages.microsoft.com/debian/10/prod sudo apt-get update sudo apt-get install -y dotnet-sdk-5.0 displayName: "Install .NET CORE" From 84e823fc52003195b83ec873ee6222e44ec4e42d Mon Sep 17 00:00:00 2001 From: ganglv <88995770+ganglyu@users.noreply.github.com> Date: Tue, 12 Jul 2022 16:43:46 +0800 Subject: [PATCH 08/16] Update azure-pipelines.yml --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 39614b5db49b..5562f129b937 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -48,6 +48,7 @@ stages: sudo dpkg -i libnl-route-3-200_*.deb sudo dpkg -i libnl-nf-3-200_*.deb sudo dpkg -i libhiredis0.14_*.deb + sudo dpkg -i sonic-host-services-data_1.0-1_all.deb workingDirectory: $(Pipeline.Workspace)/target/debs/buster/ displayName: 'Install Debian dependencies' From 71cb1936aef213874cf09a30cc38799de4c1f08d Mon Sep 17 00:00:00 2001 From: ganglv <88995770+ganglyu@users.noreply.github.com> Date: Tue, 12 Jul 2022 17:04:37 +0800 Subject: [PATCH 09/16] Update azure-pipelines.yml --- azure-pipelines.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5562f129b937..053875489dc8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,6 +22,11 @@ stages: image: sonicdev-microsoft.azurecr.io:443/sonic-slave-buster:latest steps: + - checkout: sonic-host-services-data + clean: true + submodules: recursive + displayName: 'Checkout code' + - script: | sourceBranch=$(Build.SourceBranchName) if [[ "$(Build.Reason)" == "PullRequest" ]];then @@ -48,7 +53,6 @@ stages: sudo dpkg -i libnl-route-3-200_*.deb sudo dpkg -i libnl-nf-3-200_*.deb sudo dpkg -i libhiredis0.14_*.deb - sudo dpkg -i sonic-host-services-data_1.0-1_all.deb workingDirectory: $(Pipeline.Workspace)/target/debs/buster/ displayName: 'Install Debian dependencies' From 5a7667ad5cbf7817b9a144ebbc5c3412e8df3d08 Mon Sep 17 00:00:00 2001 From: ganglv <88995770+ganglyu@users.noreply.github.com> Date: Tue, 12 Jul 2022 17:07:00 +0800 Subject: [PATCH 10/16] Update azure-pipelines.yml --- azure-pipelines.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 053875489dc8..992e1499f346 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -6,6 +6,13 @@ trigger: - master +resources: + repositories: + - repository: sonic-host-services-data + type: github + name: Azure/sonic-buildimage/src/sonic-host-services-data + endpoint: build + stages: - stage: Build From e365624fa743cb1b94b317e2f26c8d0d3da16045 Mon Sep 17 00:00:00 2001 From: ganglv <88995770+ganglyu@users.noreply.github.com> Date: Tue, 12 Jul 2022 19:42:40 +0800 Subject: [PATCH 11/16] Update azure-pipelines.yml --- azure-pipelines.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 992e1499f346..f52b3463580b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,9 +8,9 @@ trigger: resources: repositories: - - repository: sonic-host-services-data + - repository: sonic-buildimage type: github - name: Azure/sonic-buildimage/src/sonic-host-services-data + name: Azure/sonic-buildimage endpoint: build stages: @@ -29,10 +29,15 @@ stages: image: sonicdev-microsoft.azurecr.io:443/sonic-slave-buster:latest steps: - - checkout: sonic-host-services-data + - checkout: sonic-buildimage clean: true submodules: recursive displayName: 'Checkout code' + + - task: CopyFiles@2 + inputs: + contents: '$(System.DefaultWorkingDirectory)/sonic-buildimage/src/sonic-host-services-data/**' + targetFolder: $(System.DefaultWorkingDirectory)/sonic-host-services-data/ - script: | sourceBranch=$(Build.SourceBranchName) From adf284277ed94a0e1ee039628356ea00a80afca7 Mon Sep 17 00:00:00 2001 From: ganglv <88995770+ganglyu@users.noreply.github.com> Date: Wed, 13 Jul 2022 10:30:40 +0800 Subject: [PATCH 12/16] Update azure-pipelines.yml --- azure-pipelines.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f52b3463580b..cb3d8c80e546 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,11 +29,15 @@ stages: image: sonicdev-microsoft.azurecr.io:443/sonic-slave-buster:latest steps: - - checkout: sonic-buildimage + - checkout: self clean: true submodules: recursive displayName: 'Checkout code' + - checkout: sonic-buildimage + clean: true + displayName: 'Checkout code' + - task: CopyFiles@2 inputs: contents: '$(System.DefaultWorkingDirectory)/sonic-buildimage/src/sonic-host-services-data/**' @@ -102,12 +106,15 @@ stages: displayName: "Install .NET CORE" - script: | + ls -l + pushd sonic-host-services + python3 setup.py test displayName: 'Test Python 3' - task: PublishTestResults@2 inputs: - testResultsFiles: '$(System.DefaultWorkingDirectory)/test-results.xml' + testResultsFiles: '$(System.DefaultWorkingDirectory)/sonic-host-services/test-results.xml' testRunTitle: Python 3 failTaskOnFailedTests: true condition: succeededOrFailed() @@ -116,8 +123,8 @@ stages: - task: PublishCodeCoverageResults@1 inputs: codeCoverageTool: Cobertura - summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml' - reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov/' + summaryFileLocation: '$(System.DefaultWorkingDirectory)/sonic-host-services/coverage.xml' + reportDirectory: '$(System.DefaultWorkingDirectory)/sonic-host-services/htmlcov/' displayName: 'Publish Python 3 test coverage' - script: | @@ -125,7 +132,7 @@ stages: python3 setup.py bdist_wheel displayName: 'Build Python 3 wheel' - - publish: '$(System.DefaultWorkingDirectory)/dist/' + - publish: '$(System.DefaultWorkingDirectory)/sonic-host-services/dist/' artifact: wheels displayName: "Publish Python wheels" From aaef3b9a381be027cb1bb685705522647f811218 Mon Sep 17 00:00:00 2001 From: ganglv <88995770+ganglyu@users.noreply.github.com> Date: Wed, 13 Jul 2022 10:38:53 +0800 Subject: [PATCH 13/16] Update azure-pipelines.yml --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cb3d8c80e546..e83ec18a4dcc 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -40,7 +40,8 @@ stages: - task: CopyFiles@2 inputs: - contents: '$(System.DefaultWorkingDirectory)/sonic-buildimage/src/sonic-host-services-data/**' + SourceFolder: '$(System.DefaultWorkingDirectory)/sonic-buildimage/src/sonic-host-services-data/' + contents: '**' targetFolder: $(System.DefaultWorkingDirectory)/sonic-host-services-data/ - script: | From 2c0e6de1141ec7d7ac071946dff93427332982a1 Mon Sep 17 00:00:00 2001 From: ganglv <88995770+ganglyu@users.noreply.github.com> Date: Wed, 13 Jul 2022 10:45:12 +0800 Subject: [PATCH 14/16] Update azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e83ec18a4dcc..d6ea3df65016 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -107,7 +107,6 @@ stages: displayName: "Install .NET CORE" - script: | - ls -l pushd sonic-host-services python3 setup.py test @@ -130,6 +129,7 @@ stages: - script: | set -e + pushd sonic-host-services python3 setup.py bdist_wheel displayName: 'Build Python 3 wheel' From 4161b6961a8e574aa43ce3504254b252fc4ac4f6 Mon Sep 17 00:00:00 2001 From: ganglv <88995770+ganglyu@users.noreply.github.com> Date: Wed, 13 Jul 2022 11:19:04 +0800 Subject: [PATCH 15/16] Update azure-pipelines.yml --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d6ea3df65016..5f1109360f84 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,6 +22,7 @@ stages: variables: DIFF_COVER_CHECK_THRESHOLD: 50 DIFF_COVER_ENABLE: 'true' + DIFF_COVER_WORKING_DIRECTORY: $(System.DefaultWorkingDirectory)/sonic-host-services pool: vmImage: ubuntu-20.04 From 9fa4274f4006fbb4a1ecb74799a27a3f8dc7a87b Mon Sep 17 00:00:00 2001 From: ganglv <88995770+ganglyu@users.noreply.github.com> Date: Wed, 13 Jul 2022 11:52:53 +0800 Subject: [PATCH 16/16] Update azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5f1109360f84..83e3aa4dc33e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -20,7 +20,7 @@ stages: - job: displayName: "Python3" variables: - DIFF_COVER_CHECK_THRESHOLD: 50 + DIFF_COVER_CHECK_THRESHOLD: 80 DIFF_COVER_ENABLE: 'true' DIFF_COVER_WORKING_DIRECTORY: $(System.DefaultWorkingDirectory)/sonic-host-services pool: