From 4792aa172ea109cae40ff6c364d1c47c783f47dc Mon Sep 17 00:00:00 2001 From: sallyom Date: Thu, 6 Jun 2024 21:13:20 -0400 Subject: [PATCH 1/9] test workflow with otel traces Signed-off-by: sallyom --- .github/workflows/test-trace-steps.yaml | 84 +++++++++++++++++++++++++ ci/trace-steps.py | 76 ++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 .github/workflows/test-trace-steps.yaml create mode 100644 ci/trace-steps.py diff --git a/.github/workflows/test-trace-steps.yaml b/.github/workflows/test-trace-steps.yaml new file mode 100644 index 00000000..d17ecb3c --- /dev/null +++ b/.github/workflows/test-trace-steps.yaml @@ -0,0 +1,84 @@ +# To run locally +# act -W .github/workflows/test-trace-steps.yaml --container-architecture linux/amd64 -b ci/logs:/logs + +name: Test Workflow + +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v5 + + - name: Install Python dependencies + run: | + pip install --no-cache-dir opentelemetry-sdk opentelemetry-exporter-otlp opentelemetry-instrumentation + + - name: Configure OpenTelemetry Collector + run: | + echo ' + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + exporters: + logging: + loglevel: debug + service: + pipelines: + traces: + receivers: [otlp] + exporters: [logging] + ' > otel-collector-config.yaml + + - name: Run OpenTelemetry Collector + run: | + curl --proto '=https' --tlsv1.2 -fOL https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.102.1/otelcol_0.102.1_linux_amd64.tar.gz + tar -xvf otelcol_0.102.1_linux_amd64.tar.gz + chmod +x otelcol + ./otelcol --config otel-collector-config.yaml & + + - name: Start build trace + run: | + export WORKFLOW_NAME="test-workflow" + export STEP_NAME="build" + export TRACE_ACTION="start" + python ci/trace-steps.py + + - name: Build + run: | + echo "Simulating build step..." + sleep 2 + + - name: End build trace + run: | + export WORKFLOW_NAME="test-workflow" + export STEP_NAME="build" + export TRACE_ACTION="end" + python ci/trace-steps.py + + - name: Start test trace + run: | + export WORKFLOW_NAME="test-workflow" + export STEP_NAME="test" + export TRACE_ACTION="start" + python ci/trace-steps.py + + - name: Test + run: | + echo "Simulating test step..." + sleep 2 + + - name: End test trace + run: | + export WORKFLOW_NAME="test-workflow" + export STEP_NAME="test" + export TRACE_ACTION="end" + python ci/trace-steps.py diff --git a/ci/trace-steps.py b/ci/trace-steps.py new file mode 100644 index 00000000..19e51eae --- /dev/null +++ b/ci/trace-steps.py @@ -0,0 +1,76 @@ +import os +import time +import logging +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter +from opentelemetry.trace import SpanContext, TraceFlags, TraceState, NonRecordingSpan + +# Set up logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Set up OpenTelemetry tracing +trace.set_tracer_provider( + TracerProvider( + resource=Resource.create({"service.name": os.getenv("WORKFLOW_NAME")}) + ) +) +tracer = trace.get_tracer(__name__) + +# Set up OTLP exporter to send to OpenTelemetry Collector +otlp_exporter = OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True) + +# Set up span processor +span_processor = BatchSpanProcessor(otlp_exporter) +trace.get_tracer_provider().add_span_processor(span_processor) + +# Export to console for debugging +console_exporter = ConsoleSpanExporter() +trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(console_exporter)) + +def retry_operation(operation, retries=3, delay=5): + for attempt in range(retries): + try: + return operation() + except Exception as e: + logger.error(f"Attempt {attempt + 1} failed with error: {e}") + if attempt < retries - 1: + time.sleep(delay) + else: + raise + +def start_trace(step_name): + span = tracer.start_span(name=step_name) + return span + +def end_trace(span): + span.end() + +if __name__ == "__main__": + step_name = os.getenv("STEP_NAME", "default_step") + action = os.getenv("TRACE_ACTION", "start") + + if action == "start": + span = retry_operation(lambda: start_trace(step_name)) + with open(f"/tmp/trace_{step_name}.txt", "w") as f: + f.write(str(span.get_span_context().trace_id)) + elif action == "end": + trace_id = os.getenv("TRACE_ID") + if not trace_id: + with open(f"/tmp/trace_{step_name}.txt", "r") as f: + trace_id = f.read().strip() + trace_id = int(trace_id, 16) + span_context = SpanContext( + trace_id=trace_id, + span_id=0, # Span ID will be generated + trace_flags=TraceFlags(TraceFlags.SAMPLED), + trace_state=TraceState(), + is_remote=True + ) + with tracer.start_as_current_span(name=step_name, context=trace.set_span_in_context(NonRecordingSpan(span_context))): + span = retry_operation(lambda: tracer.start_span(name=step_name)) + retry_operation(lambda: end_trace(span)) + From f37bebeebadf18817ee244fd2716ba911a00c829 Mon Sep 17 00:00:00 2001 From: sallyom Date: Thu, 6 Jun 2024 21:56:28 -0400 Subject: [PATCH 2/9] add traces to testing_framework Signed-off-by: sallyom --- .github/workflows/test-trace-steps.yaml | 9 +- .github/workflows/testing_framework.yaml | 141 ++++++++++++++++++++++- 2 files changed, 143 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-trace-steps.yaml b/.github/workflows/test-trace-steps.yaml index d17ecb3c..74caa55b 100644 --- a/.github/workflows/test-trace-steps.yaml +++ b/.github/workflows/test-trace-steps.yaml @@ -3,7 +3,8 @@ name: Test Workflow -on: [push] +on: + workflow_dispatch: jobs: test: @@ -13,8 +14,10 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v5 + - name: Set up Python + uses: actions/setup-python@v5.1.0 + with: + python-version: '3.11' - name: Install Python dependencies run: | diff --git a/.github/workflows/testing_framework.yaml b/.github/workflows/testing_framework.yaml index 01b76ea7..126f7b6a 100644 --- a/.github/workflows/testing_framework.yaml +++ b/.github/workflows/testing_framework.yaml @@ -48,6 +48,47 @@ jobs: with: path: main + - name: Set up Python + uses: actions/setup-python@v5.1.0 + with: + python-version: '3.11' + + - name: Install Python dependencies for otel trace generation + run: | + pip install --no-cache-dir opentelemetry-sdk opentelemetry-exporter-otlp opentelemetry-instrumentation + + - name: Configure OpenTelemetry Collector + run: | + echo ' + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + exporters: + logging: + loglevel: debug + service: + pipelines: + traces: + receivers: [otlp] + exporters: [logging] + ' > otel-collector-config.yaml + + - name: Run OpenTelemetry Collector + run: | + curl --proto '=https' --tlsv1.2 -fOL https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.102.1/otelcol_0.102.1_linux_amd64.tar.gz + tar -xvf otelcol_0.102.1_linux_amd64.tar.gz + chmod +x otelcol + ./otelcol --config otel-collector-config.yaml & + + - name: Start integration-tests trace + run: | + export WORKFLOW_NAME="testing_framework" + export STEP_NAME="integration-tests" + export TRACE_ACTION="start" + python ci/trace-steps.py + - name: Checkout terraform module id: checkout-module uses: actions/checkout@v4.1.7 @@ -66,6 +107,13 @@ jobs: run: terraform init working-directory: terraform-test-environment-module + - name: Start bootstrap trace + run: | + export WORKFLOW_NAME="testing_framework" + export STEP_NAME="bootstrap" + export TRACE_ACTION="start" + python ci/trace-steps.py + - name: Bootstrap id: up run: terraform apply -auto-approve -lock=false @@ -74,6 +122,13 @@ jobs: TF_VAR_aws_instance_type: ${{ matrix.aws_image_type }} TF_VAR_aws_ami_architecture: ${{ matrix.aws_ami_architecture }} + - name: End bootstrap trace + run: | + export WORKFLOW_NAME="testing_framework" + export STEP_NAME="bootstrap" + export TRACE_ACTION="end" + python ci/trace-steps.py + - name: Terraform Output id: terraform-output run: | @@ -86,6 +141,13 @@ jobs: run: ansible-galaxy install -r ./provision/requirements.yml working-directory: ./main/recipes/natural_language_processing/chatbot + - name: Start ansible provision trace + run: | + export WORKFLOW_NAME="testing_framework" + export STEP_NAME="ansible-provision" + export TRACE_ACTION="start" + python ci/trace-steps.py + - name: Provision run: | ansible-playbook ./main/recipes/natural_language_processing/chatbot/provision/playbook.yml \ @@ -94,21 +156,37 @@ jobs: env: ANSIBLE_HOST_KEY_CHECKING: false - - name: Set up Python - uses: actions/setup-python@v5.1.0 - with: - python-version: '3.11' + - name: End ansible provision trace + run: | + export WORKFLOW_NAME="testing_framework" + export STEP_NAME="ansible-provision" + export TRACE_ACTION="end" + python ci/trace-steps.py - name: Install Dependencies working-directory: ./main/recipes/natural_language_processing/chatbot run: make install + - name: Start integration test trace + run: | + export WORKFLOW_NAME="testing_framework" + export STEP_NAME="run-integration-tests" + export TRACE_ACTION="start" + python ci/trace-steps.py + - name: Run Integration Tests working-directory: ./main/recipes/natural_language_processing/chatbot run: make integration-tests env: URL: ${{ steps.terraform-output.outputs.url }} + - name: End integration test trace + run: | + export WORKFLOW_NAME="testing_framework" + export STEP_NAME="run-integration-tests" + export TRACE_ACTION="end" + python ci/trace-steps.py + - name: Destroy Test Environment id: down if: always() @@ -130,6 +208,13 @@ jobs: env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + - name: Stop testing_framework trace + run: | + export WORKFLOW_NAME="testing_framework" + export STEP_NAME="integration-tests" + export TRACE_ACTION="end" + python ci/trace-steps.py + release-images: runs-on: ubuntu-22.04 needs: integration-tests @@ -173,10 +258,58 @@ jobs: steps: - uses: actions/checkout@v4.1.7 + - name: Set up Python + uses: actions/setup-python@v5.1.0 + with: + python-version: '3.11' + + - name: Install Python dependencies for otel trace generation + run: | + pip install --no-cache-dir opentelemetry-sdk opentelemetry-exporter-otlp opentelemetry-instrumentation + + - name: Configure OpenTelemetry Collector + run: | + echo ' + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + exporters: + logging: + loglevel: debug + service: + pipelines: + traces: + receivers: [otlp] + exporters: [logging] + ' > otel-collector-config.yaml + + - name: Run OpenTelemetry Collector + run: | + curl --proto '=https' --tlsv1.2 -fOL https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.102.1/otelcol_0.102.1_linux_amd64.tar.gz + tar -xvf otelcol_0.102.1_linux_amd64.tar.gz + chmod +x otelcol + ./otelcol --config otel-collector-config.yaml & + + - name: Start chatbot make bootc trace + run: | + export WORKFLOW_NAME="testing_framework" + export STEP_NAME="test-make-targets" + export TRACE_ACTION="start" + python ci/trace-steps.py + - name: chatbot working-directory: ./recipes/natural_language_processing/chatbot run: make bootc + - name: End chatbot make bootc trace + run: | + export WORKFLOW_NAME="testing_framework" + export STEP_NAME="test-make-targets" + export TRACE_ACTION="end" + python ci/trace-steps.py + - name: Publish Job Results to Slack id: slack if: always() From 8089c222813576f9943ef441ad06bdd2b03c95dc Mon Sep 17 00:00:00 2001 From: sallyom Date: Tue, 18 Jun 2024 11:15:39 -0400 Subject: [PATCH 3/9] update workflow triggers Signed-off-by: sallyom --- .github/workflows/test-trace-steps.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-trace-steps.yaml b/.github/workflows/test-trace-steps.yaml index 74caa55b..0fa41d21 100644 --- a/.github/workflows/test-trace-steps.yaml +++ b/.github/workflows/test-trace-steps.yaml @@ -4,6 +4,11 @@ name: Test Workflow on: + pull_request: + branches: + - main + paths: + - .github/workflows/test-trace-steps.yaml workflow_dispatch: jobs: @@ -14,10 +19,10 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v5.1.0 - with: - python-version: '3.11' + - name: Set up Python + uses: actions/setup-python@v5.1.0 + with: + python-version: '3.11' - name: Install Python dependencies run: | From 0a27f9f80c9f6c143e238167d445349989c11865 Mon Sep 17 00:00:00 2001 From: sallyom Date: Tue, 18 Jun 2024 11:40:49 -0400 Subject: [PATCH 4/9] fix indent Signed-off-by: sallyom --- .github/workflows/testing_framework.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/testing_framework.yaml b/.github/workflows/testing_framework.yaml index 126f7b6a..ddb94b34 100644 --- a/.github/workflows/testing_framework.yaml +++ b/.github/workflows/testing_framework.yaml @@ -122,12 +122,12 @@ jobs: TF_VAR_aws_instance_type: ${{ matrix.aws_image_type }} TF_VAR_aws_ami_architecture: ${{ matrix.aws_ami_architecture }} - - name: End bootstrap trace - run: | - export WORKFLOW_NAME="testing_framework" - export STEP_NAME="bootstrap" - export TRACE_ACTION="end" - python ci/trace-steps.py + - name: End bootstrap trace + run: | + export WORKFLOW_NAME="testing_framework" + export STEP_NAME="bootstrap" + export TRACE_ACTION="end" + python ci/trace-steps.py - name: Terraform Output id: terraform-output From 6a72550c6e567edc85778ada75507f4d6a3858e1 Mon Sep 17 00:00:00 2001 From: sallyom Date: Tue, 18 Jun 2024 13:41:44 -0400 Subject: [PATCH 5/9] add traces to chatbot workflow Signed-off-by: sallyom --- .github/workflows/chatbot.yaml | 93 ++++++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 4 deletions(-) diff --git a/.github/workflows/chatbot.yaml b/.github/workflows/chatbot.yaml index 947e7cd3..6bb352b1 100644 --- a/.github/workflows/chatbot.yaml +++ b/.github/workflows/chatbot.yaml @@ -39,11 +39,52 @@ jobs: steps: - uses: actions/checkout@v4.1.7 + - name: Set up Python + uses: actions/setup-python@v5.1.0 + with: + python-version: '3.11' + + - name: Install opentelemetry dependencies + run: | + pip install --no-cache-dir opentelemetry-sdk opentelemetry-exporter-otlp opentelemetry-instrumentation + + - name: Configure OpenTelemetry Collector + run: | + echo ' + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + exporters: + logging: + loglevel: debug + service: + pipelines: + traces: + receivers: [otlp] + exporters: [logging] + ' > otel-collector-config.yaml + + - name: Run OpenTelemetry Collector + run: | + curl --proto '=https' --tlsv1.2 -fOL https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.102.1/otelcol_0.102.1_linux_amd64.tar.gz + tar -xvf otelcol_0.102.1_linux_amd64.tar.gz + chmod +x otelcol + ./otelcol --config otel-collector-config.yaml & + - name: Install qemu dependency run: | sudo apt-get update sudo apt-get install -y qemu-user-static + - name: Start build trace + run: | + export WORKFLOW_NAME="chatbot-build-push" + export STEP_NAME="build-image" + export TRACE_ACTION="start" + python ci/trace-steps.py + - name: Build Image id: build_image uses: redhat-actions/buildah-build@v2.13 @@ -54,24 +95,54 @@ jobs: containerfiles: ./recipes/natural_language_processing/${{ env.IMAGE_NAME }}/app/Containerfile context: recipes/natural_language_processing/${{ env.IMAGE_NAME }}/app - - name: Set up Python - uses: actions/setup-python@v5.1.0 - with: - python-version: '3.11' + - name: End build trace + run: | + export WORKFLOW_NAME="chatbot-build-push" + export STEP_NAME="build-image" + export TRACE_ACTION="end" + python ci/trace-steps.py - name: Install Dependencies working-directory: ./recipes/natural_language_processing/${{ env.IMAGE_NAME }} run: make install + - name: Start download model trace + run: | + export WORKFLOW_NAME="chatbot-build-push" + export STEP_NAME="download-model" + export TRACE_ACTION="start" + python ci/trace-steps.py + - name: Download model working-directory: ./models run: make download-model-granite + - name: End download model trace + run: | + export WORKFLOW_NAME="chatbot-build-push" + export STEP_NAME="download-model" + export TRACE_ACTION="end" + python ci/trace-steps.py + + - name: Start functional test run trace + run: | + export WORKFLOW_NAME="chatbot-build-push" + export STEP_NAME="run-functional-tests" + export TRACE_ACTION="start" + python ci/trace-steps.py + - name: Run Functional Tests shell: bash run: make functional-tests working-directory: ./recipes/natural_language_processing/${{ env.IMAGE_NAME }} + - name: End functional test run trace + run: | + export WORKFLOW_NAME="chatbot-build-push" + export STEP_NAME="run-functional-tests" + export TRACE_ACTION="end" + python ci/trace-steps.py + - name: Login to Registry if: github.event_name == 'push' && github.ref == 'refs/heads/main' uses: redhat-actions/podman-login@v1.7 @@ -80,6 +151,13 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Start push image trace + run: | + export WORKFLOW_NAME="chatbot-build-push" + export STEP_NAME="push-image" + export TRACE_ACTION="start" + python ci/trace-steps.py + - name: Push Image id: push_image if: github.event_name == 'push' && github.ref == 'refs/heads/main' @@ -88,3 +166,10 @@ jobs: image: ${{ steps.build_image.outputs.image }} tags: ${{ steps.build_image.outputs.tags }} registry: ${{ env.REGISTRY }} + + - name: End push image trace + run: | + export WORKFLOW_NAME="chatbot-build-push" + export STEP_NAME="push-image" + export TRACE_ACTION="end" + python ci/trace-steps.py From 3af8da05220776c8376f7a0f2683386a075f9b3e Mon Sep 17 00:00:00 2001 From: sallyom Date: Wed, 19 Jun 2024 21:45:18 -0400 Subject: [PATCH 6/9] mTLS & configure otel-collector to ROSA-ET Signed-off-by: sallyom --- .github/workflows/chatbot.yaml | 9 ++++++++- .github/workflows/test-trace-steps.yaml | 11 +++++++++-- .github/workflows/testing_framework.yaml | 9 ++++++++- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/.github/workflows/chatbot.yaml b/.github/workflows/chatbot.yaml index 6bb352b1..5036d5de 100644 --- a/.github/workflows/chatbot.yaml +++ b/.github/workflows/chatbot.yaml @@ -57,13 +57,20 @@ jobs: grpc: endpoint: "0.0.0.0:4317" exporters: + otlphttp: + endpoint: https://apps.platform-sts.pcbk.p1.openshiftapps.com + tls: + insecure: false + cert_pem: ${{ secrets.CLIENT_CERT_ROSA_OTEL }} + key_pem: ${{ secrets.CLIENT_KEY_ROSA_OTEL }} + ca_pem: ${{ secrets.SERVER_CERT_ROSA_OTEL }} logging: loglevel: debug service: pipelines: traces: receivers: [otlp] - exporters: [logging] + exporters: [logging,otlphttp] ' > otel-collector-config.yaml - name: Run OpenTelemetry Collector diff --git a/.github/workflows/test-trace-steps.yaml b/.github/workflows/test-trace-steps.yaml index 0fa41d21..011893dc 100644 --- a/.github/workflows/test-trace-steps.yaml +++ b/.github/workflows/test-trace-steps.yaml @@ -37,14 +37,21 @@ jobs: grpc: endpoint: "0.0.0.0:4317" exporters: + otlphttp: + endpoint: https://apps.platform-sts.pcbk.p1.openshiftapps.com + tls: + insecure: false + cert_pem: ${{ secrets.CLIENT_CERT_ROSA_OTEL }} + key_pem: ${{ secrets.CLIENT_KEY_ROSA_OTEL }} + ca_pem: ${{ secrets.SERVER_CERT_ROSA_OTEL }} logging: loglevel: debug service: pipelines: traces: receivers: [otlp] - exporters: [logging] - ' > otel-collector-config.yaml + exporters: [logging,otlphttp] + ' > otel-collector-config.yaml - name: Run OpenTelemetry Collector run: | diff --git a/.github/workflows/testing_framework.yaml b/.github/workflows/testing_framework.yaml index ddb94b34..5bcc3a30 100644 --- a/.github/workflows/testing_framework.yaml +++ b/.github/workflows/testing_framework.yaml @@ -66,13 +66,20 @@ jobs: grpc: endpoint: "0.0.0.0:4317" exporters: + otlphttp: + endpoint: https://apps.platform-sts.pcbk.p1.openshiftapps.com + tls: + insecure: false + cert_pem: ${{ secrets.CLIENT_CERT_ROSA_OTEL }} + key_pem: ${{ secrets.CLIENT_KEY_ROSA_OTEL }} + ca_pem: ${{ secrets.SERVER_CERT_ROSA_OTEL }} logging: loglevel: debug service: pipelines: traces: receivers: [otlp] - exporters: [logging] + exporters: [logging,otlphttp] ' > otel-collector-config.yaml - name: Run OpenTelemetry Collector From dc012ed809deac8792ccda11a8bdf3199d787cdf Mon Sep 17 00:00:00 2001 From: sallyom Date: Thu, 20 Jun 2024 17:46:51 -0400 Subject: [PATCH 7/9] debug curl Signed-off-by: sallyom --- .github/workflows/chatbot.yaml | 4 ++- .github/workflows/test-trace-steps.yaml | 4 ++- .github/workflows/testing_framework.yaml | 6 +++-- ci/trace-steps.py | 33 +++++------------------- 4 files changed, 16 insertions(+), 31 deletions(-) diff --git a/.github/workflows/chatbot.yaml b/.github/workflows/chatbot.yaml index 5036d5de..4a67983b 100644 --- a/.github/workflows/chatbot.yaml +++ b/.github/workflows/chatbot.yaml @@ -56,6 +56,8 @@ jobs: protocols: grpc: endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" exporters: otlphttp: endpoint: https://apps.platform-sts.pcbk.p1.openshiftapps.com @@ -65,7 +67,7 @@ jobs: key_pem: ${{ secrets.CLIENT_KEY_ROSA_OTEL }} ca_pem: ${{ secrets.SERVER_CERT_ROSA_OTEL }} logging: - loglevel: debug + verbosity: detailed service: pipelines: traces: diff --git a/.github/workflows/test-trace-steps.yaml b/.github/workflows/test-trace-steps.yaml index 011893dc..b5a688c2 100644 --- a/.github/workflows/test-trace-steps.yaml +++ b/.github/workflows/test-trace-steps.yaml @@ -36,6 +36,8 @@ jobs: protocols: grpc: endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" exporters: otlphttp: endpoint: https://apps.platform-sts.pcbk.p1.openshiftapps.com @@ -45,7 +47,7 @@ jobs: key_pem: ${{ secrets.CLIENT_KEY_ROSA_OTEL }} ca_pem: ${{ secrets.SERVER_CERT_ROSA_OTEL }} logging: - loglevel: debug + verbosity: detailed service: pipelines: traces: diff --git a/.github/workflows/testing_framework.yaml b/.github/workflows/testing_framework.yaml index 5bcc3a30..55b795d5 100644 --- a/.github/workflows/testing_framework.yaml +++ b/.github/workflows/testing_framework.yaml @@ -4,7 +4,7 @@ on: schedule: # schedule the job to run once a day - cron: '0 0 * * *' - workflow_dispatch: + #workflow_dispatch: # pull_request: ## temporary for debugging development purposes # branches: @@ -65,6 +65,8 @@ jobs: protocols: grpc: endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" exporters: otlphttp: endpoint: https://apps.platform-sts.pcbk.p1.openshiftapps.com @@ -74,7 +76,7 @@ jobs: key_pem: ${{ secrets.CLIENT_KEY_ROSA_OTEL }} ca_pem: ${{ secrets.SERVER_CERT_ROSA_OTEL }} logging: - loglevel: debug + verbosity: detailed service: pipelines: traces: diff --git a/ci/trace-steps.py b/ci/trace-steps.py index 19e51eae..3e654df5 100644 --- a/ci/trace-steps.py +++ b/ci/trace-steps.py @@ -1,6 +1,4 @@ import os -import time -import logging from opentelemetry import trace from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter from opentelemetry.sdk.resources import Resource @@ -8,10 +6,6 @@ from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter from opentelemetry.trace import SpanContext, TraceFlags, TraceState, NonRecordingSpan -# Set up logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - # Set up OpenTelemetry tracing trace.set_tracer_provider( TracerProvider( @@ -21,30 +15,15 @@ tracer = trace.get_tracer(__name__) # Set up OTLP exporter to send to OpenTelemetry Collector -otlp_exporter = OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True) - -# Set up span processor -span_processor = BatchSpanProcessor(otlp_exporter) -trace.get_tracer_provider().add_span_processor(span_processor) +otlp_exporter = OTLPSpanExporter(endpoint="0.0.0.0:4317", insecure=True) +trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(otlp_exporter)) # Export to console for debugging console_exporter = ConsoleSpanExporter() trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(console_exporter)) -def retry_operation(operation, retries=3, delay=5): - for attempt in range(retries): - try: - return operation() - except Exception as e: - logger.error(f"Attempt {attempt + 1} failed with error: {e}") - if attempt < retries - 1: - time.sleep(delay) - else: - raise - def start_trace(step_name): - span = tracer.start_span(name=step_name) - return span + return tracer.start_span(name=step_name) def end_trace(span): span.end() @@ -54,7 +33,7 @@ def end_trace(span): action = os.getenv("TRACE_ACTION", "start") if action == "start": - span = retry_operation(lambda: start_trace(step_name)) + span = start_trace(step_name) with open(f"/tmp/trace_{step_name}.txt", "w") as f: f.write(str(span.get_span_context().trace_id)) elif action == "end": @@ -71,6 +50,6 @@ def end_trace(span): is_remote=True ) with tracer.start_as_current_span(name=step_name, context=trace.set_span_in_context(NonRecordingSpan(span_context))): - span = retry_operation(lambda: tracer.start_span(name=step_name)) - retry_operation(lambda: end_trace(span)) + span = start_trace(step_name) + end_trace(span) From 1f0f1f9ca6b869bb81fd038a1ecac91d93abc90d Mon Sep 17 00:00:00 2001 From: sallyom Date: Fri, 21 Jun 2024 13:38:00 -0400 Subject: [PATCH 8/9] update workflow Signed-off-by: sallyom --- .github/workflows/chatbot.yaml | 28 ++++++++------- .github/workflows/test-trace-steps.yaml | 46 +++++++++++++++---------- 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/.github/workflows/chatbot.yaml b/.github/workflows/chatbot.yaml index 4a67983b..fa418f4b 100644 --- a/.github/workflows/chatbot.yaml +++ b/.github/workflows/chatbot.yaml @@ -51,36 +51,40 @@ jobs: - name: Configure OpenTelemetry Collector run: | echo ' + extensions: + basicauth/client: + client_auth: + username: "${{ secrets.OTEL_USERNAME }}" + password: "${{ secrets.OTEL_PASSWORD }}" receivers: otlp: protocols: grpc: - endpoint: "0.0.0.0:4317" http: - endpoint: "0.0.0.0:4318" exporters: otlphttp: - endpoint: https://apps.platform-sts.pcbk.p1.openshiftapps.com + endpoint: https://otc.apps.platform-sts.pcbk.p1.openshiftapps.com + auth: + authenticator: basicauth/client tls: insecure: false - cert_pem: ${{ secrets.CLIENT_CERT_ROSA_OTEL }} - key_pem: ${{ secrets.CLIENT_KEY_ROSA_OTEL }} - ca_pem: ${{ secrets.SERVER_CERT_ROSA_OTEL }} - logging: + ca_pem: "${{ secrets.ROSA_ROOT_CERT }}" + debug: verbosity: detailed service: + extensions: [basicauth/client] pipelines: traces: receivers: [otlp] - exporters: [logging,otlphttp] + exporters: [debug, otlphttp] ' > otel-collector-config.yaml - name: Run OpenTelemetry Collector run: | - curl --proto '=https' --tlsv1.2 -fOL https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.102.1/otelcol_0.102.1_linux_amd64.tar.gz - tar -xvf otelcol_0.102.1_linux_amd64.tar.gz - chmod +x otelcol - ./otelcol --config otel-collector-config.yaml & + wget https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.103.0/otelcol-contrib_0.103.0_linux_amd64.tar.gz + tar -xvf otelcol-contrib_0.103.0_linux_amd64.tar.gz + chmod +x otelcol-contrib + ./otelcol-contrib --config otel-collector-config.yaml & - name: Install qemu dependency run: | diff --git a/.github/workflows/test-trace-steps.yaml b/.github/workflows/test-trace-steps.yaml index b5a688c2..9c406cbe 100644 --- a/.github/workflows/test-trace-steps.yaml +++ b/.github/workflows/test-trace-steps.yaml @@ -13,12 +13,10 @@ on: jobs: test: - runs-on: ubuntu-latest - + if: "!contains(github.event.pull_request.labels.*.name, 'hold-tests')" + runs-on: ubuntu-22.04 steps: - - name: Checkout code - uses: actions/checkout@v2 - + - uses: actions/checkout@v4.1.7 - name: Set up Python uses: actions/setup-python@v5.1.0 with: @@ -28,39 +26,51 @@ jobs: run: | pip install --no-cache-dir opentelemetry-sdk opentelemetry-exporter-otlp opentelemetry-instrumentation + - name: Download OpenTelemetry Collector Contrib + run: | + wget https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.103.0/otelcol-contrib_0.103.0_linux_amd64.tar.gz + tar -xvf otelcol-contrib_0.103.0_linux_amd64.tar.gz + + - name: CA secret to 1-line + env: + ROSA_OTEL_CABUNDLE: ${{ secrets.ROSA_OTEL_CABUNDLE }} + run: | + echo "$ROSA_OTEL_CABUNDLE" > /tmp/ca.crt + - name: Configure OpenTelemetry Collector run: | echo ' + extensions: + basicauth/client: + client_auth: + username: "${{ secrets.OTEL_USERNAME }}" + password: "${{ secrets.OTEL_PASSWORD }}" receivers: otlp: protocols: grpc: - endpoint: "0.0.0.0:4317" http: - endpoint: "0.0.0.0:4318" exporters: otlphttp: - endpoint: https://apps.platform-sts.pcbk.p1.openshiftapps.com + endpoint: https://otc.apps.platform-sts.pcbk.p1.openshiftapps.com + auth: + authenticator: basicauth/client tls: insecure: false - cert_pem: ${{ secrets.CLIENT_CERT_ROSA_OTEL }} - key_pem: ${{ secrets.CLIENT_KEY_ROSA_OTEL }} - ca_pem: ${{ secrets.SERVER_CERT_ROSA_OTEL }} - logging: + ca_file: /tmp/ca.crt + debug: verbosity: detailed service: + extensions: [basicauth/client] pipelines: traces: receivers: [otlp] - exporters: [logging,otlphttp] - ' > otel-collector-config.yaml + exporters: [debug, otlphttp] + ' > otel-collector-config.yaml - name: Run OpenTelemetry Collector run: | - curl --proto '=https' --tlsv1.2 -fOL https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.102.1/otelcol_0.102.1_linux_amd64.tar.gz - tar -xvf otelcol_0.102.1_linux_amd64.tar.gz - chmod +x otelcol - ./otelcol --config otel-collector-config.yaml & + ./otelcol-contrib --config otel-collector-config.yaml & - name: Start build trace run: | From 4e36394c1790d4c60ff678e2a86f98d272dcc449 Mon Sep 17 00:00:00 2001 From: sallyom Date: Sat, 22 Jun 2024 23:46:18 -0400 Subject: [PATCH 9/9] another try Signed-off-by: sallyom --- .github/workflows/test-trace-steps.yaml | 15 ++++++----- ci/trace-steps.py | 36 +++++++++++++++++++------ 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test-trace-steps.yaml b/.github/workflows/test-trace-steps.yaml index 9c406cbe..31ad6423 100644 --- a/.github/workflows/test-trace-steps.yaml +++ b/.github/workflows/test-trace-steps.yaml @@ -31,11 +31,9 @@ jobs: wget https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.103.0/otelcol-contrib_0.103.0_linux_amd64.tar.gz tar -xvf otelcol-contrib_0.103.0_linux_amd64.tar.gz - - name: CA secret to 1-line - env: - ROSA_OTEL_CABUNDLE: ${{ secrets.ROSA_OTEL_CABUNDLE }} + - name: Write secrets to files run: | - echo "$ROSA_OTEL_CABUNDLE" > /tmp/ca.crt + echo "${{ secrets.ROSA_OTEL_TLS_CERT }}" > /tmp/tls.crt - name: Configure OpenTelemetry Collector run: | @@ -52,12 +50,12 @@ jobs: http: exporters: otlphttp: - endpoint: https://otc.apps.platform-sts.pcbk.p1.openshiftapps.com + endpoint: https://otc.apps.platform-sts.pcbk.p1.openshiftapps.com:4318 auth: authenticator: basicauth/client tls: insecure: false - ca_file: /tmp/ca.crt + ca_file: /tmp/tls.crt debug: verbosity: detailed service: @@ -70,7 +68,7 @@ jobs: - name: Run OpenTelemetry Collector run: | - ./otelcol-contrib --config otel-collector-config.yaml & + ./otelcol-contrib --config otel-collector-config.yaml > otel-collector.log 2>&1 & - name: Start build trace run: | @@ -109,3 +107,6 @@ jobs: export STEP_NAME="test" export TRACE_ACTION="end" python ci/trace-steps.py + + - name: Display OpenTelemetry Collector Logs + run: cat otel-collector.log diff --git a/ci/trace-steps.py b/ci/trace-steps.py index 3e654df5..a7c77e99 100644 --- a/ci/trace-steps.py +++ b/ci/trace-steps.py @@ -1,4 +1,6 @@ import os +import time +import logging from opentelemetry import trace from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter from opentelemetry.sdk.resources import Resource @@ -6,6 +8,10 @@ from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter from opentelemetry.trace import SpanContext, TraceFlags, TraceState, NonRecordingSpan +# Set up logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + # Set up OpenTelemetry tracing trace.set_tracer_provider( TracerProvider( @@ -15,15 +21,30 @@ tracer = trace.get_tracer(__name__) # Set up OTLP exporter to send to OpenTelemetry Collector -otlp_exporter = OTLPSpanExporter(endpoint="0.0.0.0:4317", insecure=True) -trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(otlp_exporter)) +otlp_exporter = OTLPSpanExporter(endpoint="http://0.0.0.0:4317", insecure=True) + +# Set up span processor +span_processor = BatchSpanProcessor(otlp_exporter) +trace.get_tracer_provider().add_span_processor(span_processor) -# Export to console for debugging +# Optionally, export to console for debugging console_exporter = ConsoleSpanExporter() trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(console_exporter)) +def retry_operation(operation, retries=3, delay=5): + for attempt in range(retries): + try: + return operation() + except Exception as e: + logger.error(f"Attempt {attempt + 1} failed with error: {e}") + if attempt < retries - 1: + time.sleep(delay) + else: + raise + def start_trace(step_name): - return tracer.start_span(name=step_name) + span = tracer.start_span(name=step_name) + return span def end_trace(span): span.end() @@ -33,7 +54,7 @@ def end_trace(span): action = os.getenv("TRACE_ACTION", "start") if action == "start": - span = start_trace(step_name) + span = retry_operation(lambda: start_trace(step_name)) with open(f"/tmp/trace_{step_name}.txt", "w") as f: f.write(str(span.get_span_context().trace_id)) elif action == "end": @@ -41,7 +62,7 @@ def end_trace(span): if not trace_id: with open(f"/tmp/trace_{step_name}.txt", "r") as f: trace_id = f.read().strip() - trace_id = int(trace_id, 16) + trace_id = int(trace_id, 16) # Convert trace_id back to int span_context = SpanContext( trace_id=trace_id, span_id=0, # Span ID will be generated @@ -50,6 +71,5 @@ def end_trace(span): is_remote=True ) with tracer.start_as_current_span(name=step_name, context=trace.set_span_in_context(NonRecordingSpan(span_context))): - span = start_trace(step_name) + span = tracer.start_span(name=step_name) end_trace(span) -