From 228a7df97bb96a86fd48621ba069edfc0a8f63e3 Mon Sep 17 00:00:00 2001 From: Matthew Watkins Date: Wed, 28 May 2025 10:26:15 +0100 Subject: [PATCH] Fix: Broken build and test pipeline Change-Id: I41c0bde1d10705bd4ec08366bff16f1ef3b3369d Signed-off-by: Matthew Watkins --- .github/workflows/gerrit-verify.yaml | 142 ++++++++++++--------- .github/workflows/release.yaml | 17 ++- lftools/deploy.py | 42 +++++- ...a.cert.test.AaaCertMdsalProviderTest-output.txt | 10 +- ...ht.aaa.cert.test.AaaCertProviderTest-output.txt | 6 +- ....cert.test.AaaCertRpcServiceImplTest-output.txt | 11 +- ...aaa.cert.test.KeyStoresDataUtilsTest-output.txt | 5 +- 7 files changed, 151 insertions(+), 82 deletions(-) diff --git a/.github/workflows/gerrit-verify.yaml b/.github/workflows/gerrit-verify.yaml index 1a00c6a6..5b685321 100644 --- a/.github/workflows/gerrit-verify.yaml +++ b/.github/workflows/gerrit-verify.yaml @@ -47,9 +47,12 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.inputs.GERRIT_BRANCH}}-${{ github.event.inputs.GERRIT_CHANGE_ID || github.run_id }} cancel-in-progress: true +permissions: {} + jobs: clear-vote: runs-on: ubuntu-latest + permissions: {} steps: - name: Clear votes # yamllint disable-line rule:line-length @@ -68,6 +71,7 @@ jobs: needs: clear-vote # yamllint disable-line rule:line-length uses: lfit/releng-reusable-workflows/.github/workflows/compose-repo-linting.yaml@main + permissions: {} with: GERRIT_BRANCH: ${{ inputs.GERRIT_BRANCH }} GERRIT_CHANGE_ID: ${{ inputs.GERRIT_CHANGE_ID }} @@ -79,74 +83,92 @@ jobs: GERRIT_PROJECT: ${{ inputs.GERRIT_PROJECT }} GERRIT_REFSPEC: ${{ inputs.GERRIT_REFSPEC }} - prepare: + python-build: + name: 'Python Build' needs: clear-vote - runs-on: ubuntu-latest + runs-on: 'ubuntu-latest' outputs: - wheel-distribution: ${{ steps.wheel-distribution.outputs.path }} + matrix_json: "${{ steps.python-build.outputs.matrix_json }}" + artefact_name: "${{ steps.python-build.outputs.artefact_name }}" + artefact_path: "${{ steps.python-build.outputs.artefact_path }}" + permissions: + contents: write + timeout-minutes: 12 + env: + GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}" steps: - - name: Checkout change - uses: lfit/checkout-gerrit-change-action@54d751e8bd167bc91f7d665dabe33fae87aaaa63 # v0.9 + # Harden the runner used by this workflow + - uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0 with: - gerrit-refspec: ${{ inputs.GERRIT_REFSPEC }} - - name: Configure Python - uses: actions/setup-python@v5 + egress-policy: 'audit' + + - uses: lfit/checkout-gerrit-change-action@54d751e8bd167bc91f7d665dabe33fae87aaaa63 # v0.9 + + - name: 'Build Python project' + id: python-build + # yamllint disable-line rule:line-length + uses: lfreleng-actions/python-build-action@a9d0ef8a2324ac76e798ad6dc306f08b83b5b213 # v0.1.11 + + python-tests: + name: 'Python Tests' + runs-on: 'ubuntu-latest' + needs: + - clear-vote + - python-build + # Matrix job + strategy: + fail-fast: false + matrix: "${{ fromJson(needs.python-build.outputs.matrix_json) }}" + permissions: + contents: read + timeout-minutes: 12 + steps: + # Harden the runner used by this workflow + - uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0 with: - python-version: '3.8' - - name: Build package distribution files - run: >- - pipx run tox -e clean,build - - name: Record the path of wheel distribution - id: wheel-distribution - run: echo "path=$(ls dist/*.whl)" >> "$GITHUB_OUTPUT" - - name: Store the distribution files for use in other stages - # `tests` and `publish` will use the same pre-built distributions, - # so we make sure to release the exact same package that was tested - uses: actions/upload-artifact@v4 + egress-policy: audit + + - uses: lfit/checkout-gerrit-change-action@54d751e8bd167bc91f7d665dabe33fae87aaaa63 # v0.9 + + - name: "Python tests [pytest] ${{ matrix.python-version }}" + # yamllint disable-line rule:line-length + uses: lfreleng-actions/python-test-action@16adb21ca4866bebc75e7b35203ce5b376b01430 # v0.1.7 with: - name: python-distribution-files - path: dist/ - retention-days: 1 + python_version: ${{ matrix.python-version }} - test: - needs: prepare + python-audit: + name: 'Python Audit' + needs: + - clear-vote + - python-build runs-on: ubuntu-latest + # Matrix job strategy: - matrix: - python: - - "3.8" - - "3.9" + fail-fast: false + matrix: "${{ fromJson(needs.python-build.outputs.matrix_json) }}" + permissions: + contents: read + timeout-minutes: 10 steps: - - name: Checkout change - # yamllint disable-line rule:line-length - uses: lfit/checkout-gerrit-change-action@54d751e8bd167bc91f7d665dabe33fae87aaaa63 # v0.9 - with: - gerrit-refspec: ${{ inputs.GERRIT_REFSPEC }} - gerrit-project: ${{ inputs.GERRIT_PROJECT }} - gerrit-url: ${{ vars.GERRIT_URL }} - delay: "0s" - - name: Configure Python - uses: actions/setup-python@v5 - id: setup-python + # Harden the runner used by this workflow + - uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0 with: - python-version: ${{ matrix.python }} - - name: Retrieve pre-built distribution files - uses: actions/download-artifact@v4 - with: {name: python-distribution-files, path: dist/} - - name: Enable sar - run: | - sudo systemctl start sysstat - - name: Run tests - run: >- - TOX_SKIP_ENV='(docs*|license|pre-commit)' - pipx run --python '${{ steps.setup-python.outputs.python-path }}' - tox --installpkg '${{ needs.prepare.outputs.wheel-distribution }}' - -- -rFEx --durations 10 --color yes # pytest args + egress-policy: 'audit' + - uses: lfit/checkout-gerrit-change-action@54d751e8bd167bc91f7d665dabe33fae87aaaa63 # v0.9 + + - name: "Audit dependencies ${{ matrix.python-version }}" + # yamllint disable-line rule:line-length + uses: lfreleng-actions/python-audit-action@4c51bc76f9876b4f294f8afa4bb002b0b89aec68 # v0.1.3 + with: + python_version: "${{ matrix.python-version }}" + never_fail: true docs: needs: clear-vote runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout change # yamllint disable-line rule:line-length @@ -158,8 +180,7 @@ jobs: delay: "0s" fetch-depth: 0 - name: Configure Python - uses: actions/setup-python@v5 - id: setup-python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.8" - name: Run docs tests @@ -170,6 +191,8 @@ jobs: docs-linkcheck: needs: clear-vote runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout change # yamllint disable-line rule:line-length @@ -181,8 +204,7 @@ jobs: delay: "0s" fetch-depth: 0 - name: Configure Python - uses: actions/setup-python@v5 - id: setup-python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.8" - name: Run docs-linkcheck tests @@ -192,6 +214,8 @@ jobs: license: needs: clear-vote runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout change # yamllint disable-line rule:line-length @@ -202,8 +226,7 @@ jobs: gerrit-url: ${{ vars.GERRIT_URL }} delay: "0s" - name: Configure Python - uses: actions/setup-python@v5 - id: setup-python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.8" - name: Run license tests @@ -212,8 +235,9 @@ jobs: vote: if: ${{ always() }} - needs: [prepare, test, docs, docs-linkcheck, license, lint] + needs: [clear-vote, python-build, python-tests, python-audit, docs, docs-linkcheck, license, lint] runs-on: ubuntu-latest + permissions: {} steps: - name: Get workflow conclusion # yamllint disable-line rule:line-length diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b4827364..f1410375 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -4,20 +4,27 @@ name: PyPI release # yamllint disable-line rule:truthy on: push +permissions: {} + jobs: publish: if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') runs-on: ubuntu-latest + env: + GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + permissions: + contents: write steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Configure Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.8" - - name: Build package distribution files - run: >- - pipx run tox -e clean,build + - name: 'Build Python project' + id: python-build + # yamllint disable-line rule:line-length + uses: lfreleng-actions/python-build-action@a9d0ef8a2324ac76e798ad6dc306f08b83b5b213 # v0.1.11 - name: Configure TWINE run: | echo "TWINE_USERNAME=__token__" >> "$GITHUB_ENV" diff --git a/lftools/deploy.py b/lftools/deploy.py index 5583b46c..253664a9 100755 --- a/lftools/deploy.py +++ b/lftools/deploy.py @@ -13,6 +13,7 @@ import concurrent.futures import datetime import errno +import fnmatch import glob import gzip import io @@ -26,6 +27,7 @@ import subprocess import sys import tempfile import zipfile +from pathlib import Path import boto3 import requests @@ -260,12 +262,48 @@ def copy_archives(workspace, pattern=None): no_dups_pattern = _remove_duplicates_and_sort(pattern) paths = [] + + # Debug: List all files in workspace for troubleshooting + log.debug("Workspace contents before pattern matching:") + for root, dirs, files in os.walk(workspace): + for file in files: + rel_path = os.path.relpath(os.path.join(root, file), workspace) + log.debug(" {}".format(rel_path)) + + # Use pathlib for more reliable pattern matching across Python versions + workspace_path = Path(workspace) + for p in no_dups_pattern: if p == "": # Skip empty patterns as they are invalid continue - search = os.path.join(workspace, p) - paths.extend(glob.glob(search, recursive=True)) + log.debug("Searching for pattern: {}".format(p)) + + # Handle recursive patterns with pathlib.rglob() for better Python 3.8 compatibility + if p.startswith("**/"): + # Use rglob for recursive patterns like "**/*.txt" + pattern_suffix = p[3:] # Remove "**/" prefix + found_paths = list(workspace_path.rglob(pattern_suffix)) + log.debug("Using rglob for pattern '{}' -> rglob('{}')".format(p, pattern_suffix)) + elif "**" in p: + # For other recursive patterns, fall back to manual traversal with fnmatch + found_paths = [] + for file_path in workspace_path.rglob("*"): + if file_path.is_file(): + relative_path = file_path.relative_to(workspace_path) + if fnmatch.fnmatch(str(relative_path), p): + found_paths.append(file_path) + log.debug("Using fnmatch for complex pattern '{}'".format(p)) + else: + # For simple patterns without **, use glob + found_paths = list(workspace_path.glob(p)) + log.debug("Using glob for simple pattern '{}'".format(p)) + + # Convert to absolute string paths + absolute_paths = [str(path) for path in found_paths if path.is_file()] + log.debug("Found files for pattern '{}': {}".format(p, absolute_paths)) + paths.extend(absolute_paths) + log.debug("Files found: {}".format(paths)) no_dups_paths = _remove_duplicates_and_sort(paths) diff --git a/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertMdsalProviderTest-output.txt b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertMdsalProviderTest-output.txt index e529ce52..c30a9da5 100644 --- a/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertMdsalProviderTest-output.txt +++ b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertMdsalProviderTest-output.txt @@ -1,6 +1,4 @@ -[main] INFO org.opendaylight.aaa.cert.impl.ODLKeyTool - odlTest.jks is created -[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized -[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized -[main] INFO org.opendaylight.aaa.cert.impl.ODLKeyTool - odlTest.jks is created -[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized -[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized +------------------------------------------------------------------------------- +Test set: org.opendaylight.aaa.cert.test.AaaCertMdsalProviderTest +------------------------------------------------------------------------------- +Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.001 sec diff --git a/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertProviderTest-output.txt b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertProviderTest-output.txt index 80ba15ad..d3cab9de 100644 --- a/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertProviderTest-output.txt +++ b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertProviderTest-output.txt @@ -1,2 +1,4 @@ -[main] INFO org.opendaylight.aaa.cert.impl.AaaCertProvider - aaa Certificate Service Initalized -[main] INFO org.opendaylight.aaa.cert.impl.ODLKeyTool - ctlTest.jks is created +------------------------------------------------------------------------------- +Test set: org.opendaylight.aaa.cert.test.AaaCertProviderTest +------------------------------------------------------------------------------- +Tests run: 2, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.005 sec diff --git a/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertRpcServiceImplTest-output.txt b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertRpcServiceImplTest-output.txt index 97bfffab..03a5fd3c 100644 --- a/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertRpcServiceImplTest-output.txt +++ b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertRpcServiceImplTest-output.txt @@ -1,7 +1,4 @@ -[main] INFO org.opendaylight.aaa.cert.impl.ODLKeyTool - odlTest.jks is created -[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized -[main] INFO org.opendaylight.aaa.cert.impl.AaaCertRpcServiceImpl - AaaCert Rpc Service has been initialized -[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized -[main] INFO org.opendaylight.aaa.cert.impl.AaaCertRpcServiceImpl - AaaCert Rpc Service has been initialized -[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized -[main] INFO org.opendaylight.aaa.cert.impl.AaaCertRpcServiceImpl - AaaCert Rpc Service has been initialized +------------------------------------------------------------------------------- +Test set: org.opendaylight.aaa.cert.test.AaaCertRpcServiceImplTest +------------------------------------------------------------------------------- +Tests run: 3, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.002 sec diff --git a/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.KeyStoresDataUtilsTest-output.txt b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.KeyStoresDataUtilsTest-output.txt index 13003b64..ac2b7522 100644 --- a/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.KeyStoresDataUtilsTest-output.txt +++ b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.KeyStoresDataUtilsTest-output.txt @@ -1 +1,4 @@ -[main] INFO org.opendaylight.aaa.cert.impl.ODLKeyTool - odlTest.jks is created +------------------------------------------------------------------------------- +Test set: org.opendaylight.aaa.cert.test.KeyStoresDataUtilsTest +------------------------------------------------------------------------------- +Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.001 sec -- 2.16.6