From 30a6dacd8befef0dc78942a53068fa52a101cc9f Mon Sep 17 00:00:00 2001 From: "Lott, Christopher (cl778h)" Date: Wed, 6 Nov 2019 11:30:52 -0500 Subject: [PATCH] Add .pypirc config file provider to release job Add .pypirc config file provider to builders in release job. Extend PyPI documentation with recommended directory layout. Change-Id: I96814ad3c78187835ab26dd8adbb7532753315e9 Signed-off-by: Lott, Christopher (cl778h) --- docs/jjb/lf-python-jobs.rst | 150 +++++++++++++-------- jjb/lf-release-jobs.yaml | 2 + .../fix-pypi-config-release-d78e0dbe5e72d38d.yaml | 5 + 3 files changed, 104 insertions(+), 53 deletions(-) create mode 100644 releasenotes/notes/fix-pypi-config-release-d78e0dbe5e72d38d.yaml diff --git a/docs/jjb/lf-python-jobs.rst b/docs/jjb/lf-python-jobs.rst index 2cb9516e..3b142647 100644 --- a/docs/jjb/lf-python-jobs.rst +++ b/docs/jjb/lf-python-jobs.rst @@ -333,13 +333,23 @@ variables before running. https://docs.openstack.org/infra/jenkins-job-builder/triggers.html#triggers.gerrit -PyPI Verify ------------ +PyPI Merge +---------- -Verifies a Python library project on creation of a patch set. Runs tox -then builds a source distribution and (optionally) a binary -distribution. The project repository must have a setup.py file with -configuration for packaging the component. +Creates and uploads distribution files on merge of a patch set. Runs +tox, builds a source distribution and (optionally) a binary +distribution, and uploads the distribution(s) to a PyPI repository. +The project git repository must have a setup.py file +with configuration for packaging the component. + +This job should use a staging repository like testpypi.python.org, +which sets up use of release jobs to promote the distributions later. +This job can also use a public release area like the global PyPI +repository if the release process is not needed. These PyPI +repositories allow upload of a package at a specific version once, +they do not allow overwrite of a package. This means that a merge job +will fail in the upload step if the package version already exists in +the target repository. The tox runner is pyenv aware so if the image contains an installation of pyenv at /opt/pyenv it will pick it up and run Python tests with @@ -351,15 +361,73 @@ pyenv variables before running. export PYENV_ROOT="/opt/pyenv" export PATH="$PYENV_ROOT/bin:$PATH" +Installable package projects should consider using a directory layout +like the one shown below. All Python files are below a directory, and +the package source files are nested in a subdirectory. This layout +allows straightforward build-job triggers using the directory paths so +Jenkins jobs are triggered appropriately. For example, a PyPI merge +job should not be triggered by a non-Python change, because the job +cannot upload the same package twice. + +.. code-block:: bash + + git-repo-name/ + │ + ├── docs/ + │ └── index.rst + │ + ├── helloworld-package/ + │ │ + │ └── helloworld/ + │ │ ├── __init__.py + │ │ ├── helloworld.py + │ │ └── helpers.py + │ │ + │ ├── tests/ + │ │ ├── helloworld_tests.py + │ │ └── helloworld_mocks.py + │ │ + │ ├── requirements.txt + │ └── setup.py + │ └── tox.ini + │ + ├── releases/ + │ └── release-helloworld.yaml + │ + ├── .gitignore + ├── LICENSE + └── README.md + + +Use of these templates requires a .pypirc configuration file in the +Jenkins builder home directory. An example appears next that uses API +tokens. Note that in the [pypi] entry the repository key-value pair is +optional, it defaults to pypi.org. + +.. code-block:: bash + + [distutils] # this tells distutils what package indexes you can push to + index-servers = pypi-test pypi + + [pypi-test] + repository: https://test.pypi.org/legacy/ + username: __token__ + password: pypi-test-api-token-goes-here + + [pypi] + username: __token__ + password: pypi-api-token-goes-here + + :Template Names: - - {project-name}-pypi-verify-{stream} - - gerrit-pypi-verify - - github-pypi-verify + - {project-name}-pypi-merge-{stream} + - gerrit-pypi-merge + - github-pypi-merge -:Comment Trigger: **recheck|reverify** post a comment with one of the - triggers to launch this job manually. Do not include any other - text or vote in the same comment. +:Comment Trigger: **remerge** post a comment with the trigger to launch + this job manually. Do not include any other text or vote in the + same comment. :Required Parameters: @@ -374,6 +442,9 @@ pyenv variables before running. :branch: The branch to build against. (default: master) :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) :build-timeout: Timeout in minutes before aborting build. (default: 15) + :cron: Cron schedule when to trigger the job. Supports daily builds. + This parameter also supports multiline input via YAML pipe | character in + cases where one may want to provide more than 1 cron timer. (default: empty) :disable-job: Whether to disable the job (default: false) :dist-binary: Whether to build a binary wheel distribution. (default: true) :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) @@ -381,6 +452,8 @@ pyenv variables before running. (default: false, in series) :pre-build-script: Shell script to execute before the tox builder. For example, install system prerequisites. (default: a shell comment) + :pypi-repo: Key for the PyPI target repository in the .pypirc file, + ideally a server like test.pypy.org. (default: pypi-test) :python-version: Python version to invoke pip install of tox-pyenv (default: python3) :stream: Keyword representing a release code-name. @@ -401,17 +474,13 @@ pyenv variables before running. https://docs.openstack.org/infra/jenkins-job-builder/triggers.html#triggers.gerrit -PyPI Merge ----------- +PyPI Verify +----------- -Creates and uploads distribution files on merge of a patch set. Runs -tox, builds a source distribution and (optionally) a binary -distribution, and uploads the distribution(s) to a PyPI repository. -This job should use a staging repository like testpypi.python.org, -which sets up use of release jobs to promote the distributions later. -This job can also use a public release area like the global PyPI -repository. The project git repository must have a setup.py file -with configuration for packaging the component. +Verifies a Python library project on creation of a patch set. Runs tox +then builds a source distribution and (optionally) a binary +distribution. The project repository must have a setup.py file with +configuration for packaging the component. The tox runner is pyenv aware so if the image contains an installation of pyenv at /opt/pyenv it will pick it up and run Python tests with @@ -423,35 +492,15 @@ pyenv variables before running. export PYENV_ROOT="/opt/pyenv" export PATH="$PYENV_ROOT/bin:$PATH" - -Requires a .pypirc configuration file in the Jenkins builder home directory, -an example appears next that uses API tokens. Note that in the [pypi] entry -the repository key-value pair is optional, it defaults to pypi.org. - -.. code-block:: bash - - [distutils] # this tells distutils what package indexes you can push to - index-servers = pypi-test pypi - - [pypi-test] - repository: https://test.pypi.org/legacy/ - username: __token__ - password: pypi-test-api-token-goes-here - - [pypi] - username: __token__ - password: pypi-api-token-goes-here - - :Template Names: - - {project-name}-pypi-merge-{stream} - - gerrit-pypi-merge - - github-pypi-merge + - {project-name}-pypi-verify-{stream} + - gerrit-pypi-verify + - github-pypi-verify -:Comment Trigger: **remerge** post a comment with the trigger to launch - this job manually. Do not include any other text or vote in the - same comment. +:Comment Trigger: **recheck|reverify** post a comment with one of the + triggers to launch this job manually. Do not include any other + text or vote in the same comment. :Required Parameters: @@ -466,9 +515,6 @@ the repository key-value pair is optional, it defaults to pypi.org. :branch: The branch to build against. (default: master) :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) :build-timeout: Timeout in minutes before aborting build. (default: 15) - :cron: Cron schedule when to trigger the job. Supports daily builds. - This parameter also supports multiline input via YAML pipe | character in - cases where one may want to provide more than 1 cron timer. (default: empty) :disable-job: Whether to disable the job (default: false) :dist-binary: Whether to build a binary wheel distribution. (default: true) :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) @@ -476,8 +522,6 @@ the repository key-value pair is optional, it defaults to pypi.org. (default: false, in series) :pre-build-script: Shell script to execute before the tox builder. For example, install system prerequisites. (default: a shell comment) - :pypi-repo: Key for the PyPI target repository in the .pypirc file, - ideally a server like test.pypy.org. (default: pypi-test) :python-version: Python version to invoke pip install of tox-pyenv (default: python3) :stream: Keyword representing a release code-name. diff --git a/jjb/lf-release-jobs.yaml b/jjb/lf-release-jobs.yaml index c8384824..07b988ff 100644 --- a/jjb/lf-release-jobs.yaml +++ b/jjb/lf-release-jobs.yaml @@ -342,6 +342,8 @@ variable: SIGUL_PKI - file-id: signing-pubkey variable: SIGNING_PUBKEY + - file-id: pypirc + target: "$HOME/.pypirc" - shell: !include-raw-escape: ../shell/sigul-configuration.sh - shell: !include-raw-escape: ../shell/sigul-install.sh - inject: diff --git a/releasenotes/notes/fix-pypi-config-release-d78e0dbe5e72d38d.yaml b/releasenotes/notes/fix-pypi-config-release-d78e0dbe5e72d38d.yaml new file mode 100644 index 00000000..1ebca776 --- /dev/null +++ b/releasenotes/notes/fix-pypi-config-release-d78e0dbe5e72d38d.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Add .pypirc config file provider to builders in release job. + Extend PyPI documentation with recommended directory layout. -- 2.16.6