From: DW Talton Date: Fri, 28 Feb 2020 20:30:59 +0000 (-0700) Subject: Replace python test harness X-Git-Tag: v0.31.2~2 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F32%2F63232%2F11;p=releng%2Flftools.git Replace python test harness This commit is a comprehensive overhaul of the test harness used in lftools. First, it removes coala. Black is added as a defacto code formatter to ensure consistency across all commits. Black is added as a pre-commit hook. Flake8 is added in conjunction with black, and configured according to Black's specification. Issue-ID: RELENG-2766 Signed-off-by: DW Talton Change-Id: Ib688b7cf9af8a22b7d067d3417a8e03f35f0bd64 --- diff --git a/.coafile b/.coafile deleted file mode 100644 index 6c057d37..00000000 --- a/.coafile +++ /dev/null @@ -1,59 +0,0 @@ -[all] -ignore = .tox/**, - .git/**, - .gitignore, - .gitreview, - .gitmodules, - .pytest_cache/**, - node_modules/**, - docs/conf.py, - lftools/cli/nexus2/**, - lftools/cli/nexus3/**, - .venv/** - -[all.Documentation] -bears = WriteGoodLintBear -files = docs/**/*.rst -ignore += docs/commands/nexus.rst - -[all.MarkDown] -bears = MarkdownBear,SpaceConsistencyBear,WriteGoodLintBear -files = **.md, **.markdown -use_spaces = true - -[all.Python] -bears = BanditBear, - PEP8Bear, - PyCommentedCodeBear, - PyDocStyleBear, - PyFlakesBear, - PyImportSortBear -files = lftools/**.py -known_first_party_imports = lftools -known_third_party_imports = - defusedxml, - email_validator, - httplib2, - jenkins, - jsonschema, - oauth2client, - pytest, - ruamel.yaml, - six, - shade, - xdg, - yaml, - pygerrit2, - pygithub, - bs4 - -pydocstyle_ignore = D203, D213, D301 -max_line_length = 120 - -[all.Shell] -bears = ShellCheckBear, - SpaceConsistencyBear -files = **.sh, shell/* -shell = bash -indent_size = 4 -use_spaces = yeah diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 43bd6f95..2accbe2c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,10 @@ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v1.3.0 + rev: v1.2.3 hooks: - id: trailing-whitespace - - - repo: https://github.com/jorisroovers/gitlint - rev: v0.12.0 + - repo: https://github.com/ambv/black + rev: stable hooks: - - id: gitlint + - id: black diff --git a/lftools/deploy.py b/lftools/deploy.py index 9db41fae..70b91022 100755 --- a/lftools/deploy.py +++ b/lftools/deploy.py @@ -588,7 +588,7 @@ def upload_maven_file_to_nexus( raise requests.HTTPError("Nexus Error: {}".format(error_msg)) -def deploy_nexus(nexus_repo_url, deploy_dir, snapshot=False): +def deploy_nexus(nexus_repo_url, deploy_dir, snapshot=False, workers=2): """Deploy a local directory of files to a Nexus repository. One purpose of this is so that we can get around the problematic @@ -649,13 +649,9 @@ def deploy_nexus(nexus_repo_url, deploy_dir, snapshot=False): file_list.append(file) - # Multithreaded upload - # default threads to CPU count / 2 so we're a nice neighbor to other builds - log.info("#######################################################") log.info("Deploying directory {} to {}".format(deploy_dir, nexus_repo_url)) - workers = int(cpu_count() / 2) with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: # this creates a dict where the key is the Future object, and the value is the file name # see concurrent.futures.Future for more info diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..2785c338 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,9 @@ +[tool.black] +line-length = 120 +target-version = ['py36'] +exclude = ''' +/( + lftools + | \.venv +)/ +''' diff --git a/requirements-test.txt b/requirements-test.txt index ce946138..d3967715 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,6 @@ -r requirements.txt -pytest -pytest-click -pytest-datafiles -pytest-mock -pytest-responses +pytest==5.3.5 +pytest-click==0.3 +pytest-datafiles==2.0 +pytest-mock==2.0.0 +pytest-responses==0.4.0 diff --git a/requirements.txt b/requirements.txt index 757d7835..8617a306 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,48 @@ -bs4 -click -defusedxml # Needed due to tox complains on parseString not safe -docker -email_validator -httplib2 -jsonschema -lxml -oauth2client -pygerrit2 -pygithub -python-jenkins -pyyaml -requests -ruamel.yaml -setuptools>=36.5.0 -six -tabulate -tqdm +appdirs==1.4.3 +aspy.yaml==1.3.0 +attrs==19.3.0 +beautifulsoup4==4.8.2 +bs4==0.0.1 +certifi==2019.11.28 +cfgv==2.0.1 +chardet==3.0.4 +click==7.1 +defusedxml==0.6.0 +Deprecated==1.2.7 +distlib==0.3.0 +dnspython==1.16.0 +docker==4.2.0 +email-validator==1.0.5 +filelock==3.0.12 +httplib2==0.17.0 +identify==1.4.11 +idna==2.9 +jsonschema==3.2.0 +lxml==4.5.0 +multi-key-dict==2.0.3 +nodeenv==1.3.5 +oauth2client==4.1.3 +pbr==5.4.4 +pre-commit==1.21.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pygerrit2==2.0.10 +PyGithub==1.46 +PyJWT==1.7.1 +pyrsistent==0.15.7 +python-jenkins==1.7.0 +PyYAML==5.3 +requests==2.23.0 +rsa==4.0 +ruamel.yaml==0.16.10 +ruamel.yaml.clib==0.2.0 +six==1.14.0 +soupsieve==2.0 +tabulate==0.8.6 +toml==0.10.0 +tqdm==4.43.0 +urllib3==1.25.8 +virtualenv==20.0.9 +websocket-client==0.57.0 +wrapt==1.12.1 xdg==3.0.2 diff --git a/tox.ini b/tox.ini index fe8e6332..73c163f0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,48 +1,49 @@ [tox] -minversion = 2.0 envlist = - coala, docs, docs-linkcheck, license, pre-commit, - py35, py36, py37, py38 +minversion = 3.7 skip_missing_interpreters = true +ignore_basepython_conflict = true + +[flake8] +max-complexity = 22 +max-line-length = 120 +ignore = E203, W503, C901, E402, B011 + +[pep8] +max-line-length = 120 + +[pytest] +markers = datafiles +addopts = -p no:warnings [testenv] basepython = python3 deps = -r{toxinidir}/requirements-test.txt commands = pytest -[testenv:coala] -basepython = python3 -deps = - coala - coala-bears - nodeenv - numpy -commands = - nodeenv -p - npm install --global remark-cli remark-lint write-good - coala --non-interactive - [testenv:docs] +description = Build the documentation with sphinx basepython = python3 deps = -rrequirements-docs.txt extras = openstack -commands = - sphinx-build -W -b html -n -d {envtmpdir}/doctrees ./docs/ {toxinidir}/docs/_build/html +commands = sphinx-build -W -b html -n -d {envtmpdir}/doctrees ./docs/ {toxinidir}/docs/_build/html [testenv:docs-linkcheck] +description = Check the documentation links with sphinx basepython = python3 deps = -rrequirements-docs.txt extras = openstack commands = sphinx-build -W -b linkcheck -d {envtmpdir}/doctrees ./docs/ {toxinidir}/docs/_build/linkcheck [testenv:latest-upstream] +description = Latest upstream test. Used for what exactly? basepython = python3 deps = -r{toxinidir}/requirements-test.txt commands = @@ -50,21 +51,22 @@ commands = pytest [testenv:license] +description = Check all files for license header basepython = python3 -deps = - lftools +deps = lftools commands = lftools license check-dir lftools lftools license check-dir -r '.+' shell [testenv:pre-commit] +description = Precommit checks for black, gitlint, etc. basepython = python3 deps = pre-commit commands = pre-commit run --all-files - pre-commit run gitlint --hook-stage commit-msg --commit-msg-filename .git/COMMIT_EDITMSG [testenv:reno] +description = OpenStack Reno checks basepython = python3 deps = reno commands = reno {posargs:--help}