Replace python test harness

This commit is a comprehensive overhaul of the test harness used
in lftools. First, it removes coala. Black is added as a defacto
code formatter to ensure consistency across all commits. Black is
added as a pre-commit hook. Flake8 is added in conjunction with
black, and configured according to Black's specification.

Issue-ID: RELENG-2766
Signed-off-by: DW Talton <dtalton@contractor.linuxfoundation.org>
Change-Id: Ib688b7cf9af8a22b7d067d3417a8e03f35f0bd64

diff --git a/.coafile b/.coafile
deleted file mode 100644 (file)
index 6c057d3..0000000
--- a/.coafile
+++ /dev/null
@@ -1,59 +0,0 @@
-[all]
-ignore = .tox/**,
-    .git/**,
-    .gitignore,
-    .gitreview,
-    .gitmodules,
-    .pytest_cache/**,
-    node_modules/**,
-    docs/conf.py,
-    lftools/cli/nexus2/**,
-    lftools/cli/nexus3/**,
-    .venv/**
-
-[all.Documentation]
-bears = WriteGoodLintBear
-files = docs/**/*.rst
-ignore += docs/commands/nexus.rst
-
-[all.MarkDown]
-bears = MarkdownBear,SpaceConsistencyBear,WriteGoodLintBear
-files = **.md, **.markdown
-use_spaces = true
-
-[all.Python]
-bears = BanditBear,
-    PEP8Bear,
-    PyCommentedCodeBear,
-    PyDocStyleBear,
-    PyFlakesBear,
-    PyImportSortBear
-files = lftools/**.py
-known_first_party_imports = lftools
-known_third_party_imports =
-    defusedxml,
-    email_validator,
-    httplib2,
-    jenkins,
-    jsonschema,
-    oauth2client,
-    pytest,
-    ruamel.yaml,
-    six,
-    shade,
-    xdg,
-    yaml,
-    pygerrit2,
-    pygithub,
-    bs4
-
-pydocstyle_ignore = D203, D213, D301
-max_line_length = 120
-
-[all.Shell]
-bears = ShellCheckBear,
-    SpaceConsistencyBear
-files = **.sh, shell/*
-shell = bash
-indent_size = 4
-use_spaces = yeah

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 43bd6f9..2accbe2 100644 (file)
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,11 +1,10 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v1.3.0
+    rev: v1.2.3
     hooks:
       - id: trailing-whitespace
-
-  - repo: https://github.com/jorisroovers/gitlint
-    rev: v0.12.0
+  - repo: https://github.com/ambv/black
+    rev: stable
     hooks:
-      - id: gitlint
+    - id: black

diff --git a/lftools/deploy.py b/lftools/deploy.py
index 9db41fa..70b9102 100755 (executable)
--- a/lftools/deploy.py
+++ b/lftools/deploy.py
@@ -588,7 +588,7 @@ def upload_maven_file_to_nexus(
         raise requests.HTTPError("Nexus Error: {}".format(error_msg))
 
 
-def deploy_nexus(nexus_repo_url, deploy_dir, snapshot=False):
+def deploy_nexus(nexus_repo_url, deploy_dir, snapshot=False, workers=2):
     """Deploy a local directory of files to a Nexus repository.
 
     One purpose of this is so that we can get around the problematic
@@ -649,13 +649,9 @@ def deploy_nexus(nexus_repo_url, deploy_dir, snapshot=False):
 
             file_list.append(file)
 
-    # Multithreaded upload
-    # default threads to CPU count / 2 so we're a nice neighbor to other builds
-
     log.info("#######################################################")
     log.info("Deploying directory {} to {}".format(deploy_dir, nexus_repo_url))
 
-    workers = int(cpu_count() / 2)
     with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
         # this creates a dict where the key is the Future object, and the value is the file name
         # see concurrent.futures.Future for more info

diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644 (file)
index 0000000..2785c33
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,9 @@
+[tool.black]
+line-length = 120
+target-version = ['py36']
+exclude = '''
+/(
+    lftools
+  | \.venv
+)/
+'''

diff --git a/requirements-test.txt b/requirements-test.txt
index ce94613..d396771 100644 (file)
--- a/requirements-test.txt
+++ b/requirements-test.txt
@@ -1,6 +1,6 @@
 -r requirements.txt
-pytest
-pytest-click
-pytest-datafiles
-pytest-mock
-pytest-responses
+pytest==5.3.5
+pytest-click==0.3
+pytest-datafiles==2.0
+pytest-mock==2.0.0
+pytest-responses==0.4.0

diff --git a/requirements.txt b/requirements.txt
index 757d783..8617a30 100644 (file)
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,20 +1,48 @@
-bs4
-click
-defusedxml # Needed due to tox complains on parseString not safe
-docker
-email_validator
-httplib2
-jsonschema
-lxml
-oauth2client
-pygerrit2
-pygithub
-python-jenkins
-pyyaml
-requests
-ruamel.yaml
-setuptools>=36.5.0
-six
-tabulate
-tqdm
+appdirs==1.4.3
+aspy.yaml==1.3.0
+attrs==19.3.0
+beautifulsoup4==4.8.2
+bs4==0.0.1
+certifi==2019.11.28
+cfgv==2.0.1
+chardet==3.0.4
+click==7.1
+defusedxml==0.6.0
+Deprecated==1.2.7
+distlib==0.3.0
+dnspython==1.16.0
+docker==4.2.0
+email-validator==1.0.5
+filelock==3.0.12
+httplib2==0.17.0
+identify==1.4.11
+idna==2.9
+jsonschema==3.2.0
+lxml==4.5.0
+multi-key-dict==2.0.3
+nodeenv==1.3.5
+oauth2client==4.1.3
+pbr==5.4.4
+pre-commit==1.21.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pygerrit2==2.0.10
+PyGithub==1.46
+PyJWT==1.7.1
+pyrsistent==0.15.7
+python-jenkins==1.7.0
+PyYAML==5.3
+requests==2.23.0
+rsa==4.0
+ruamel.yaml==0.16.10
+ruamel.yaml.clib==0.2.0
+six==1.14.0
+soupsieve==2.0
+tabulate==0.8.6
+toml==0.10.0
+tqdm==4.43.0
+urllib3==1.25.8
+virtualenv==20.0.9
+websocket-client==0.57.0
+wrapt==1.12.1
 xdg==3.0.2

diff --git a/tox.ini b/tox.ini
index fe8e633..73c163f 100644 (file)
--- a/tox.ini
+++ b/tox.ini
@@ -1,48 +1,49 @@
 [tox]
-minversion = 2.0
 envlist =
-    coala,
     docs,
     docs-linkcheck,
     license,
     pre-commit,
-    py35,
     py36,
     py37,
     py38
+minversion = 3.7
 skip_missing_interpreters = true
+ignore_basepython_conflict = true
+
+[flake8]
+max-complexity = 22
+max-line-length = 120
+ignore = E203, W503, C901, E402, B011
+
+[pep8]
+max-line-length = 120
+
+[pytest]
+markers = datafiles
+addopts = -p no:warnings
 
 [testenv]
 basepython = python3
 deps = -r{toxinidir}/requirements-test.txt
 commands = pytest
 
-[testenv:coala]
-basepython = python3
-deps =
-    coala
-    coala-bears
-    nodeenv
-    numpy
-commands =
-    nodeenv -p
-    npm install --global remark-cli remark-lint write-good
-    coala --non-interactive
-
 [testenv:docs]
+description = Build the documentation with sphinx
 basepython = python3
 deps = -rrequirements-docs.txt
 extras = openstack
-commands =
-    sphinx-build -W -b html -n -d {envtmpdir}/doctrees ./docs/ {toxinidir}/docs/_build/html
+commands = sphinx-build -W -b html -n -d {envtmpdir}/doctrees ./docs/ {toxinidir}/docs/_build/html
 
 [testenv:docs-linkcheck]
+description = Check the documentation links with sphinx
 basepython = python3
 deps = -rrequirements-docs.txt
 extras = openstack
 commands = sphinx-build -W -b linkcheck -d {envtmpdir}/doctrees ./docs/ {toxinidir}/docs/_build/linkcheck
 
 [testenv:latest-upstream]
+description = Latest upstream test. Used for what exactly?
 basepython = python3
 deps = -r{toxinidir}/requirements-test.txt
 commands =
@@ -50,21 +51,22 @@ commands =
     pytest
 
 [testenv:license]
+description = Check all files for license header
 basepython = python3
-deps =
-    lftools
+deps = lftools
 commands =
     lftools license check-dir lftools
     lftools license check-dir -r '.+' shell
 
 [testenv:pre-commit]
+description = Precommit checks for black, gitlint, etc.
 basepython = python3
 deps = pre-commit
 commands =
     pre-commit run --all-files
-    pre-commit run gitlint --hook-stage commit-msg --commit-msg-filename .git/COMMIT_EDITMSG
 
 [testenv:reno]
+description = OpenStack Reno checks
 basepython = python3
 deps = reno
 commands = reno {posargs:--help}