From: Matthew Watkins Date: Tue, 19 Sep 2023 14:06:57 +0000 (+0100) Subject: Chore: Bypass parallel fetch, build system updates X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=56101b58369c465ed6e90cce34002d92716ec2c4;p=releng%2Fpython-one-password.git Chore: Bypass parallel fetch, build system updates Optionally allow bypass of parallel credential fetch Rename no_tags -> hide_tags Fix cosmetic documentation error Change-Id: I4bd685c2489432e1d5794ed07194ab93d3bf524d Signed-off-by: Matthew Watkins --- diff --git a/README.rst b/README.rst index 1115e86..47d37b4 100644 --- a/README.rst +++ b/README.rst @@ -188,7 +188,7 @@ any changes:: It is a good idea to update the hooks to the latest version:: -``% pre-commit autoupdate`` + % pre-commit autoupdate Don't forget to tell your contributors to also install and use pre-commit. diff --git a/src/python_one_password/credentials.py b/src/python_one_password/credentials.py index 503a9d1..e8cf0ab 100755 --- a/src/python_one_password/credentials.py +++ b/src/python_one_password/credentials.py @@ -46,6 +46,13 @@ def fetch( show_default=False, help="Enable verbose debug output/logging", ), + no_parallel: bool = typer.Option( + False, + "--no-parallel", + "-np", + show_default=False, + help="Disables parallel operations (multiprocessing)", + ), include: Optional[List[str]] = typer.Option( ["All"], "--include", @@ -60,7 +67,7 @@ def fetch( envvar="OP_VAULT_EXC", help="Excludes the specified vault(s) from processing", ), - no_tags: bool = typer.Option( + hide_tags: bool = typer.Option( False, "--no-tags", "-n", help="Hide metadata tags in credential summary/output" ), ): @@ -69,9 +76,9 @@ def fetch( f.validate_import_data_opts(include, exclude) vaults_dictionary = f.populate_vault_json(include, exclude) f.show_vault_summary(vaults_dictionary) - f.import_credentials(vaults_dictionary) + f.import_credentials(vaults_dictionary, no_parallel) credential_data = f.caching.load_cache("credentials") - f.credential_summary(credential_data, no_tags, True) + f.credential_summary(credential_data, hide_tags, True) @app.command() @@ -100,14 +107,14 @@ def show( show_default=False, help="Enable verbose debug output/logging", ), - no_tags: bool = typer.Option( + hide_tags: bool = typer.Option( False, "--no-tags", "-n", help="Hide metadata tags in credential summary/output" ), ): """Show credentials in current/filtered working set""" f.startup_tasks(debug) credential_data = caching.load_cache("credentials") - f.credential_summary(credential_data, no_tags, False) + f.credential_summary(credential_data, hide_tags, False) @app.command() @@ -119,7 +126,7 @@ def refine( show_default=False, help="Enable verbose debug output/logging", ), - no_tags: bool = typer.Option( + hide_tags: bool = typer.Option( False, "--no-tags", "-n", help="Hide metadata tags in credential summary/output" ), match: Optional[List[str]] = typer.Option( @@ -146,7 +153,7 @@ def refine( f.startup_tasks(debug) f.validate_filter_items_opts(match, reject) credential_data = f.filter_credentials(match, reject, ignore_case) - f.credential_summary(credential_data, no_tags, True) + f.credential_summary(credential_data, hide_tags, True) if f.prompt("Update working credential set to selection?"): caching.save_cache(credential_data, "credentials") else: diff --git a/src/python_one_password/functions.py b/src/python_one_password/functions.py index f00d820..48dc2cc 100755 --- a/src/python_one_password/functions.py +++ b/src/python_one_password/functions.py @@ -217,7 +217,7 @@ def match_strings(ignore_case: bool, search_pattern: str, string: str) -> bool: def credential_summary( - credential_list: List[Dict], no_tags: bool, prompt_flag: bool + credential_list: List[Dict], hide_tags: bool, prompt_flag: bool ) -> None: """Displays a list/summary of the current working set of credentials""" @@ -233,7 +233,7 @@ def credential_summary( name = credential["title"] # Tag display might make for messy output and take up excessive screen space # We therefore provide an option to suppress them in summary/output - if no_tags: + if hide_tags: log.info("%s %s", cred_id, name) continue # Not all credentials have tags, catch the exception @@ -355,7 +355,7 @@ def get_credentials_mp(vault: str) -> tuple[str, List[str]]: return (vault, vault_credentials) -def import_credentials(vaults: Dict[str, str]) -> None: +def import_credentials(vaults: Dict[str, str], no_parallel: bool) -> None: """Given a dictionary of vaults, populates the credential database(s)""" log.info("Importing credential metadata from 1Password database...") @@ -364,11 +364,20 @@ def import_credentials(vaults: Dict[str, str]) -> None: # Use timers to profile performance timer = start_timer() - with multiprocessing.Pool() as pool: - # Call the function for each item in parallel - for vault, credentials in pool.map(get_credentials_mp, vaults): + if no_parallel: + # Fetches credentials in series with no multiprocessing + log.info("Parallel retrieval of credentials is DISABLED") + for vault in vaults: + null, credentials = get_credentials_mp(vault) vault_creds_dictionary = {vault: credentials} vault_credentials.append(vault_creds_dictionary) + else: + log.info("Parallel retrieval of credentials is ENABLED") + with multiprocessing.Pool() as pool: + # Call the function for each item in parallel + for vault, credentials in pool.map(get_credentials_mp, vaults): + vault_creds_dictionary = {vault: credentials} + vault_credentials.append(vault_creds_dictionary) stop_timer(timer)