From d02b15305d8e8ba15a8f2601bc7c207c06feb9d4 Mon Sep 17 00:00:00 2001 From: Anil Belur Date: Thu, 13 Nov 2025 23:35:16 +1000 Subject: [PATCH] Fix: Force SCP mode for file transfer with bastion Force SCP mode for Ansible file transfers with bastion. Add ANSIBLE_SCP_IF_SSH=True environment variable when local_build=true to force Ansible to use SCP instead of SFTP for file transfers. Root cause: Ansible defaults to SFTP for file transfers, which causes failures when connecting through bastion/jump hosts that don't have sftp-server installed. The existing --scp-extra-args '-O' flag was being ignored because Ansible was using SFTP, not SCP. Error without this fix: failed to transfer file to /home/ubuntu/.ansible/tmp/... bash: line 1: /usr/lib/sftp-server: No such file or directory scp: Connection closed Solution: Set ANSIBLE_SCP_IF_SSH=True to force Ansible to use SCP mode when local_build=true (builds through bastion). Once SCP is being used, the --scp-extra-args '-O' flag applies, forcing legacy SCP protocol which works correctly through the bastion. Changes: - Added local.ansible_env_vars conditional in all templates - When local_build=true: includes ANSIBLE_SCP_IF_SSH=True - When local_build=false: standard Ansible environment (Jenkins builds) - Updated provisioner blocks to use local.ansible_env_vars Templates updated: - builder.pkr.hcl - docker.pkr.hcl - devstack.pkr.hcl - devstack-pre-pip-yoga.pkr.hcl - windows-builder.pkr.hcl Backward compatibility: Existing Jenkins builds continue to work unchanged since local_build defaults to false. Only affects builds with local_build=true (packer-build-action through bastion). Change-Id: Iae0d6f2284fb2cdefa827c951bd11b51a5a56e19 Signed-off-by: Anil Belur --- ...-ansible-scp-sftp-bastion-391c386d9e4f8a21.yaml | 50 ++++++++++++++++++++++ templates/builder.pkr.hcl | 29 +++++++++---- templates/devstack-pre-pip-yoga.pkr.hcl | 29 +++++++++---- templates/devstack.pkr.hcl | 29 +++++++++---- templates/docker.pkr.hcl | 29 +++++++++---- templates/windows-builder.pkr.hcl | 29 +++++++++---- 6 files changed, 150 insertions(+), 45 deletions(-) create mode 100644 releasenotes/notes/fix-ansible-scp-sftp-bastion-391c386d9e4f8a21.yaml diff --git a/releasenotes/notes/fix-ansible-scp-sftp-bastion-391c386d9e4f8a21.yaml b/releasenotes/notes/fix-ansible-scp-sftp-bastion-391c386d9e4f8a21.yaml new file mode 100644 index 0000000..dfd137b --- /dev/null +++ b/releasenotes/notes/fix-ansible-scp-sftp-bastion-391c386d9e4f8a21.yaml @@ -0,0 +1,50 @@ +--- +fixes: + - | + Fix Ansible file transfer failures when building through bastion/jump hosts. + + **Problem**: Packer builds with ``local_build=true`` (bastion/jump host mode) + were failing during Ansible provisioning with SFTP errors: + + .. code-block:: text + + failed to transfer file to /home/ubuntu/.ansible/tmp/... + bash: line 1: /usr/lib/sftp-server: No such file or directory + scp: Connection closed + + **Root Cause**: Ansible defaults to SFTP for file transfers, not SCP. + The existing ``--scp-extra-args '-O'`` flag was being ignored because + Ansible was using SFTP, not SCP. Many cloud instances and bastion hosts + don't have ``sftp-server`` installed, causing transfer failures. + + **Resolution**: Added ``ANSIBLE_SCP_IF_SSH=True`` environment variable + when ``local_build=true`` to force Ansible to use SCP mode instead of + SFTP. This makes the ``--scp-extra-args '-O'`` flag take effect, which + forces the legacy SCP protocol that works through bastion hosts. + + **Implementation**: Created conditional ``ansible_env_vars`` in the + locals block of all templates: + + - When ``local_build=true``: Sets ``ANSIBLE_SCP_IF_SSH=True`` + - When ``local_build=false``: Standard Ansible environment (no change) + + **Backward Compatibility**: This change only affects builds with + ``local_build=true`` (packer-build-action through bastion). Existing + Jenkins builds continue to work unchanged since ``local_build`` + defaults to false. + + Templates updated: + + - templates/builder.pkr.hcl + - templates/docker.pkr.hcl + - templates/devstack.pkr.hcl + - templates/devstack-pre-pip-yoga.pkr.hcl + - templates/windows-builder.pkr.hcl + + **Usage**: When building through a bastion/jump host, set: + + .. code-block:: bash + + packer build -var local_build=true ... + + Reference: https://docs.ansible.com/ansible/latest/collections/ansible/builtin/ssh_connection.html diff --git a/templates/builder.pkr.hcl b/templates/builder.pkr.hcl index a92b05c..17e33e7 100644 --- a/templates/builder.pkr.hcl +++ b/templates/builder.pkr.hcl @@ -158,7 +158,7 @@ variable "vm_volume_size" { } locals { - # SSH arguments for local builds only + # SSH arguments - SCP compatibility for local builds ssh_extra_args = var.local_build ? [ "--scp-extra-args", "'-O'", "--ssh-extra-args", @@ -166,6 +166,24 @@ locals { ] : [ "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa" ] + + # Ansible environment variables - force SCP for local builds to work with bastion + ansible_env_vars = var.local_build ? [ + "ANSIBLE_NOCOWS=1", + "ANSIBLE_PIPELINING=False", + "ANSIBLE_HOST_KEY_CHECKING=False", + "ANSIBLE_SCP_IF_SSH=True", + "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", + "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", + "ANSIBLE_STDOUT_CALLBACK=debug" + ] : [ + "ANSIBLE_NOCOWS=1", + "ANSIBLE_PIPELINING=False", + "ANSIBLE_HOST_KEY_CHECKING=False", + "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", + "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", + "ANSIBLE_STDOUT_CALLBACK=debug" + ] } source "docker" "builder" { @@ -216,14 +234,7 @@ build { } provisioner "ansible" { - ansible_env_vars = [ - "ANSIBLE_NOCOWS=1", - "ANSIBLE_PIPELINING=False", - "ANSIBLE_HOST_KEY_CHECKING=False", - "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", - "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", - "ANSIBLE_STDOUT_CALLBACK=debug" - ] + ansible_env_vars = local.ansible_env_vars command = "./common-packer/ansible-playbook.sh" extra_arguments = local.ssh_extra_args playbook_file = "provision/local-builder.yaml" diff --git a/templates/devstack-pre-pip-yoga.pkr.hcl b/templates/devstack-pre-pip-yoga.pkr.hcl index c3f72d0..d1d7dbd 100644 --- a/templates/devstack-pre-pip-yoga.pkr.hcl +++ b/templates/devstack-pre-pip-yoga.pkr.hcl @@ -161,7 +161,7 @@ variable "vm_volume_size" { } locals { - # SSH arguments for local builds only + # SSH arguments - SCP compatibility for local builds ssh_extra_args = var.local_build ? [ "--extra-vars", "os_branch=stable/yoga rdo_branch=yoga", "--scp-extra-args", "'-O'", @@ -171,6 +171,24 @@ locals { "--extra-vars", "os_branch=stable/yoga rdo_branch=yoga", "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa" ] + + # Ansible environment variables - force SCP for local builds to work with bastion + ansible_env_vars = var.local_build ? [ + "ANSIBLE_NOCOWS=1", + "ANSIBLE_PIPELINING=False", + "ANSIBLE_HOST_KEY_CHECKING=False", + "ANSIBLE_SCP_IF_SSH=True", + "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", + "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", + "ANSIBLE_STDOUT_CALLBACK=debug" + ] : [ + "ANSIBLE_NOCOWS=1", + "ANSIBLE_PIPELINING=False", + "ANSIBLE_HOST_KEY_CHECKING=False", + "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", + "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", + "ANSIBLE_STDOUT_CALLBACK=debug" + ] } source "docker" "devstack-pre-pip-yoga" { @@ -221,14 +239,7 @@ build { } provisioner "ansible" { - ansible_env_vars = [ - "ANSIBLE_NOCOWS=1", - "ANSIBLE_PIPELINING=False", - "ANSIBLE_HOST_KEY_CHECKING=False", - "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", - "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", - "ANSIBLE_STDOUT_CALLBACK=debug" - ] + ansible_env_vars = local.ansible_env_vars command = "./common-packer/ansible-playbook.sh" extra_arguments = local.ssh_extra_args playbook_file = "provision/devstack-pre-pip-centos.yaml" diff --git a/templates/devstack.pkr.hcl b/templates/devstack.pkr.hcl index 0489ccc..9d4a2d4 100644 --- a/templates/devstack.pkr.hcl +++ b/templates/devstack.pkr.hcl @@ -163,7 +163,7 @@ variable "vm_volume_size" { } locals { - # SSH arguments for local builds only + # SSH arguments - SCP compatibility for local builds ssh_extra_args = var.local_build ? [ "--extra-vars", "os_branch=stable/yoga rdo_branch=yoga", "--scp-extra-args", "'-O'", @@ -173,6 +173,24 @@ locals { "--extra-vars", "os_branch=stable/yoga rdo_branch=yoga", "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa" ] + + # Ansible environment variables - force SCP for local builds to work with bastion + ansible_env_vars = var.local_build ? [ + "ANSIBLE_NOCOWS=1", + "ANSIBLE_PIPELINING=False", + "ANSIBLE_HOST_KEY_CHECKING=False", + "ANSIBLE_SCP_IF_SSH=True", + "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", + "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", + "ANSIBLE_STDOUT_CALLBACK=debug" + ] : [ + "ANSIBLE_NOCOWS=1", + "ANSIBLE_PIPELINING=False", + "ANSIBLE_HOST_KEY_CHECKING=False", + "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", + "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", + "ANSIBLE_STDOUT_CALLBACK=debug" + ] } source "docker" "devstack" { @@ -223,14 +241,7 @@ build { } provisioner "ansible" { - ansible_env_vars = [ - "ANSIBLE_NOCOWS=1", - "ANSIBLE_PIPELINING=False", - "ANSIBLE_HOST_KEY_CHECKING=False", - "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", - "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", - "ANSIBLE_STDOUT_CALLBACK=debug" - ] + ansible_env_vars = local.ansible_env_vars command = "./common-packer/ansible-playbook.sh" extra_arguments = local.ssh_extra_args playbook_file = "provision/devstack-centos.yaml" diff --git a/templates/docker.pkr.hcl b/templates/docker.pkr.hcl index 359f329..0b9f745 100644 --- a/templates/docker.pkr.hcl +++ b/templates/docker.pkr.hcl @@ -164,7 +164,7 @@ variable "vm_volume_size" { } locals { - # SSH arguments for local builds only + # SSH arguments - SCP compatibility for local builds ssh_extra_args = var.local_build ? [ "--scp-extra-args", "'-O'", "--ssh-extra-args", @@ -172,6 +172,24 @@ locals { ] : [ "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa" ] + + # Ansible environment variables - force SCP for local builds to work with bastion + ansible_env_vars = var.local_build ? [ + "ANSIBLE_NOCOWS=1", + "ANSIBLE_PIPELINING=False", + "ANSIBLE_HOST_KEY_CHECKING=False", + "ANSIBLE_SCP_IF_SSH=True", + "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", + "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", + "ANSIBLE_STDOUT_CALLBACK=debug" + ] : [ + "ANSIBLE_NOCOWS=1", + "ANSIBLE_PIPELINING=False", + "ANSIBLE_HOST_KEY_CHECKING=False", + "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", + "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", + "ANSIBLE_STDOUT_CALLBACK=debug" + ] } source "docker" "docker" { @@ -222,14 +240,7 @@ build { } provisioner "ansible" { - ansible_env_vars = [ - "ANSIBLE_NOCOWS=1", - "ANSIBLE_PIPELINING=False", - "ANSIBLE_HOST_KEY_CHECKING=False", - "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", - "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", - "ANSIBLE_STDOUT_CALLBACK=debug" - ] + ansible_env_vars = local.ansible_env_vars command = "./common-packer/ansible-playbook.sh" extra_arguments = local.ssh_extra_args playbook_file = "provision/local-docker.yaml" diff --git a/templates/windows-builder.pkr.hcl b/templates/windows-builder.pkr.hcl index 5f63d0e..9a7d99f 100644 --- a/templates/windows-builder.pkr.hcl +++ b/templates/windows-builder.pkr.hcl @@ -160,7 +160,7 @@ variable "vm_volume_size" { } locals { - # SSH arguments for local builds only + # SSH arguments - SCP compatibility for local builds ssh_extra_args = var.local_build ? [ "--extra-vars", "ansible_shell_type=powershell", "--extra-vars", "ansible_shell_executable=None", @@ -172,6 +172,24 @@ locals { "--extra-vars", "ansible_shell_executable=None", "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa" ] + + # Ansible environment variables - force SCP for local builds to work with bastion + ansible_env_vars = var.local_build ? [ + "ANSIBLE_NOCOWS=1", + "ANSIBLE_PIPELINING=False", + "ANSIBLE_HOST_KEY_CHECKING=False", + "ANSIBLE_SCP_IF_SSH=True", + "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", + "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", + "ANSIBLE_STDOUT_CALLBACK=debug" + ] : [ + "ANSIBLE_NOCOWS=1", + "ANSIBLE_PIPELINING=False", + "ANSIBLE_HOST_KEY_CHECKING=False", + "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", + "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", + "ANSIBLE_STDOUT_CALLBACK=debug" + ] } source "openstack" "windows-builder" { @@ -209,14 +227,7 @@ build { } provisioner "ansible" { - ansible_env_vars = [ - "ANSIBLE_NOCOWS=1", - "ANSIBLE_PIPELINING=False", - "ANSIBLE_HOST_KEY_CHECKING=False", - "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", - "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", - "ANSIBLE_STDOUT_CALLBACK=debug" - ] + ansible_env_vars = local.ansible_env_vars command = "./common-packer/ansible-playbook.sh" extra_arguments = local.ssh_extra_args playbook_file = "provision/local-windows-builder.yaml" -- 2.16.6