Merge pull request #241 from stackhpc/yoga-leafcloud-ci

yoga: Backport DNF and Apt auth support

ansible/group_vars/all/apt

@@ -45,3 +45,12 @@ apt_repositories: []
 # when replacing the distribution repositories via apt_repositories.
 # Default is false.
 apt_disable_sources_list: false
+
+# List of Apt auth configurations. Each item is a dict with the following keys:
+# * machine: 'machine' entry in the auth file
+# * login: 'login' entry in the auth file
+# * password: 'password' entry in the auth file
+# * filename: Name of a file in /etc/apt/auth.conf.d in which to store
+#   the auth configuration. The extension should be ``.conf``.
+# Default is an empty list.
+apt_auth: []

ansible/roles/apt/defaults/main.yml

@@ -48,3 +48,12 @@ apt_repositories: []
 # when replacing the distribution repositories via apt_repositories.
 # Default is false.
 apt_disable_sources_list: false
+
+# List of Apt auth configurations. Each item is a dict with the following keys:
+# * machine: 'machine' entry in the auth file
+# * login: 'login' entry in the auth file
+# * password: 'password' entry in the auth file
+# * filename: Name of a file in /etc/apt/auth.conf.d in which to store
+#   the auth configuration. The extension should be ``.conf``.
+# Default is an empty list.
+apt_auth: []

ansible/roles/apt/files/auth_schema.json

@@ -0,0 +1,28 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "description": "List of Apt auth configurations",
+    "type": "array",
+    "items": {
+        "description": "Apt auth configuration",
+        "type": "object",
+        "required": ["machine", "login", "password", "filename"],
+        "properties": {
+            "machine": {
+                "type": "string",
+		"minLength": 1
+            },
+            "login": {
+                "type": "string",
+		"minLength": 1
+            },
+            "password": {
+                "type": "string",
+		"minLength": 1
+            },
+            "filename": {
+                "type": "string",
+		"minLength": 1
+            }
+        }
+    }
+}

ansible/roles/apt/tasks/auth.yml

@@ -0,0 +1,32 @@
+---
+- name: Validate Apt auth config
+  ansible.utils.validate:
+    criteria: "{{ lookup('ansible.builtin.file', 'auth_schema.json') }}"
+    data: "{{ apt_auth }}"
+
+- name: Ensure the Apt auth.conf.d directory exists
+  ansible.builtin.file:
+    path: "/etc/apt/auth.conf.d"
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+  become: true
+
+- name: Configure Apt auth files
+  ansible.builtin.template:
+    src: "auth.conf.j2"
+    dest: "/etc/apt/auth.conf.d/{{ auth.filename }}"
+    owner: root
+    group: root
+    mode: 0600
+  become: true
+  # apt_auth contains sensitive data, so iterate over indices to avoid exposing
+  # them in Ansible output.
+  loop: "{{ apt_auth | map(attribute='filename') }}"
+  loop_control:
+    index_var: auth_index
+  vars:
+    auth: "{{ apt_auth[auth_index] }}"
+  notify:
+    - Update apt cache

ansible/roles/apt/tasks/main.yml

@@ -6,3 +6,5 @@
 - import_tasks: keys.yml
 
 - import_tasks: repos.yml
+
+- import_tasks: auth.yml

ansible/roles/apt/templates/auth.conf.j2

@@ -0,0 +1,5 @@
+# {{ ansible_managed }}
+
+machine {{ auth.machine }}
+login {{ auth.login }}
+password {{ auth.password }}

ansible/roles/dnf/tasks/custom-repo.yml

@@ -16,12 +16,14 @@
     metalink: "{{ item.value.metalink | default(omit)}}"
     mirrorlist: "{{ item.value.mirrorlist | default(omit)}}"
     mirrorlist_expire: "{{ item.value.mirrorlist_expire | default(omit)}}"
+    password: "{{ item.value.password | default(omit) }}"
     priority: "{{ item.value.priority | default(omit)}}"
     proxy: "{{ item.value.proxy | default(omit)}}"
     proxy_password: "{{ item.value.proxy_password | default(omit)}}"
     proxy_username: "{{ item.value.proxy_username | default(omit)}}"
     repo_gpgcheck: "{{ item.value.repo_gpgcheck | default(omit)}}"
     sslverify: "{{ item.value.sslverify | default(omit)}}"
+    username: "{{ item.value.username | default(omit) }}"
     state: "{{ item.value.state | default(omit)}}"
   with_dict: "{{ dnf_custom_repos }}"
   register: register_dnf_command

doc/source/configuration/reference/hosts.rst

@@ -442,6 +442,39 @@ that is signed by the key.
        components: all
        signed_by: example-key.asc
 
+Apt auth configuration
+----------------------
+
+Some repositories may require authentication using HTTP basic auth. Apt
+supports specifying credentials in URLs in ``sources.list`` files, but these
+files must be world-readable. A more secure setup involves writing credentials
+to `auth.conf
+<https://manpages.ubuntu.com/manpages/jammy/man5/apt_auth.conf.5.html>`__
+files which can have more restrictive permissions.
+
+Auth configuration is defined by the ``apt_auth`` variable. The format is a
+list, with each item mapping to a dict/map with the following items:
+
+* ``machine``: ``machine`` entry in the auth file
+* ``login``: ``machine`` entry in the auth file
+* ``password``: ``machine`` entry in the auth file
+* ``filename``: Name of a file in ``/etc/apt/auth.conf.d`` in which to store
+  the auth configuration. The extension should be ``.conf``.
+
+The default value of ``apt_auth`` is an empty list.
+
+In the following example, credentials are provided for package repositories at
+apt.example.com.
+
+.. code-block:: yaml
+   :caption: ``apt.yml``
+
+   apt_auth:
+     - machine: apt.example.com
+       login: my-username
+       password: my-password
+       filename: example.conf
+
 Development tools
 =================
 *tags:*

etc/kayobe/apt.yml

@@ -46,6 +46,17 @@
 # Default is false.
 #apt_disable_sources_list:
 
+# List of Apt auth configurations. Each item is a dict with the following keys:
+# * machine: 'machine' entry in the auth file
+# * login: 'login' entry in the auth file
+# * password: 'password' entry in the auth file
+# * filename: Name of a file in which to store the auth configuration. The
+#   extension should be '.conf'.
+# * filename: Name of a file in /etc/apt/auth.conf.d in which to store
+#   the auth configuration. The extension should be ``.conf``.
+# Default is an empty list.
+#apt_auth:
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes

playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2

@@ -149,6 +149,11 @@ apt_repositories:
     components: contrib
     signed_by: td-agent.asc
 apt_disable_sources_list: true
+apt_auth:
+  - machine: https://apt.example.com
+    login: foo
+    password: bar
+    filename: test.conf
 {% endif %}
 
 {% if ansible_facts.os_family == 'RedHat' %}

playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py

@@ -219,6 +219,16 @@ def test_apt_custom_package_repository_is_available(host):
     assert host.package("td-agent").is_installed
 
 
+@pytest.mark.skipif(not _is_apt(), reason="Apt only supported on Ubuntu")
+def test_apt_auth(host):
+    apt_auth = host.file("/etc/apt/apt.auth.d/test.conf")
+    assert apt_auth.exists
+    auth_lines = apt_auth.content_string.split()
+    assert "machine https://apt.example.com" in auth_lines
+    assert "login foo" in auth_lines
+    assert "password bar" in auth_lines
+
+
 @pytest.mark.parametrize('repo', ["appstream", "baseos", "extras", "epel"])
 @pytest.mark.skipif(not _is_dnf_mirror(), reason="DNF OpenDev mirror only for CentOS 8")
 def test_dnf_local_package_mirrors(host, repo):

releasenotes/notes/apt-auth-97d0291600836dec.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Adds support for auth configuration for Apt respositories and proxies using
+    ``auth.conf`` files.

releasenotes/notes/dnf-credentials-afc3bf7940cb6e99.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Adds support for specifying credentials (username and password) for custom
+    DNF repositories.

requirements.txt

@@ -9,3 +9,4 @@ selinux # MIT
 # INI parsing
 oslo.config>=5.2.0 # Apache-2.0
 paramiko # LGPL
+jsonschema<5 # MIT