Differences

This shows you the differences between two versions of the page.

--- devplayground [2025/06/10 15:27]
pdonnell [Configuring CephFS Mounts]
+++ devplayground [2026/02/03 22:40] (current)
djgalloway
@@ Line 52: / Line 52: @@
 ==== Configuring CephFS Mounts ====
-The following script can be run to setup CephFS mounts on a new developer playground machines:
+The following ansible playbook can be run to setup CephFS mounts on a new developer playground machines:
 <code>
-#!/bin/bash
+---
+- name: Configure Ceph Client and Mounts
+  hosts: all
+  become: true
+  vars:
+    admin_node: "doli01.front.sepia.ceph.com"
+    ceph_conf_path: "/etc/ceph/ceph.conf"
+    keyring_path: "/etc/ceph/client.sepian.keyring"
+    client_keyring_path: "/etc/ceph/keyring"
+    secret_path: "/etc/ceph/client.sepian.secret"
+    mounts:
+      - { path: "/teuthology", fstype: "ceph", src: "/teuthology-archive", mds_namespace: "teuthology", opts: "_netdev,ro" }
+      - { path: "/scratch", fstype: "ceph", src: "/", mds_namespace: "scratch", opts: "_netdev" }
-HOST="$1"
+  tasks:
+    - name: "1. Gather Ceph configuration using raw commands"
+      delegate_to: "{{ admin_node }}"
+      block:
+        - name: "▶️ Get LRC fsid"
+          ansible.builtin.raw: >
+            env CEPH_KEYRING={{ keyring_path }} ceph --id sepian fsid
+          register: ceph_fsid
+          changed_when: false
-function run {
+        - name: "▶️ Generate minimal ceph.conf (raw)"
-  printf '%s\n' "$*" >&2
+          ansible.builtin.raw: >
-  "$@"
+            env CEPH_KEYRING={{ keyring_path }} ceph --id sepian config generate-minimal-conf
-}
+          register: ceph_conf_content
+          changed_when: false
-function mssh {
+        - name: "▶️ Fetch Ceph keyring (raw)"
-  run ssh "$HOST" -- "$*"
+          ansible.builtin.raw: >
-}
+            cat {{ keyring_path }}
+          register: keyring_content
+          changed_when: false
-ADMIN_NODE=reesi003.front.sepia.ceph.com
+        - name: "▶️ Generate client secret (raw)"
+          ansible.builtin.raw: >
+            ceph-authtool {{ keyring_path }} -n client.sepian -p
+          register: secret_content
+          changed_when: false
-mssh sudo mkdir -p -m 755 /etc/ceph
+        - name: "▶️ Get Ceph monitor list (raw)"
-ssh "$ADMIN_NODE" 'env CEPH_KEYRING=/etc/ceph/client.sepian.keyring ceph --id sepian config generate-minimal-conf' | mssh sudo tee /etc/ceph/ceph.conf
+          ansible.builtin.raw: >
-mssh sudo chmod 644 /etc/ceph/ceph.conf
+            env CEPH_KEYRING={{ keyring_path }} ceph --id sepian mon dump --format json 2>/dev/null | jq -r '[.mons[] | .public_addrs.addrvec[] | select(.type=="v1").addr] | join(",")'
-ssh "$ADMIN_NODE" 'cat /etc/ceph/client.sepian.keyring' | mssh sudo tee /etc/ceph/client.sepian.keyring
+          register: mon_hosts
-mssh sudo chmod 644 /etc/ceph/client.sepian.keyring
+          changed_when: false
-ssh "$ADMIN_NODE" 'ceph-authtool /etc/ceph/client.sepian.keyring -n client.sepian -p' | mssh sudo tee /etc/ceph/client.sepian.secret
-mssh sudo chmod 600 /etc/ceph/client.sepian.secret
-MONS=$(ssh "$ADMIN_NODE" 'env CEPH_KEYRING=/etc/ceph/client.sepian.keyring ceph --id sepian mon dump --format json | jq -r '\''[.mons[] | .public_addrs.addrvec[] | select(.type=="v1").addr] | join(",")'\')
+    - name: "2. Configure Ceph client files"
+      block:
+        # -----------------------------
+        # CentOS
+        # -----------------------------
+        - name: "▶️ Install Ceph Squid repo on CentOS"
+          ansible.builtin.dnf:
+            name: centos-release-ceph-squid.noarch
+            state: present
+          when: ansible_facts['distribution'] == "CentOS"
+        - name: "▶️ Install ceph-common on CentOS"
+          ansible.builtin.dnf:
+            name: ceph-common
+            state: present
+          when: ansible_facts['distribution'] == "CentOS"
+        # -----------------------------
+        # Fedora + RHEL (upstream Squid repo)
+        # -----------------------------
+        - name: "▶️ Set Ceph EL major version (Fedora → el9)"
+          ansible.builtin.set_fact:
+            ceph_el_major: >-
+              {{ '9' if ansible_facts['distribution'] == 'Fedora'
+                 else ansible_facts['distribution_major_version'] }}
+          when: ansible_facts['distribution'] in ["RedHat", "Fedora"]
+        - name: "▶️ Install Ceph Squid upstream repo RPM (via URL)"
+          ansible.builtin.command: >-
+            dnf -y install
+            https://download.ceph.com/rpm-squid/el{{ ceph_el_major }}/noarch/ceph-release-1-1.el{{ ceph_el_major }}.noarch.rpm
+          args:
+            creates: /etc/yum.repos.d/ceph.repo
+          when: ansible_facts['distribution'] in ["RedHat", "Fedora"]
+        - name: "▶️ Install ceph-common on Fedora / RHEL"
+          ansible.builtin.dnf:
+            name: ceph-common
+            state: present
+          when: ansible_facts['distribution'] in ["RedHat", "Fedora"]
-function genmount {
+        - name: "▶️ Ensure /etc/ceph directory exists"
-  local secret=$(mssh sudo cat /etc/ceph/client.sepian.secret)
+          ansible.builtin.file:
-  mssh sudo chattr -i -- "$2"
+            path: "/etc/ceph"
-  # create mountpoint
+            state: directory
-  mssh sudo mkdir -p -- "$2"
+            mode: '0755'
-  # make the mountpoint directory (shadowed) unwriteable to prevent accidental modification
-  mssh sudo chmod 000 -- "$2"
-  # set it immutable to enforce that even for root
-  mssh sudo chattr +i -- "$2"
-  mssh sudo sed -i "/mds_namespace=$3/d" /etc/fstab
-  printf '%s:%s\t%s\tceph\tname=sepian,secret=%s,mds_namespace=%s,_netdev\t0\t2\n' "$MONS" "$1" "$2" "$secret" "$3" | mssh sudo tee -a /etc/fstab
-}
-mssh sudo umount /teuthology
+        - name: "▶️ Deploy ceph.conf"
-mssh sudo umount /scratch
+          ansible.builtin.copy:
-mssh sudo umount /postfile
+            content: "{{ ceph_conf_content.stdout }}"
+            dest: "{{ ceph_conf_path }}"
+            mode: '0644'
-genmount /teuthology-archive /teuthology teuthology
+        - name: "▶️ Create temporary file for keyring import"
-genmount / /scratch scratch
+          ansible.builtin.tempfile:
-genmount / /postfile postfile
+            state: file
+            suffix: .keyring
+          register: tmp_keyring
-mssh sudo systemctl daemon-reload
+        - name: "▶️ Write keyring content to temporary file"
-mssh sudo mount /teuthology
+          ansible.builtin.copy:
-mssh sudo mount /scratch
+            content: "{{ keyring_content.stdout }}"
-mssh sudo mount /postfile
+            dest: "{{ tmp_keyring.path }}"
-mssh sudo ln -s /teuthology /a
+            mode: '0600'
+        - name: "▶️ Deploy client keyring"
+          ansible.builtin.raw: >
+            ceph-authtool {{ client_keyring_path }} --create-keyring --import-keyring {{ tmp_keyring.path }}
+        - name: "▶️ Clean up temporary keyring file"
+          ansible.builtin.file:
+            path: "{{ tmp_keyring.path }}"
+            state: absent
+        - name: "▶️ Deploy client secret file (for other tools)"
+          ansible.builtin.copy:
+            content: "{{ secret_content.stdout }}"
+            dest: "{{ secret_path }}"
+            mode: '0600'
+    - name: "3. Set up CephFS mounts"
+      block:
+        - name: "▶️ Unmount filesystems if they currently exist"
+          ansible.posix.mount:
+            path: "{{ item.path }}"
+            state: unmounted
+          loop: "{{ mounts }}"
+        - name: "▶️ Create mount point directories"
+          ansible.builtin.file:
+            path: "{{ item.path }}"
+            state: directory
+            mode: '000'
+          loop: "{{ mounts }}"
+        - name: "▶️ Set immutable attribute on mount points"
+          ansible.builtin.file:
+            path: "{{ item.path }}"
+            attr: +i
+          loop: "{{ mounts }}"
+        - name: "▶️ Configure CephFS mounts in /etc/fstab"
+          ansible.posix.mount:
+            path: "{{ item.path }}"
+            src: "sepian@{{ ceph_fsid.stdout | trim }}.{{ item.mds_namespace }}={{ item.src }}"
+            fstype: "{{ item.fstype }}"
+            opts: "{{ item.opts }}"
+            state: mounted
+            dump: 0
+            passno: 0
+          loop: "{{ mounts }}"
+          notify: Reload Systemd
+        - name: "▶️ Create symlink for /a -> /teuthology"
+          ansible.builtin.file:
+            src: "/teuthology"
+            dest: "/a"
+            state: link
+            force: true
+    - name: "Force handlers to run before mounting"
+      ansible.builtin.meta: flush_handlers
+  handlers:
+    - name: Reload Systemd
+      listen: Reload Systemd
+      ansible.builtin.systemd:
+        daemon_reload: true
 </code>
 ==== Teuthology scheduling ====

Sepia Lab Wiki

User Tools

Site Tools

Differences

Page Tools