Differences

This shows you the differences between two versions of the page.

--- devplayground [2024/07/30 18:29]
dmick [Playing Nice]
+++ devplayground [2025/12/17 22:16] (current)
pdonnell [Configuring CephFS Mounts]
@@ Line 8: / Line 8: @@
 ^ Name                                             ^ Notes            ^
-| [[hardware:senta|senta01.front.sepia.ceph.com]]  | General purpose  |
+| [[hardware:senta|senta01.front.sepia.ceph.com]]  | General purpose (currently unavailable) |
 | [[hardware:senta|senta02.front.sepia.ceph.com]]  | General purpose  |
 | [[hardware:senta|senta03.front.sepia.ceph.com]]  | General purpose  |
@@ Line 52: / Line 52: @@
 ==== Configuring CephFS Mounts ====
-The following script can be run to setup CephFS mounts on a new developer playground machines:
+The following ansible playbook can be run to setup CephFS mounts on a new developer playground machines:
 <code>
-#!/bin/bash
+---
+- name: Configure Ceph Client and Mounts
+  hosts: all
+  become: true
+  vars:
+    admin_node: "doli01.front.sepia.ceph.com"
+    ceph_conf_path: "/etc/ceph/ceph.conf"
+    keyring_path: "/etc/ceph/client.sepian.keyring"
+    client_keyring_path: "/etc/ceph/keyring"
+    secret_path: "/etc/ceph/client.sepian.secret"
+    mounts:
+      - { path: "/teuthology", fstype: "ceph", src: "/teuthology-archive", mds_namespace: "teuthology", opts: "_netdev,ro" }
+      - { path: "/scratch", fstype: "ceph", src: "/", mds_namespace: "scratch", opts: "_netdev" }
-HOST="$1"
+  tasks:
+    - name: "1. Gather Ceph configuration using raw commands"
+      delegate_to: "{{ admin_node }}"
+      block:
+        - name: "▶️ Get LRC fsid"
+          ansible.builtin.raw: >
+            env CEPH_KEYRING={{ keyring_path }} ceph --id sepian fsid
+          register: ceph_fsid
+          changed_when: false
-function run {
+        - name: "▶️ Generate minimal ceph.conf (raw)"
-  printf '%s\n' "$*" >&2
+          ansible.builtin.raw: >
-  "$@"
+            env CEPH_KEYRING={{ keyring_path }} ceph --id sepian config generate-minimal-conf
-}
+          register: ceph_conf_content
+          changed_when: false
-function mssh {
+        - name: "▶️ Fetch Ceph keyring (raw)"
-  run ssh "$HOST" -- "$*"
+          ansible.builtin.raw: >
-}
+            cat {{ keyring_path }}
+          register: keyring_content
+          changed_when: false
-mssh sudo mkdir -p -m 755 /etc/ceph
+        - name: "▶️ Generate client secret (raw)"
-ssh reesi001.front.sepia.ceph.com 'env CEPH_KEYRING=/etc/ceph/client.sepian.keyring ceph --id sepian config generate-minimal-conf' | mssh sudo tee /etc/ceph/ceph.conf
+          ansible.builtin.raw: >
-mssh sudo chmod 644 /etc/ceph/ceph.conf
+            ceph-authtool {{ keyring_path }} -n client.sepian -p
-ssh reesi001.front.sepia.ceph.com 'cat /etc/ceph/client.sepian.keyring' | mssh sudo tee /etc/ceph/client.sepian.keyring
+          register: secret_content
-mssh sudo chmod 644 /etc/ceph/client.sepian.keyring
+          changed_when: false
-ssh reesi001.front.sepia.ceph.com 'ceph-authtool /etc/ceph/client.sepian.keyring -n client.sepian -p' | mssh sudo tee /etc/ceph/client.sepian.secret
-mssh sudo chmod 600 /etc/ceph/client.sepian.secret
+        - name: "▶️ Get Ceph monitor list (raw)"
+          ansible.builtin.raw: >
+            env CEPH_KEYRING={{ keyring_path }} ceph --id sepian mon dump --format json 2>/dev/null | jq -r '[.mons[] | .public_addrs.addrvec[] | select(.type=="v1").addr] | join(",")'
+          register: mon_hosts
+          changed_when: false
-function genmount {
+    - name: "2. Configure Ceph client files"
-  local secret=$(mssh sudo cat /etc/ceph/client.sepian.secret)
+      block:
-  # create mountpoint
+        - name: "▶️ Ensure /etc/ceph directory exists"
-  mssh sudo mkdir -p -- "$2"
+          ansible.builtin.file:
-  # make the mountpoint directory (shadowed) unwriteable to prevent accidental modification
+            path: "/etc/ceph"
-  mssh sudo chmod 000 -- "$2"
+            state: directory
-  # set it immutable to enforce that even for root
+            mode: '0755'
-  mssh sudo chattr +i -- "$2"
-  printf '172.21.2.201,172.21.2.202,172.21.2.203:%s\t%s\tceph\tname=sepian,secret=%s,mds_namespace=%s,_netdev\t0\t2\n' "$1" "$2" "$secret" "$3" | mssh sudo tee -a /etc/fstab
-}
-genmount /teuthology-archive /teuthology teuthology
+        - name: "▶️ Deploy ceph.conf"
-genmount / /scratch scratch
+          ansible.builtin.copy:
-genmount / /postfile postfile
+            content: "{{ ceph_conf_content.stdout }}"
+            dest: "{{ ceph_conf_path }}"
+            mode: '0644'
-mssh sudo systemctl daemon-reload
+        - name: "▶️ Create temporary file for keyring import"
-mssh sudo mount /teuthology
+          ansible.builtin.tempfile:
-mssh sudo mount /scratch
+            state: file
-mssh sudo mount /postfile
+            suffix: .keyring
-mssh sudo ln -s /teuthology /a
+          register: tmp_keyring
-</code>
+        - name: "▶️ Write keyring content to temporary file"
+          ansible.builtin.copy:
+            content: "{{ keyring_content.stdout }}"
+            dest: "{{ tmp_keyring.path }}"
+            mode: '0600'
+        - name: "▶️ Deploy client keyring"
+          ansible.builtin.raw: >
+            ceph-authtool {{ client_keyring_path }} --create-keyring --import-keyring {{ tmp_keyring.path }}
+        - name: "▶️ Clean up temporary keyring file"
+          ansible.builtin.file:
+            path: "{{ tmp_keyring.path }}"
+            state: absent
+        - name: "▶️ Deploy client secret file (for other tools)"
+          ansible.builtin.copy:
+            content: "{{ secret_content.stdout }}"
+            dest: "{{ secret_path }}"
+            mode: '0600'
+    - name: "3. Set up CephFS mounts"
+      block:
+        - name: "▶️ Install ceph-common on Ubuntu/Debian"
+          ansible.builtin.apt:
+            name: ceph-common
+            state: present
+            update_cache: yes
+          when: ansible_facts['os_family'] == "Debian"
+        - name: "▶️ Install Ceph Squid repo on RHEL derivatives"
+          ansible.builtin.dnf:
+            name: centos-release-ceph-squid.noarch
+            state: present
+          when: ansible_facts['os_family'] == "RedHat"
+        - name: "▶️ Install ceph-common on RHEL derivatives"
+          ansible.builtin.dnf:
+            name: ceph-common
+            state: present
+          when: ansible_facts['os_family'] == "RedHat"
+        - name: "▶️ Unmount filesystems if they currently exist"
+          ansible.posix.mount:
+            path: "{{ item.path }}"
+            state: unmounted
+          loop: "{{ mounts }}"
+        - name: "▶️ Create mount point directories"
+          ansible.builtin.file:
+            path: "{{ item.path }}"
+            state: directory
+            mode: '000'
+          loop: "{{ mounts }}"
+        - name: "▶️ Set immutable attribute on mount points"
+          ansible.builtin.file:
+            path: "{{ item.path }}"
+            attr: +i
+          loop: "{{ mounts }}"
+        - name: "▶️ Configure CephFS mounts in /etc/fstab"
+          ansible.posix.mount:
+            path: "{{ item.path }}"
+            src: "sepian@{{ ceph_fsid.stdout | trim }}.{{ item.mds_namespace }}={{ item.src }}"
+            fstype: "{{ item.fstype }}"
+            opts: "{{ item.opts }}"
+            state: mounted
+            dump: 0
+            passno: 0
+          loop: "{{ mounts }}"
+          notify: Reload Systemd
+        - name: "▶️ Create symlink for /a -> /teuthology"
+          ansible.builtin.file:
+            src: "/teuthology"
+            dest: "/a"
+            state: link
+            force: true
+    - name: "Force handlers to run before mounting"
+      ansible.builtin.meta: flush_handlers
+  handlers:
+    - name: Reload Systemd
+      listen: Reload Systemd
+      ansible.builtin.systemd:
+        daemon_reload: true
+</code>
 ==== Teuthology scheduling ====

Sepia Lab Wiki

User Tools

Site Tools

Differences

Page Tools