Differences

This shows you the differences between two versions of the page.

--- devplayground [2024/06/03 19:49]
pdonnell [Teuthology scheduling]
+++ devplayground [2025/12/17 22:16] (current)
pdonnell [Configuring CephFS Mounts]
@@ Line 8: / Line 8: @@
 ^ Name                                             ^ Notes            ^
-| [[hardware:senta|senta01.front.sepia.ceph.com]]  | General purpose  |
+| [[hardware:senta|senta01.front.sepia.ceph.com]]  | General purpose (currently unavailable) |
 | [[hardware:senta|senta02.front.sepia.ceph.com]]  | General purpose  |
 | [[hardware:senta|senta03.front.sepia.ceph.com]]  | General purpose  |
@@ Line 19: / Line 19: @@
 ==== Playing Nice ====
-Developer playgrounds should be able to build the ''main'' branch. It is okay to use ''./install-deps.sh'' top-level scrip from the ceph source tree to update dependencies. Do not run that script from an older release of Ceph as it may break other developer's work. If you need to build an older release, lock a throwaway node like [[hardware:smithi]] and build there. Or, use a container to do the build/testing!
+Developer playgrounds should be able to build the ''main'' branch. It is okay to use ''./install-deps.sh'' top-level script from the ceph source tree to update dependencies. Do not run that script from an older release of Ceph as it may break other developer's work. If you need to build an older release, lock a throwaway node like [[hardware:smithi]] and build there. Or, use a container to do the build/testing!
 Using the developer machines to look at teuthology QA artifacts is encouraged. Try to avoid using a text editor to look at large (1GB+) debug logs as this can be RAM intensive/disruptive. Instead, prefer ''less'' or use ''tail -c xM | $EDITOR -'' to look at portions of the log in a text editor.
@@ Line 52: / Line 52: @@
 ==== Configuring CephFS Mounts ====
-The following script can be run to setup CephFS mounts on a new developer playground machines:
+The following ansible playbook can be run to setup CephFS mounts on a new developer playground machines:
 <code>
-#!/bin/bash
+---
+- name: Configure Ceph Client and Mounts
+  hosts: all
+  become: true
+  vars:
+    admin_node: "doli01.front.sepia.ceph.com"
+    ceph_conf_path: "/etc/ceph/ceph.conf"
+    keyring_path: "/etc/ceph/client.sepian.keyring"
+    client_keyring_path: "/etc/ceph/keyring"
+    secret_path: "/etc/ceph/client.sepian.secret"
+    mounts:
+      - { path: "/teuthology", fstype: "ceph", src: "/teuthology-archive", mds_namespace: "teuthology", opts: "_netdev,ro" }
+      - { path: "/scratch", fstype: "ceph", src: "/", mds_namespace: "scratch", opts: "_netdev" }
-HOST="$1"
+  tasks:
+    - name: "1. Gather Ceph configuration using raw commands"
+      delegate_to: "{{ admin_node }}"
+      block:
+        - name: "▶️ Get LRC fsid"
+          ansible.builtin.raw: >
+            env CEPH_KEYRING={{ keyring_path }} ceph --id sepian fsid
+          register: ceph_fsid
+          changed_when: false
-function run {
+        - name: "▶️ Generate minimal ceph.conf (raw)"
-  printf '%s\n' "$*" >&2
+          ansible.builtin.raw: >
-  "$@"
+            env CEPH_KEYRING={{ keyring_path }} ceph --id sepian config generate-minimal-conf
-}
+          register: ceph_conf_content
+          changed_when: false
-function mssh {
+        - name: "▶️ Fetch Ceph keyring (raw)"
-  run ssh "$HOST" -- "$*"
+          ansible.builtin.raw: >
-}
+            cat {{ keyring_path }}
+          register: keyring_content
+          changed_when: false
-mssh sudo mkdir -p -m 755 /etc/ceph
+        - name: "▶️ Generate client secret (raw)"
-ssh reesi001.front.sepia.ceph.com 'env CEPH_KEYRING=/etc/ceph/client.sepian.keyring ceph --id sepian config generate-minimal-conf' | mssh sudo tee /etc/ceph/ceph.conf
+          ansible.builtin.raw: >
-mssh sudo chmod 644 /etc/ceph/ceph.conf
+            ceph-authtool {{ keyring_path }} -n client.sepian -p
-ssh reesi001.front.sepia.ceph.com 'cat /etc/ceph/client.sepian.keyring' | mssh sudo tee /etc/ceph/client.sepian.keyring
+          register: secret_content
-mssh sudo chmod 644 /etc/ceph/client.sepian.keyring
+          changed_when: false
-ssh reesi001.front.sepia.ceph.com 'ceph-authtool /etc/ceph/client.sepian.keyring -n client.sepian -p' | mssh sudo tee /etc/ceph/client.sepian.secret
-mssh sudo chmod 600 /etc/ceph/client.sepian.secret
+        - name: "▶️ Get Ceph monitor list (raw)"
+          ansible.builtin.raw: >
+            env CEPH_KEYRING={{ keyring_path }} ceph --id sepian mon dump --format json 2>/dev/null | jq -r '[.mons[] | .public_addrs.addrvec[] | select(.type=="v1").addr] | join(",")'
+          register: mon_hosts
+          changed_when: false
-function genmount {
+    - name: "2. Configure Ceph client files"
-  local secret=$(mssh sudo cat /etc/ceph/client.sepian.secret)
+      block:
-  # create mountpoint
+        - name: "▶️ Ensure /etc/ceph directory exists"
-  mssh sudo mkdir -p -- "$2"
+          ansible.builtin.file:
-  # make the mountpoint directory (shadowed) unwriteable to prevent accidental modification
+            path: "/etc/ceph"
-  mssh sudo chmod 000 -- "$2"
+            state: directory
-  # set it immutable to enforce that even for root
+            mode: '0755'
-  mssh sudo chattr +i -- "$2"
-  printf '172.21.2.201,172.21.2.202,172.21.2.203:%s\t%s\tceph\tname=sepian,secret=%s,mds_namespace=%s,_netdev\t0\t2\n' "$1" "$2" "$secret" "$3" | mssh sudo tee -a /etc/fstab
-}
-genmount /teuthology-archive /teuthology teuthology
+        - name: "▶️ Deploy ceph.conf"
-genmount / /scratch scratch
+          ansible.builtin.copy:
-genmount / /postfile postfile
+            content: "{{ ceph_conf_content.stdout }}"
+            dest: "{{ ceph_conf_path }}"
+            mode: '0644'
-mssh sudo systemctl daemon-reload
+        - name: "▶️ Create temporary file for keyring import"
-mssh sudo mount /teuthology
+          ansible.builtin.tempfile:
-mssh sudo mount /scratch
+            state: file
-mssh sudo mount /postfile
+            suffix: .keyring
-mssh sudo ln -s /teuthology /a
+          register: tmp_keyring
-</code>
+        - name: "▶️ Write keyring content to temporary file"
+          ansible.builtin.copy:
+            content: "{{ keyring_content.stdout }}"
+            dest: "{{ tmp_keyring.path }}"
+            mode: '0600'
+        - name: "▶️ Deploy client keyring"
+          ansible.builtin.raw: >
+            ceph-authtool {{ client_keyring_path }} --create-keyring --import-keyring {{ tmp_keyring.path }}
+        - name: "▶️ Clean up temporary keyring file"
+          ansible.builtin.file:
+            path: "{{ tmp_keyring.path }}"
+            state: absent
+        - name: "▶️ Deploy client secret file (for other tools)"
+          ansible.builtin.copy:
+            content: "{{ secret_content.stdout }}"
+            dest: "{{ secret_path }}"
+            mode: '0600'
+    - name: "3. Set up CephFS mounts"
+      block:
+        - name: "▶️ Install ceph-common on Ubuntu/Debian"
+          ansible.builtin.apt:
+            name: ceph-common
+            state: present
+            update_cache: yes
+          when: ansible_facts['os_family'] == "Debian"
+        - name: "▶️ Install Ceph Squid repo on RHEL derivatives"
+          ansible.builtin.dnf:
+            name: centos-release-ceph-squid.noarch
+            state: present
+          when: ansible_facts['os_family'] == "RedHat"
+        - name: "▶️ Install ceph-common on RHEL derivatives"
+          ansible.builtin.dnf:
+            name: ceph-common
+            state: present
+          when: ansible_facts['os_family'] == "RedHat"
+        - name: "▶️ Unmount filesystems if they currently exist"
+          ansible.posix.mount:
+            path: "{{ item.path }}"
+            state: unmounted
+          loop: "{{ mounts }}"
+        - name: "▶️ Create mount point directories"
+          ansible.builtin.file:
+            path: "{{ item.path }}"
+            state: directory
+            mode: '000'
+          loop: "{{ mounts }}"
+        - name: "▶️ Set immutable attribute on mount points"
+          ansible.builtin.file:
+            path: "{{ item.path }}"
+            attr: +i
+          loop: "{{ mounts }}"
+        - name: "▶️ Configure CephFS mounts in /etc/fstab"
+          ansible.posix.mount:
+            path: "{{ item.path }}"
+            src: "sepian@{{ ceph_fsid.stdout | trim }}.{{ item.mds_namespace }}={{ item.src }}"
+            fstype: "{{ item.fstype }}"
+            opts: "{{ item.opts }}"
+            state: mounted
+            dump: 0
+            passno: 0
+          loop: "{{ mounts }}"
+          notify: Reload Systemd
+        - name: "▶️ Create symlink for /a -> /teuthology"
+          ansible.builtin.file:
+            src: "/teuthology"
+            dest: "/a"
+            state: link
+            force: true
+    - name: "Force handlers to run before mounting"
+      ansible.builtin.meta: flush_handlers
+  handlers:
+    - name: Reload Systemd
+      listen: Reload Systemd
+      ansible.builtin.systemd:
+        daemon_reload: true
+</code>
 ==== Teuthology scheduling ====
@@ Line 114: / Line 226: @@
 </code>
+Note: killing a run is (generally) still necessary on [[services:teuthology|teuthology VM]]. This is because teuthology-kill requires killing the test processes running there.
 ==== Configuring LVM volumes using spare disks ====
@@ Line 146: / Line 258: @@
 sudo chattr +i $HOME/build
 echo "/dev/DevPlayground/$(whoami)-build $HOME/build xfs defaults 1 1" | sudo tee -a /etc/fstab
-sudo systemctl daemon reload
+sudo systemctl daemon-reload
 sudo mount $HOME/build
 </code>

Sepia Lab Wiki

User Tools

Site Tools

Differences

Page Tools