Differences

This shows you the differences between two versions of the page.

--- devplayground [2024/05/22 18:51]
pdonnell
+++ devplayground [2025/06/10 17:44] (current)
pdonnell [Configuring CephFS Mounts]
@@ Line 8: / Line 8: @@
 ^ Name                                             ^ Notes            ^
-| [[hardware:senta|senta01.front.sepia.ceph.com]]  | General purpose  |
+| [[hardware:senta|senta01.front.sepia.ceph.com]]  | General purpose (currently unavailable) |
 | [[hardware:senta|senta02.front.sepia.ceph.com]]  | General purpose  |
 | [[hardware:senta|senta03.front.sepia.ceph.com]]  | General purpose  |
@@ Line 19: / Line 19: @@
 ==== Playing Nice ====
-Developer playgrounds should be able to build the ''main'' branch. It is okay to use ''./install-deps.sh'' top-level scrip from the ceph source tree to update dependencies. Do not run that script from an older release of Ceph as it may break other developer's work. If you need to build an older release, lock a throwaway node like [[hardware:smithi]] and build there. Or, use a container to do the build/testing!
+Developer playgrounds should be able to build the ''main'' branch. It is okay to use ''./install-deps.sh'' top-level script from the ceph source tree to update dependencies. Do not run that script from an older release of Ceph as it may break other developer's work. If you need to build an older release, lock a throwaway node like [[hardware:smithi]] and build there. Or, use a container to do the build/testing!
 Using the developer machines to look at teuthology QA artifacts is encouraged. Try to avoid using a text editor to look at large (1GB+) debug logs as this can be RAM intensive/disruptive. Instead, prefer ''less'' or use ''tail -c xM | $EDITOR -'' to look at portions of the log in a text editor.
@@ Line 48: / Line 48: @@
 *******************************************************************************
 </code>
 ==== Configuring CephFS Mounts ====
-The following script can be run to setup CephFS mounts on a new developer playground machines:
+The following ansible playbook can be run to setup CephFS mounts on a new developer playground machines:
 <code>
-#!/bin/bash
+---
+- name: Configure Ceph Client and Mounts
+  hosts: all
+  become: true
+  vars:
+    admin_node: "reesi003.front.sepia.ceph.com"
+    ceph_conf_path: "/etc/ceph/ceph.conf"
+    keyring_path: "/etc/ceph/client.sepian.keyring"
+    secret_path: "/etc/ceph/client.sepian.secret"
+    mounts:
+      - { path: "/teuthology", fstype: "ceph", src: "/teuthology-archive", mds_namespace: "teuthology", opts: "_netdev,ro" }
+      - { path: "/scratch", fstype: "ceph", src: "/", mds_namespace: "scratch", opts: "_netdev" }
+      - { path: "/postfile", fstype: "ceph", src: "/", mds_namespace: "postfile", opts: "_netdev,ro" }
-HOST="$1"
+  tasks:
+    - name: "1. Gather Ceph configuration using raw commands"
+      delegate_to: "{{ admin_node }}"
+      block:
+        - name: "▶️ Generate minimal ceph.conf (raw)"
+          ansible.builtin.raw: >
+            env CEPH_KEYRING=/etc/ceph/client.sepian.keyring ceph --id sepian config generate-minimal-conf
+          register: ceph_conf_content
+          changed_when: false
-function run {
+        - name: "▶️ Fetch Ceph keyring (raw)"
-  printf '%s\n' "$*" >&2
+          ansible.builtin.raw: >
-  "$@"
+            cat {{ keyring_path }}
-}
+          register: keyring_content
+          changed_when: false
-function mssh {
+        - name: "▶️ Generate client secret (raw)"
-  run ssh "$HOST" -- "$*"
+          ansible.builtin.raw: >
-}
+            ceph-authtool {{ keyring_path }} -n client.sepian -p
+          register: secret_content
+          changed_when: false
-mssh sudo mkdir -p -m 755 /etc/ceph
+        - name: "▶️ Get Ceph monitor list (raw)"
-ssh reesi001.front.sepia.ceph.com 'env CEPH_KEYRING=/etc/ceph/client.sepian.keyring ceph --id sepian config generate-minimal-conf' | mssh sudo tee /etc/ceph/ceph.conf
+          ansible.builtin.raw: >
-mssh sudo chmod 644 /etc/ceph/ceph.conf
+            env CEPH_KEYRING=/etc/ceph/client.sepian.keyring ceph --id sepian mon dump --format json 2>/dev/null | jq -r '[.mons[] | .public_addrs.addrvec[] | select(.type=="v1").addr] | join(",")'
-ssh reesi001.front.sepia.ceph.com 'cat /etc/ceph/client.sepian.keyring' | mssh sudo tee /etc/ceph/client.sepian.keyring
+          register: mon_hosts
-mssh sudo chmod 644 /etc/ceph/client.sepian.keyring
+          changed_when: false
-ssh reesi001.front.sepia.ceph.com 'ceph-authtool /etc/ceph/client.sepian.keyring -n client.sepian -p' | mssh sudo tee /etc/ceph/client.sepian.secret
-mssh sudo chmod 600 /etc/ceph/client.sepian.secret
+    - name: "2. Configure Ceph client files"
+      block:
+        - name: "▶️ Ensure /etc/ceph directory exists"
+          ansible.builtin.file:
+            path: "/etc/ceph"
+            state: directory
+            mode: '0755'
-function genmount {
+        - name: "▶️ Deploy ceph.conf"
-  local secret=$(mssh sudo cat /etc/ceph/client.sepian.secret)
+          ansible.builtin.copy:
-  # create mountpoint
+            content: "{{ ceph_conf_content.stdout }}"
-  mssh sudo mkdir -p -- "$2"
+            dest: "{{ ceph_conf_path }}"
-  # make the mountpoint directory (shadowed) unwriteable to prevent accidental modification
+            mode: '0644'
-  mssh sudo chmod 000 -- "$2"
-  # set it immutable to enforce that even for root
-  mssh sudo chattr +i -- "$2"
-  printf '172.21.2.201,172.21.2.202,172.21.2.203:%s\t%s\tceph\tname=sepian,secret=%s,mds_namespace=%s,_netdev\t0\t2\n' "$1" "$2" "$secret" "$3" | mssh sudo tee -a /etc/fstab
-}
-genmount /teuthology-archive /teuthology teuthology
+        - name: "▶️ Deploy client keyring"
-genmount / /scratch scratch
+          ansible.builtin.copy:
-genmount / /postfile postfile
+            content: "{{ keyring_content.stdout }}"
+            dest: "{{ keyring_path }}"
+            mode: '0644'
-mssh sudo systemctl daemon-reload
+        - name: "▶️ Deploy client secret file (for other tools)"
-mssh sudo mount /teuthology
+          ansible.builtin.copy:
-mssh sudo mount /scratch
+            content: "{{ secret_content.stdout }}"
-mssh sudo mount /postfile
+            dest: "{{ secret_path }}"
-mssh sudo ln -s /teuthology /a
+            mode: '0600'
+    - name: "3. Set up CephFS mounts"
+      block:
+        - name: "▶️ Unmount filesystems if they currently exist"
+          ansible.posix.mount:
+            path: "{{ item.path }}"
+            state: unmounted
+          loop: "{{ mounts }}"
+        - name: "▶️ Create mount point directories"
+          ansible.builtin.file:
+            path: "{{ item.path }}"
+            state: directory
+            mode: '000'
+          loop: "{{ mounts }}"
+        - name: "▶️ Set immutable attribute on mount points"
+          ansible.builtin.file:
+            path: "{{ item.path }}"
+            attr: +i
+          register: immutable_file
+          changed_when: "'i' not in immutable_file.diff.before.attributes"
+          loop: "{{ mounts }}"
+        - name: "▶️ Configure CephFS mounts in /etc/fstab"
+          ansible.posix.mount:
+            path: "{{ item.path }}"
+            src: "{{ mon_hosts.stdout | trim }}:{{ item.src }}"
+            fstype: "{{ item.fstype }}"
+            opts: "name=sepian,secret={{ secret_content.stdout | trim }},mds_namespace={{ item.mds_namespace }},{{ item.opts }}"
+            state: mounted
+            dump: 2
+            passno: 2
+          loop: "{{ mounts }}"
+          notify: Reload Systemd
+        - name: "▶️ Create symlink for /a -> /teuthology"
+          ansible.builtin.file:
+            src: "/teuthology"
+            dest: "/a"
+            state: link
+            force: true
+    - name: "Force handlers to run before mounting"
+      ansible.builtin.meta: flush_handlers
+  handlers:
+    - name: Reload Systemd
+      listen: Reload Systemd
+      ansible.builtin.systemd:
+        daemon_reload: true
+</code>
+==== Teuthology scheduling ====
+Configure the dev playground node to schedule jobs:
+<code>
+sudo tee /etc/teuthology.yaml <<EOF
+default_machine_type: smithi
+queue_host: teuthology.front.sepia.ceph.com
+queue_port: 11300
+active_machine_types:
+  - smithi
+EOF
 </code>
+Note: killing a run is (generally) still necessary on [[services:teuthology|teuthology VM]]. This is because teuthology-kill requires killing the test processes running there.
 ==== Configuring LVM volumes using spare disks ====
@@ Line 128: / Line 219: @@
 chmod 000 $HOME/build
 sudo chattr +i $HOME/build
-echo "/dev/DevPlayground/$(whoami)-build $HOME/build xfs defaults 1 1' | sudo tee -a /etc/fstab
+echo "/dev/DevPlayground/$(whoami)-build $HOME/build xfs defaults 1 1" | sudo tee -a /etc/fstab
-sudo systemctl daemon reload
+sudo systemctl daemon-reload
 sudo mount $HOME/build
 </code>
@@ Line 150: / Line 241: @@
 <code>
-wipefs -a /dev/DevPlayground/$(whoami)-osd.*; shred -v -n 0 -z -s 16M /dev/DevPlayground/$(whoami)-osd.*; env OSD=8 ~/ceph/src/vstart.sh --bluestore-devs $(echo /dev/DevPlayground/$(whoami)-osd.* | tr ' ' ',')
+wipefs -a /dev/DevPlayground/$(whoami)-osd.*
+shred -v -n 0 -z -s 16M /dev/DevPlayground/$(whoami)-osd.*
+env OSD=8 ~/ceph/src/vstart.sh \
+    --bluestore-devs $(echo /dev/DevPlayground/$(whoami)-osd.* | tr ' ' ',')
 </code>
 Feel free to make any other volumes that you require.

Sepia Lab Wiki

User Tools

Site Tools

Differences

Page Tools