devplayground
Differences
This shows you the differences between two versions of the page.
| Both sides previous revision Previous revision Next revision | Previous revision | ||
|
devplayground [2025/06/10 15:27] pdonnell [Configuring CephFS Mounts] |
devplayground [2025/06/10 17:44] (current) pdonnell [Configuring CephFS Mounts] |
||
|---|---|---|---|
| Line 52: | Line 52: | ||
| ==== Configuring CephFS Mounts ==== | ==== Configuring CephFS Mounts ==== | ||
| - | The following script can be run to setup CephFS mounts on a new developer playground machines: | + | The following ansible playbook can be run to setup CephFS mounts on a new developer playground machines: |
| <code> | <code> | ||
| - | #!/bin/bash | + | --- |
| + | - name: Configure Ceph Client and Mounts | ||
| + | hosts: all | ||
| + | become: true | ||
| + | vars: | ||
| + | admin_node: "reesi003.front.sepia.ceph.com" | ||
| + | ceph_conf_path: "/etc/ceph/ceph.conf" | ||
| + | keyring_path: "/etc/ceph/client.sepian.keyring" | ||
| + | secret_path: "/etc/ceph/client.sepian.secret" | ||
| + | mounts: | ||
| + | - { path: "/teuthology", fstype: "ceph", src: "/teuthology-archive", mds_namespace: "teuthology", opts: "_netdev,ro" } | ||
| + | - { path: "/scratch", fstype: "ceph", src: "/", mds_namespace: "scratch", opts: "_netdev" } | ||
| + | - { path: "/postfile", fstype: "ceph", src: "/", mds_namespace: "postfile", opts: "_netdev,ro" } | ||
| - | HOST="$1" | + | tasks: |
| + | - name: "1. Gather Ceph configuration using raw commands" | ||
| + | delegate_to: "{{ admin_node }}" | ||
| + | block: | ||
| + | - name: "▶️ Generate minimal ceph.conf (raw)" | ||
| + | ansible.builtin.raw: > | ||
| + | env CEPH_KEYRING=/etc/ceph/client.sepian.keyring ceph --id sepian config generate-minimal-conf | ||
| + | register: ceph_conf_content | ||
| + | changed_when: false | ||
| - | function run { | + | - name: "▶️ Fetch Ceph keyring (raw)" |
| - | printf '%s\n' "$*" >&2 | + | ansible.builtin.raw: > |
| - | "$@" | + | cat {{ keyring_path }} |
| - | } | + | register: keyring_content |
| + | changed_when: false | ||
| - | function mssh { | + | - name: "▶️ Generate client secret (raw)" |
| - | run ssh "$HOST" -- "$*" | + | ansible.builtin.raw: > |
| - | } | + | ceph-authtool {{ keyring_path }} -n client.sepian -p |
| + | register: secret_content | ||
| + | changed_when: false | ||
| - | ADMIN_NODE=reesi003.front.sepia.ceph.com | + | - name: "▶️ Get Ceph monitor list (raw)" |
| + | ansible.builtin.raw: > | ||
| + | env CEPH_KEYRING=/etc/ceph/client.sepian.keyring ceph --id sepian mon dump --format json 2>/dev/null | jq -r '[.mons[] | .public_addrs.addrvec[] | select(.type=="v1").addr] | join(",")' | ||
| + | register: mon_hosts | ||
| + | changed_when: false | ||
| - | mssh sudo mkdir -p -m 755 /etc/ceph | + | - name: "2. Configure Ceph client files" |
| - | ssh "$ADMIN_NODE" 'env CEPH_KEYRING=/etc/ceph/client.sepian.keyring ceph --id sepian config generate-minimal-conf' | mssh sudo tee /etc/ceph/ceph.conf | + | block: |
| - | mssh sudo chmod 644 /etc/ceph/ceph.conf | + | - name: "▶️ Ensure /etc/ceph directory exists" |
| - | ssh "$ADMIN_NODE" 'cat /etc/ceph/client.sepian.keyring' | mssh sudo tee /etc/ceph/client.sepian.keyring | + | ansible.builtin.file: |
| - | mssh sudo chmod 644 /etc/ceph/client.sepian.keyring | + | path: "/etc/ceph" |
| - | ssh "$ADMIN_NODE" 'ceph-authtool /etc/ceph/client.sepian.keyring -n client.sepian -p' | mssh sudo tee /etc/ceph/client.sepian.secret | + | state: directory |
| - | mssh sudo chmod 600 /etc/ceph/client.sepian.secret | + | mode: '0755' |
| - | MONS=$(ssh "$ADMIN_NODE" 'env CEPH_KEYRING=/etc/ceph/client.sepian.keyring ceph --id sepian mon dump --format json | jq -r '\''[.mons[] | .public_addrs.addrvec[] | select(.type=="v1").addr] | join(",")'\') | + | - name: "▶️ Deploy ceph.conf" |
| + | ansible.builtin.copy: | ||
| + | content: "{{ ceph_conf_content.stdout }}" | ||
| + | dest: "{{ ceph_conf_path }}" | ||
| + | mode: '0644' | ||
| - | function genmount { | + | - name: "▶️ Deploy client keyring" |
| - | local secret=$(mssh sudo cat /etc/ceph/client.sepian.secret) | + | ansible.builtin.copy: |
| - | mssh sudo chattr -i -- "$2" | + | content: "{{ keyring_content.stdout }}" |
| - | # create mountpoint | + | dest: "{{ keyring_path }}" |
| - | mssh sudo mkdir -p -- "$2" | + | mode: '0644' |
| - | # make the mountpoint directory (shadowed) unwriteable to prevent accidental modification | + | |
| - | mssh sudo chmod 000 -- "$2" | + | |
| - | # set it immutable to enforce that even for root | + | |
| - | mssh sudo chattr +i -- "$2" | + | |
| - | mssh sudo sed -i "/mds_namespace=$3/d" /etc/fstab | + | |
| - | printf '%s:%s\t%s\tceph\tname=sepian,secret=%s,mds_namespace=%s,_netdev\t0\t2\n' "$MONS" "$1" "$2" "$secret" "$3" | mssh sudo tee -a /etc/fstab | + | |
| - | } | + | |
| - | mssh sudo umount /teuthology | + | - name: "▶️ Deploy client secret file (for other tools)" |
| - | mssh sudo umount /scratch | + | ansible.builtin.copy: |
| - | mssh sudo umount /postfile | + | content: "{{ secret_content.stdout }}" |
| + | dest: "{{ secret_path }}" | ||
| + | mode: '0600' | ||
| - | genmount /teuthology-archive /teuthology teuthology | + | - name: "3. Set up CephFS mounts" |
| - | genmount / /scratch scratch | + | block: |
| - | genmount / /postfile postfile | + | - name: "▶️ Unmount filesystems if they currently exist" |
| + | ansible.posix.mount: | ||
| + | path: "{{ item.path }}" | ||
| + | state: unmounted | ||
| + | loop: "{{ mounts }}" | ||
| - | mssh sudo systemctl daemon-reload | + | - name: "▶️ Create mount point directories" |
| - | mssh sudo mount /teuthology | + | ansible.builtin.file: |
| - | mssh sudo mount /scratch | + | path: "{{ item.path }}" |
| - | mssh sudo mount /postfile | + | state: directory |
| - | mssh sudo ln -s /teuthology /a | + | mode: '000' |
| + | loop: "{{ mounts }}" | ||
| + | |||
| + | - name: "▶️ Set immutable attribute on mount points" | ||
| + | ansible.builtin.file: | ||
| + | path: "{{ item.path }}" | ||
| + | attr: +i | ||
| + | register: immutable_file | ||
| + | changed_when: "'i' not in immutable_file.diff.before.attributes" | ||
| + | loop: "{{ mounts }}" | ||
| + | |||
| + | - name: "▶️ Configure CephFS mounts in /etc/fstab" | ||
| + | ansible.posix.mount: | ||
| + | path: "{{ item.path }}" | ||
| + | src: "{{ mon_hosts.stdout | trim }}:{{ item.src }}" | ||
| + | fstype: "{{ item.fstype }}" | ||
| + | opts: "name=sepian,secret={{ secret_content.stdout | trim }},mds_namespace={{ item.mds_namespace }},{{ item.opts }}" | ||
| + | state: mounted | ||
| + | dump: 2 | ||
| + | passno: 2 | ||
| + | loop: "{{ mounts }}" | ||
| + | notify: Reload Systemd | ||
| + | |||
| + | - name: "▶️ Create symlink for /a -> /teuthology" | ||
| + | ansible.builtin.file: | ||
| + | src: "/teuthology" | ||
| + | dest: "/a" | ||
| + | state: link | ||
| + | force: true | ||
| + | |||
| + | - name: "Force handlers to run before mounting" | ||
| + | ansible.builtin.meta: flush_handlers | ||
| + | |||
| + | handlers: | ||
| + | - name: Reload Systemd | ||
| + | listen: Reload Systemd | ||
| + | ansible.builtin.systemd: | ||
| + | daemon_reload: true | ||
| </code> | </code> | ||
| ==== Teuthology scheduling ==== | ==== Teuthology scheduling ==== | ||
devplayground.1749569278.txt.gz · Last modified: 2025/06/10 15:27 by pdonnell
Page Tools
Except where otherwise noted, content on this wiki is licensed under the following license: CC Attribution-Share Alike 4.0 International
