This shows you the differences between two versions of the page.
| Both sides previous revision Previous revision | |||
|
devplayground [2026/02/03 22:40] djgalloway |
devplayground [2026/09/14 14:23] (current) djgalloway |
||
|---|---|---|---|
| Line 8: | Line 8: | ||
| ^ Name ^ Notes ^ | ^ Name ^ Notes ^ | ||
| - | | [[hardware:senta|senta01.front.sepia.ceph.com]] | General purpose (currently unavailable) | | ||
| - | | [[hardware:senta|senta02.front.sepia.ceph.com]] | General purpose | | ||
| - | | [[hardware:senta|senta03.front.sepia.ceph.com]] | General purpose | | ||
| | [[hardware:vossi|vossi01.front.sepia.ceph.com]] | General purpose | | | [[hardware:vossi|vossi01.front.sepia.ceph.com]] | General purpose | | ||
| | [[hardware:vossi|vossi04.front.sepia.ceph.com]] | CephFS Team | | | [[hardware:vossi|vossi04.front.sepia.ceph.com]] | CephFS Team | | ||
| Line 52: | Line 49: | ||
| ==== Configuring CephFS Mounts ==== | ==== Configuring CephFS Mounts ==== | ||
| - | The following ansible playbook can be run to setup CephFS mounts on a new developer playground machines: | + | The following ansible playbook can be run to setup CephFS mounts on a new developer playground machines: https://wiki.sepia.ceph.com/doku.php?id=services:cephfs |
| - | <code> | ||
| - | --- | ||
| - | - name: Configure Ceph Client and Mounts | ||
| - | hosts: all | ||
| - | become: true | ||
| - | vars: | ||
| - | admin_node: "doli01.front.sepia.ceph.com" | ||
| - | ceph_conf_path: "/etc/ceph/ceph.conf" | ||
| - | keyring_path: "/etc/ceph/client.sepian.keyring" | ||
| - | client_keyring_path: "/etc/ceph/keyring" | ||
| - | secret_path: "/etc/ceph/client.sepian.secret" | ||
| - | mounts: | ||
| - | - { path: "/teuthology", fstype: "ceph", src: "/teuthology-archive", mds_namespace: "teuthology", opts: "_netdev,ro" } | ||
| - | - { path: "/scratch", fstype: "ceph", src: "/", mds_namespace: "scratch", opts: "_netdev" } | ||
| - | tasks: | ||
| - | - name: "1. Gather Ceph configuration using raw commands" | ||
| - | delegate_to: "{{ admin_node }}" | ||
| - | block: | ||
| - | - name: "▶️ Get LRC fsid" | ||
| - | ansible.builtin.raw: > | ||
| - | env CEPH_KEYRING={{ keyring_path }} ceph --id sepian fsid | ||
| - | register: ceph_fsid | ||
| - | changed_when: false | ||
| - | |||
| - | - name: "▶️ Generate minimal ceph.conf (raw)" | ||
| - | ansible.builtin.raw: > | ||
| - | env CEPH_KEYRING={{ keyring_path }} ceph --id sepian config generate-minimal-conf | ||
| - | register: ceph_conf_content | ||
| - | changed_when: false | ||
| - | |||
| - | - name: "▶️ Fetch Ceph keyring (raw)" | ||
| - | ansible.builtin.raw: > | ||
| - | cat {{ keyring_path }} | ||
| - | register: keyring_content | ||
| - | changed_when: false | ||
| - | |||
| - | - name: "▶️ Generate client secret (raw)" | ||
| - | ansible.builtin.raw: > | ||
| - | ceph-authtool {{ keyring_path }} -n client.sepian -p | ||
| - | register: secret_content | ||
| - | changed_when: false | ||
| - | |||
| - | - name: "▶️ Get Ceph monitor list (raw)" | ||
| - | ansible.builtin.raw: > | ||
| - | env CEPH_KEYRING={{ keyring_path }} ceph --id sepian mon dump --format json 2>/dev/null | jq -r '[.mons[] | .public_addrs.addrvec[] | select(.type=="v1").addr] | join(",")' | ||
| - | register: mon_hosts | ||
| - | changed_when: false | ||
| - | |||
| - | - name: "2. Configure Ceph client files" | ||
| - | block: | ||
| - | # ----------------------------- | ||
| - | # CentOS | ||
| - | # ----------------------------- | ||
| - | - name: "▶️ Install Ceph Squid repo on CentOS" | ||
| - | ansible.builtin.dnf: | ||
| - | name: centos-release-ceph-squid.noarch | ||
| - | state: present | ||
| - | when: ansible_facts['distribution'] == "CentOS" | ||
| - | | ||
| - | - name: "▶️ Install ceph-common on CentOS" | ||
| - | ansible.builtin.dnf: | ||
| - | name: ceph-common | ||
| - | state: present | ||
| - | when: ansible_facts['distribution'] == "CentOS" | ||
| - | | ||
| - | # ----------------------------- | ||
| - | # Fedora + RHEL (upstream Squid repo) | ||
| - | # ----------------------------- | ||
| - | - name: "▶️ Set Ceph EL major version (Fedora → el9)" | ||
| - | ansible.builtin.set_fact: | ||
| - | ceph_el_major: >- | ||
| - | {{ '9' if ansible_facts['distribution'] == 'Fedora' | ||
| - | else ansible_facts['distribution_major_version'] }} | ||
| - | when: ansible_facts['distribution'] in ["RedHat", "Fedora"] | ||
| - | | ||
| - | - name: "▶️ Install Ceph Squid upstream repo RPM (via URL)" | ||
| - | ansible.builtin.command: >- | ||
| - | dnf -y install | ||
| - | https://download.ceph.com/rpm-squid/el{{ ceph_el_major }}/noarch/ceph-release-1-1.el{{ ceph_el_major }}.noarch.rpm | ||
| - | args: | ||
| - | creates: /etc/yum.repos.d/ceph.repo | ||
| - | when: ansible_facts['distribution'] in ["RedHat", "Fedora"] | ||
| - | | ||
| - | - name: "▶️ Install ceph-common on Fedora / RHEL" | ||
| - | ansible.builtin.dnf: | ||
| - | name: ceph-common | ||
| - | state: present | ||
| - | when: ansible_facts['distribution'] in ["RedHat", "Fedora"] | ||
| - | |||
| - | - name: "▶️ Ensure /etc/ceph directory exists" | ||
| - | ansible.builtin.file: | ||
| - | path: "/etc/ceph" | ||
| - | state: directory | ||
| - | mode: '0755' | ||
| - | |||
| - | - name: "▶️ Deploy ceph.conf" | ||
| - | ansible.builtin.copy: | ||
| - | content: "{{ ceph_conf_content.stdout }}" | ||
| - | dest: "{{ ceph_conf_path }}" | ||
| - | mode: '0644' | ||
| - | |||
| - | - name: "▶️ Create temporary file for keyring import" | ||
| - | ansible.builtin.tempfile: | ||
| - | state: file | ||
| - | suffix: .keyring | ||
| - | register: tmp_keyring | ||
| - | |||
| - | - name: "▶️ Write keyring content to temporary file" | ||
| - | ansible.builtin.copy: | ||
| - | content: "{{ keyring_content.stdout }}" | ||
| - | dest: "{{ tmp_keyring.path }}" | ||
| - | mode: '0600' | ||
| - | |||
| - | - name: "▶️ Deploy client keyring" | ||
| - | ansible.builtin.raw: > | ||
| - | ceph-authtool {{ client_keyring_path }} --create-keyring --import-keyring {{ tmp_keyring.path }} | ||
| - | |||
| - | - name: "▶️ Clean up temporary keyring file" | ||
| - | ansible.builtin.file: | ||
| - | path: "{{ tmp_keyring.path }}" | ||
| - | state: absent | ||
| - | |||
| - | - name: "▶️ Deploy client secret file (for other tools)" | ||
| - | ansible.builtin.copy: | ||
| - | content: "{{ secret_content.stdout }}" | ||
| - | dest: "{{ secret_path }}" | ||
| - | mode: '0600' | ||
| - | |||
| - | - name: "3. Set up CephFS mounts" | ||
| - | block: | ||
| - | - name: "▶️ Unmount filesystems if they currently exist" | ||
| - | ansible.posix.mount: | ||
| - | path: "{{ item.path }}" | ||
| - | state: unmounted | ||
| - | loop: "{{ mounts }}" | ||
| - | |||
| - | - name: "▶️ Create mount point directories" | ||
| - | ansible.builtin.file: | ||
| - | path: "{{ item.path }}" | ||
| - | state: directory | ||
| - | mode: '000' | ||
| - | loop: "{{ mounts }}" | ||
| - | |||
| - | - name: "▶️ Set immutable attribute on mount points" | ||
| - | ansible.builtin.file: | ||
| - | path: "{{ item.path }}" | ||
| - | attr: +i | ||
| - | loop: "{{ mounts }}" | ||
| - | |||
| - | - name: "▶️ Configure CephFS mounts in /etc/fstab" | ||
| - | ansible.posix.mount: | ||
| - | path: "{{ item.path }}" | ||
| - | src: "sepian@{{ ceph_fsid.stdout | trim }}.{{ item.mds_namespace }}={{ item.src }}" | ||
| - | fstype: "{{ item.fstype }}" | ||
| - | opts: "{{ item.opts }}" | ||
| - | state: mounted | ||
| - | dump: 0 | ||
| - | passno: 0 | ||
| - | loop: "{{ mounts }}" | ||
| - | notify: Reload Systemd | ||
| - | |||
| - | - name: "▶️ Create symlink for /a -> /teuthology" | ||
| - | ansible.builtin.file: | ||
| - | src: "/teuthology" | ||
| - | dest: "/a" | ||
| - | state: link | ||
| - | force: true | ||
| - | |||
| - | - name: "Force handlers to run before mounting" | ||
| - | ansible.builtin.meta: flush_handlers | ||
| - | |||
| - | handlers: | ||
| - | - name: Reload Systemd | ||
| - | listen: Reload Systemd | ||
| - | ansible.builtin.systemd: | ||
| - | daemon_reload: true | ||
| - | </code> | ||
| ==== Teuthology scheduling ==== | ==== Teuthology scheduling ==== | ||
| Line 243: | Line 63: | ||
| active_machine_types: | active_machine_types: | ||
| - smithi | - smithi | ||
| + | - trial | ||
| + | - gibba | ||
| EOF | EOF | ||
| </code> | </code> | ||
| - | Note: killing a run is (generally) still necessary on [[services:teuthology|teuthology VM]]. This is because teuthology-kill requires killing the test processes running there. | + | Note: killing a run is (generally) still necessary on [[services:teuthology|teuthology host]]. This is because teuthology-kill requires killing the test processes running there. |
| ==== Configuring LVM volumes using spare disks ==== | ==== Configuring LVM volumes using spare disks ==== | ||