Red Hat purchased 5 servers and a 100Gb switch to be used for RGW Scale testing in Q1 2021. I just decided to put them in Sepia due to physical space constraints in Octo.

Purchasing ticket: https://redhat.service-now.com/surl.do?n=PNT0916744
Racking ticket: https://redhat.service-now.com/surl.do?n=PNT0916745

These PXE using Legacy/BIOS mode and can be provisioned via Cobbler normally.

These nodes have a regular 1Gb uplink but then 2x 25Gb connections per server to a QFX5120-32C for backend storage traffic. This switch is managed by Ceph Infra and not uplinked.

I purchased qty 5 100Gb → 4x 25Gb breakout cables with this. See https://docs.google.com/spreadsheets/d/1_ETaz5ollV_66ODFGXiVWEibecdBZIvHg9X1zN1pF88/edit#gid=0

I just created a /24 vlan on the switch giving each server it's own Aggregated Ethernet interface.

The switch is named qfx-rgw.ipmi.sepia.ceph.com and can be accessed via SSH. Here's the current switch config that seems to be working.

system {
    root-authentication {
        ssh-rsa "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDoEI+g74GcJe/03nDypMzDWqg93K1h13xQPFvCIN7ElNZ+ulY5av5KLnv/gmLbYGRPlgxC7LNZXE7YQPUo2DfA8awnDkAkE/nWvL4fQjUxmaiPthjIWsKvk9EvPIqzLjDbwsiLiRPE0Sxka0qBa1GHU1QYN6cw7l9nWF0dG/TsKw+QmuArBKMgm9Th/QVLq1ARHC/07M1n195QvBjdrKSzOUl+2L1tADHg++sEWDq/MJMiylyNdxp79Wz7aTIlR6l8NV+Ega9LUb6B6xmO7k+J+/hJF7gYOeFl+YqnZttEdz+o8FbLjhTjOF9QkBsGWlrawXtQln/cVCryV+LAt57L dgalloway@p50"; ## SECRET-DATA
    }
    services {
        ssh {
            root-login allow;
        }
        telnet;
    }
    host-name qfx-rgw;
    syslog {
        user * {
            any emergency;
        }
        file messages {
            any notice;
            authorization info;
        }
        file interactive-commands {
            interactive-commands any;
        }
    }
    processes {
        dhcp-service {
            traceoptions {
                file dhcp_logfile size 10m;
                level all;
                flag packet;
            }
        }
    }
}
chassis {
    aggregated-devices {
        ethernet {
            device-count 5;
        }
    }
    fpc 0 {
        pic 0 {
            port-range 0 3 {
                channel-speed 25g;
            }
        }
    }
}
# Placeholder for QFX platform config. 
interfaces {
    et-0/0/0 {
        unit 0 {
            family inet {
                dhcp {
                    vendor-id Juniper-qfx5120-32c;
                }
            }
        }
    }
    et-0/0/0:0 {
        enable;
        ether-options {
            802.3ad ae5;
        }
        inactive: unit 0 {
            family inet {
                dhcp {
                    vendor-id Juniper-qfx5120-32c;
                }
            }
        }
    }
    et-0/0/0:1 {
        enable;
        ether-options {
            802.3ad ae5;
        }
        inactive: unit 0 {
            family inet {
                dhcp {
                    vendor-id Juniper-qfx5120-32c;
                }
            }
        }
    }
    et-0/0/0:2 {
        enable;
        ether-options {
            802.3ad ae4;
        }
        inactive: unit 0 {
            family inet {
                dhcp {
                    vendor-id Juniper-qfx5120-32c;
                }
            }
        }
    }
    et-0/0/0:3 {
        enable;
        ether-options {
            802.3ad ae4;
        }
        inactive: unit 0 {
            family inet {
                dhcp {
                    vendor-id Juniper-qfx5120-32c;
                }
            }
        }
    }
    et-0/0/1 {
        unit 0 {
            family inet {
                dhcp {
                    vendor-id Juniper-qfx5120-32c;
                }
            }
        }
    }
    et-0/0/1:0 {
        enable;
        ether-options {
            802.3ad ae3;
        }
        inactive: unit 0 {
            family inet {
                dhcp {
                    vendor-id Juniper-qfx5120-32c;
                }
            }
        }
    }
    et-0/0/1:1 {
        enable;
        ether-options {
            802.3ad ae3;
        }
        inactive: unit 0 {
            family inet {
                dhcp {
                    vendor-id Juniper-qfx5120-32c;
                }
            }
        }

    }
    et-0/0/1:2 {
        enable;
        ether-options {
            802.3ad ae2;
        }
        inactive: unit 0 {
            family inet {
                dhcp {
                    vendor-id Juniper-qfx5120-32c;
                }
            }
        }

    }
    et-0/0/1:3 {
        enable;
        ether-options {
            802.3ad ae2;
        }
        inactive: unit 0 {
            family inet {
                dhcp {
                    vendor-id Juniper-qfx5120-32c;
                }
            }
        }
    }
    et-0/0/2 {
        unit 0 {
            family inet {
                dhcp {
                    vendor-id Juniper-qfx5120-32c;
                }
            }
        }
    }
    et-0/0/2:0 {
        enable;
        ether-options {
            802.3ad ae1;
        }
        inactive: unit 0 {
            family inet {
                dhcp {
                    vendor-id Juniper-qfx5120-32c;
                }
            }
        }
    }
    et-0/0/2:1 {
        enable;
        ether-options {
            802.3ad ae1;
        }
        inactive: unit 0 {
            family inet {
                dhcp {
                    vendor-id Juniper-qfx5120-32c;
                }
            }
        }
    }
    ae1 {
        aggregated-ether-options {
            lacp {
                active;
            }
        }
        unit 0 {
            family ethernet-switching {
                vlan {
                    members default;
                }
            }
        }
    }
    ae2 {
        aggregated-ether-options {
            lacp {
                active;
            }
        }
        unit 0 {
            family ethernet-switching {
                vlan {
                    members default;
                }
            }
        }
    }
    ae3 {
        aggregated-ether-options {
            lacp {
                active;
            }
        }
        unit 0 {
            family ethernet-switching {
                vlan {
                    members default;
                }
            }
        }
    }
    ae4 {
        aggregated-ether-options {
            lacp {
                active;
            }
        }
        unit 0 {
            family ethernet-switching {
                vlan {
                    members default;
                }
            }
        }
    }
    ae5 {
        aggregated-ether-options {
            lacp {
                active;
            }
        }
        unit 0 {
            family ethernet-switching {
                vlan {
                    members default;
                }
            }
        }
    }
    em0 {
        unit 0 {
            family inet {
                address 172.21.33.220/20;
            }
        }
    }
    irb {
        vlan-tagging;
        unit 0 {
            vlan-id 1;
            family inet {
                address 172.20.23.254/24;
            }
        }
    }
    vme {
        unit 0 {
            family inet {
                dhcp {
                    vendor-id Juniper-qfx5120-32c-AK32024047;
                }
            }
        }
    }
}
forwarding-options {
    storm-control-profiles default {
        all;
    }
}
routing-options {
    static {
        route 0.0.0.0/0 next-hop 172.21.47.254;
    }
}
protocols {
    lldp {
        port-id-subtype interface-name;
        interface all;
    }
    lldp-med {
        interface all;
    }
    igmp-snooping {
        vlan default;
    }
}
vlans {
    default {
        vlan-id 1;
        l3-interface irb.0;
    }
}

Here's what I ran to set up the bonds initially. The bonding type may change.

nmcli connection add type bond con-name bond0 ifname bond0 bond.options "mode=802.3ad"
nmcli connection add type ethernet slave-type bond con-name bond0-port1 ifname eno1np0 master bond0
nmcli connection add type ethernet slave-type bond con-name bond0-port2 ifname eno2np1 master bond0

### Change the IP in this line!
nmcli connection modify bond0 ipv4.addresses '172.20.23.X/24'

nmcli connection modify bond0 ipv4.gateway '172.20.23.254'
nmcli connection modify bond0 ipv4.method manual
nmcli connection modify bond0 connection.autoconnect-slaves 1
nmcli connection up bond0