We Have Oxide at Home - Part 1 - Oxide On Proxmox

Booting from DVD/CD...
CD Loader 1.2
 
Building the boot loader arguments
Looking up /BOOT/LOADER... File not found
Looking up /boot/loader... Found
Relocating the loader and the BTX
Starting the BTX loader
 
Configuring devices.
Hostname: unknown
 
 
 -- Welcome to Oxide Helios! -------------------------------------------------
 
    This bootable ISO allows you to install Helios on a traditional
    install-to-disk system; e.g., a desktop PC or a BIOS/EFI-boot
    server.
 
    To install, use "diskinfo" to locate the disk you wish to install
    to, and then use "install-helios" to format it and install the
    operating system.
 
    More information is available in the "Installing on a physical
    machine using the ISO" section of the README at:
 
        https://github.com/oxidecomputer/helios-engvm
 
 -----------------------------------------------------------------------------
 
# diskinfo
TYPE    DISK                    VID      PID              SIZE          RMV SSD
SATA    c2t0d0                  QEMU     HARDDISK          256.00 GiB   no  no
# install-helios helios01 c2t0d0

root@helios01:~# dladm show-ether
LINK            PTYPE    STATE    AUTO  SPEED-DUPLEX                    PAUSE
e1000g0         current  unknown  yes   0G-h                            bi
root@helios01:~# dladm show-phys -m
LINK         SLOT     ADDRESS            INUSE CLIENT
e1000g0      primary  bc:24:11:f4:ef:51  no   --
root@helios01:~# ipadm create-if e1000g0
root@helios01:~# ipadm create-addr -T dhcp -h $(hostname) e1000g0/dhcp
Mar  4 00:38:37 helios01 in.routed[1045]: route 0.0.0.0/8 --> 0.0.0.0 nexthop is not directly connected
root@helios01:~# svcadm restart network/service
root@helios01:~# ipadm show-addr
ADDROBJ           TYPE     STATE        ADDR
lo0/v4            static   ok           127.0.0.1/8
e1000g0/dhcp      dhcp     ok           192.168.100.209/24
lo0/v6            static   ok           ::1/128

ls ~/.ssh/id_*.pub >/dev/null 2>&1 || ssh-keygen -t ed25519 -N "" -f ~/.ssh/id_ed25519
for key in ~/.ssh/id_*.pub; do
    grep -qF "$(cat "$key")" ~/.ssh/authorized_keys 2>/dev/null || cat "$key" >> ~/.ssh/authorized_keys
done

# On your workstation:
git clone https://github.com/oxidecomputer/helios-engvm.git
cd helios-engvm
./aws/gen_userdata.sh | nc <HELIOS_IP> 1701

# Nn the Helios VM
nc -l 1701 </dev/null | bash -x

pfexec zfs create -V 8G rpool/swap
echo '/dev/zvol/dsk/rpool/swap - - swap - no -' | pfexec tee -a /etc/vfstab
pfexec /sbin/swapadd

# Update OS (creates new BE, requires reboot)
pfexec pkg update -v
pfexec reboot
 
# After reboot, delete old BE immediately to save disk space
pfexec beadm destroy -F helios        # Original install BE
# Delete any other old BEs shown by: beadm list
 
# Verify you're on the latest
uname -v                              # Should show latest helios version
 
# Install ALL required packages in one command (no new BE created - just additions)
pfexec pkg install -v \
    /developer/build-essential \
    /developer/illumos-tools \
    /developer/pkg-config \
    /library/libxmlsec1 \
    /system/zones/brand/omicron1/tools

curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y --default-toolchain 1.91.1
source ~/.cargo/env
rustc -V                              # Should show 1.91.1

git clone https://github.com/oxidecomputer/omicron.git ~/omicron
cd ~/omicron
source env.sh

# Builder prerequisites (downloads CockroachDB, ClickHouse, dpd, mgd, console assets)
pfexec ./tools/install_builder_prerequisites.sh -y
 
# Runner prerequisites (installs OPTE xde driver, downloads SoftNPU)
pfexec ./tools/install_runner_prerequisites.sh -y

pfexec chown -R $USER:staff ~/.cargo ~/.rustup ~/omicron/target ~/omicron/out

# Quick substitution if your network is 192.168.100.x:
sed -i 's/192\.168\.1\./192.168.100./g' smf/sled-agent/non-gimlet/config-rss.toml

source ~/.cargo/env
 
# Build the packaging tool (release mode - takes 45-60 min first time depending on your system )
cargo build --release --bin omicron-package
 
# Create a packaging target for non-simulated dev
./target/release/omicron-package -t default target create -p dev
 
# Build all packages
./target/release/omicron-package package

pub const COCKROACHDB_REDUNDANCY: usize = 3;  // default is 5

source ~/.cargo/env
cargo build --release --bin omicron-package
./target/release/omicron-package package

cd ~/omicron
vi smf/sled-agent/non-gimlet/config.toml

source ~/.cargo/env
 
# Create virtual disks (vdevs), simulated network (simnets), and SoftNPU zone
pfexec cargo xtask virtual-hardware create \
  --gateway-ip 192.168.100.1 \
  --pxa-start 192.168.100.70 \
  --pxa-end 192.168.100.90 \
  --vdev-size 42949672960
 
# Install (creates sled-agent SMF service, brings up all zones)
pfexec ./target/release/omicron-package install

# Watch zone count (expect ~22 when fully up, with CRDB=3 and 3 vdevs)
watch -n5 'zoneadm list -cnv | grep running | wc -l'
 
# Watch sled-agent log
tail -F $(svcs -L sled-agent)

# DNS should resolve Nexus IPs
dig recovery.sys.oxide.test @192.168.100.70 +short
 
# API should respond
curl -sf http://192.168.100.72/v1/ping

# Find a CRDB zone and execute
CRDB_ZONE=$(zoneadm list -cnv | grep oxz_cockroachdb | head -1 | awk '{print $2}')
ZPATH=$(zoneadm list -cnv | grep "$CRDB_ZONE" | awk '{print $4}')
pfexec cp /tmp/q.sql $ZPATH/root/tmp/q.sql
CRDB_IP=$(pfexec zlogin $CRDB_ZONE ipadm show-addr -p -o ADDR | grep fd00 | head -1 | sed 's|/.*||')
pfexec zlogin $CRDB_ZONE /opt/oxide/cockroachdb/bin/cockroach sql \
  --insecure --host "[$CRDB_IP]:32221" --file /tmp/q.sql

-- Find recovery silo ID
SELECT id, name FROM omicron.public.silo;
 
-- Set generous quotas 
-- (Yes this should be cleaner but my focus is on getting this working)
UPDATE omicron.public.silo_quotas
SET cpus = 9999999999,
    memory_bytes = 999999999999999999,
    storage_bytes = 999999999999999999
WHERE silo_id = '<recovery-silo-uuid>';

UPDATE omicron.public.zpool
SET control_plane_storage_buffer = 5368709120
WHERE time_deleted IS NULL;

-- Create user-facing IPv4 pool
INSERT INTO omicron.public.ip_pool (id, name, description, time_created, time_modified, time_deleted, rcgen, ip_version, reservation_type, pool_type)
VALUES (gen_random_uuid(), 'default', 'Default IP pool for instances', now(), now(), NULL, 1, 'v4', 'external_silos', 'unicast');
 
-- Get the IDs
SELECT id FROM omicron.public.ip_pool WHERE name = 'default';
SELECT id FROM omicron.public.silo WHERE name = 'recovery';
 
-- Link pool to recovery silo as default
INSERT INTO omicron.public.ip_pool_resource (ip_pool_id, resource_type, resource_id, is_default, pool_type, ip_version)
VALUES ('<pool-uuid>', 'silo', '<recovery-silo-uuid>', true, 'unicast', 'v4');
 
-- Add IP range
INSERT INTO omicron.public.ip_pool_range (id, time_created, time_modified, time_deleted, first_address, last_address, ip_pool_id, rcgen)
VALUES (gen_random_uuid(), now(), now(), NULL, '192.168.100.81', '192.168.100.90', '<pool-uuid>', 1);

cd ~/omicron
source ~/.cargo/env
 
# Wait for baseline service (MUST be "online" before proceeding)
svcs omicron/baseline
 
# Uninstall broken state
echo "y" | pfexec ./target/release/omicron-package uninstall
 
# Destroy and recreate virtual hardware
pfexec cargo xtask virtual-hardware destroy
pfexec cargo xtask virtual-hardware create \
  --gateway-ip 192.168.100.1 \
  --pxa-start 192.168.100.70 \
  --pxa-end 192.168.100.90 \
  --vdev-size 42949672960
 
# Reinstall
pfexec ./target/release/omicron-package install
 
# Monitor (takes ~5-8 minutes for all 22 zones)
watch -n5 'zoneadm list -cnv | grep running | wc -l'