AWSのクォータが通ってもGPUインスタンスは起動できなかった話

$ aws service-quotas get-service-quota \
    --service-code ec2 --quota-code L-DB2E81BA \
    --region ap-northeast-1 --query 'Quota.Value'
64.0

An error occurred (InsufficientInstanceCapacity) when calling the RunInstances
operation (reached max retries: 2): Insufficient capacity.

aws ec2 describe-instance-type-offerings \
  --location-type availability-zone \
  --filters "Name=instance-type,Values=g5.xlarge" \
  --region ap-northeast-1 \
  --query 'InstanceTypeOfferings[].Location' --output text

for SUBNET in $(aws ec2 describe-subnets --region $AWS_REGION \
                  --query 'Subnets[].SubnetId' --output text); do
  AZ=$(aws ec2 describe-subnets --subnet-ids $SUBNET --region $AWS_REGION \
         --query 'Subnets[0].AvailabilityZone' --output text)
  echo "--- 試行: $AZ ---"
  if aws ec2 run-instances \
      --region $AWS_REGION \
      --image-id $AMI_ID \
      --instance-type g6.xlarge \
      --key-name my-key \
      --security-group-ids $SG_ID \
      --subnet-id $SUBNET \
      --associate-public-ip-address \
      --block-device-mappings '[
        {"DeviceName":"/dev/sda1","Ebs":{"VolumeSize":30,"VolumeType":"gp3","DeleteOnTermination":true}},
        {"DeviceName":"/dev/sdf","Ebs":{"VolumeSize":100,"VolumeType":"gp3","DeleteOnTermination":false}}
      ]' \
      --query 'Instances[0].InstanceId' --output text 2>/dev/null; then
    echo "✅ 成功: $AZ"; break
  fi
  echo "❌ 容量なし"
done

--- 試行: ap-northeast-1c (subnet-xxxxxxxx) ---
i-0xxxxxxxxxxxxxxxx
✅ 成功: ap-northeast-1c

aws pricing get-products \
  --region us-east-1 --service-code AmazonEC2 \
  --filters \
    "Type=TERM_MATCH,Field=instanceType,Value=g6.xlarge" \
    "Type=TERM_MATCH,Field=location,Value=Asia Pacific (Tokyo)" \
    "Type=TERM_MATCH,Field=operatingSystem,Value=Linux" \
    "Type=TERM_MATCH,Field=preInstalledSw,Value=NA" \
    "Type=TERM_MATCH,Field=tenancy,Value=Shared" \
    "Type=TERM_MATCH,Field=capacitystatus,Value=Used" \
  --query 'PriceList[0]' --output text | python3 -c "
import json,sys
d = json.load(sys.stdin)
for t in d['terms']['OnDemand'].values():
    for p in t['priceDimensions'].values():
        print(p['pricePerUnit']['USD'], '/', p['unit'])
"

$ nvidia-smi
Command 'nvidia-smi' not found, but can be installed with:
sudo apt install nvidia-utils-470
sudo apt install nvidia-utils-535
sudo apt install nvidia-utils-580
...

sudo apt update
sudo apt install -y nvidia-driver-580-server
sudo reboot

+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 580.173.02             Driver Version: 580.173.02     CUDA Version: 13.0     |
+-----------------------------------------+------------------------+----------------------+
|   0  NVIDIA L4                      Off |   00000000:31:00.0 Off |                    0 |
| N/A   43C    P0             29W /   72W |       0MiB /  23034MiB |      4%      Default |
+-----------------------------------------+------------------------+----------------------+

$ lsblk -o NAME,SIZE,MODEL
NAME           SIZE MODEL
nvme0n1         30G Amazon Elastic Block Store
├─nvme0n1p1     29G
nvme1n1        100G Amazon Elastic Block Store
nvme2n1      232.8G Amazon EC2 NVMe Instance Storage

sudo mkswap /dev/nvme2n1
sudo swapon /dev/nvme2n1

$ free -h
               total        used        free      shared  buff/cache   available
Mem:            15Gi       869Mi        13Gi       2.9Mi       511Mi        14Gi
Swap:          232Gi          0B       232Gi

sudo mkfs -t ext4 /dev/nvme1n1
sudo mkdir -p /mnt/models
sudo mount /dev/nvme1n1 /mnt/models
UUID=$(sudo blkid -s UUID -o value /dev/nvme1n1)
echo "UUID=$UUID /mnt/models ext4 defaults,nofail 0 2" | sudo tee -a /etc/fstab

sudo systemctl edit ollama

[Service]
Environment="OLLAMA_MODELS=/mnt/models"

$ systemctl show ollama -p Environment | grep OLLAMA_MODELS
Environment=PATH=/usr/local/sbin:... OLLAMA_MODELS=/mnt/models:

OLLAMA_MODELS=/mnt/models:
                         ↑ これ

$ curl http://localhost:11434/
curl: (7) Failed to connect to localhost port 11434 after 0 ms

sudo mkdir -p /etc/systemd/system/ollama.service.d
 
sudo tee /etc/systemd/system/ollama.service.d/override.conf > /dev/null << 'EOF'
[Service]
Environment="OLLAMA_MODELS=/mnt/models"
EOF
 
sudo systemctl daemon-reload
sudo systemctl restart ollama

$ systemctl show ollama -p Environment | tr ' ' '\n' | grep OLLAMA
OLLAMA_MODELS=/mnt/models

# これを待つ
aws ec2 wait instance-status-ok --instance-ids $INSTANCE_ID --region $AWS_REGION