Troubleshooting Guide¶

# System health check
hop3 system:check --verbose

# Application status
hop3 apps

# Recent logs
hop3 app:logs myapp --lines 50

# System info
hop3 system:info

hop3 app:logs myapp --lines 100

# Check if required packages are installed
dpkg -l | grep <package>

# Install missing packages
sudo apt install <package>

# Check Python version
python3 --version

# Ensure pip is up to date
pip install --upgrade pip

# Check for conflicting requirements
pip check

# Clear npm cache
npm cache clean --force

# Check Node version
node --version

# Try fresh install
rm -rf node_modules package-lock.json
npm install

# Check Ruby/Bundler versions
ruby --version
bundler --version

# Clear bundle cache
bundle clean --force

# Check application state
hop3 app:status myapp

# Check process logs
hop3 app:logs myapp --lines 50

# Check uWSGI status
systemctl status uwsgi-hop3

# Verify Procfile exists
cat /home/hop3/apps/myapp/src/Procfile

# Procfile format should be:
# web: gunicorn app:app

# Test command manually
cd /home/hop3/apps/myapp/src
source ../venv/bin/activate
<your-command>  # e.g., gunicorn app:app

# Check if port is in use
ss -tlnp | grep <port>

# Verify PORT environment variable
hop3 config:show myapp | grep PORT

# List all config
hop3 config:show myapp

# Set missing variables
hop3 config:set myapp KEY=value

# Check Nginx error log
sudo tail -50 /var/log/nginx/error.log

# Check if application socket exists
ls -la /tmp/uwsgi-*.sock

# Check Nginx configuration
sudo nginx -t

# Restart application
hop3 app:restart myapp

# Check if it started
hop3 app:status myapp

# Check socket permissions
ls -la /tmp/uwsgi-myapp.sock

# Should be owned by hop3:www-data

# Check uWSGI logs
sudo journalctl -u uwsgi-hop3 -n 50

# Restart uWSGI
sudo systemctl restart uwsgi-hop3

# Test configuration
sudo nginx -t

# Reload after fixing
sudo systemctl reload nginx

# Check if app has HOST_NAME configured
hop3 config:show myapp | grep HOST_NAME

# Verify Nginx config exists
ls /etc/nginx/sites-enabled/myapp.conf

# Set hostname
hop3 config:set myapp HOST_NAME=myapp.example.com

# Redeploy
hop3 deploy myapp

# Check DNS resolution
dig myapp.example.com

# Should return your server's IP

# Check Nginx config
cat /etc/nginx/sites-enabled/myapp.conf | grep server_name

# Check database addon
hop3 addons:list

# Check if DATABASE_URL is set
hop3 config:show myapp | grep DATABASE

# Test database service
hop3 system:check --verbose

# Attach addon to app
hop3 addons:attach mydb --app myapp

# Redeploy to pick up DATABASE_URL
hop3 deploy myapp

# PostgreSQL
sudo systemctl status postgresql
sudo systemctl start postgresql

# MySQL
sudo systemctl status mysql
sudo systemctl start mysql

# Check addon info
hop3 addons:info mydb

# Verify DATABASE_URL format
# postgresql://user:pass@host:port/dbname

# Edit pg_hba.conf
sudo nano /etc/postgresql/*/main/pg_hba.conf

# Add lines for Docker networks:
# host all all 172.16.0.0/12 md5
# host all all 192.168.0.0/16 md5

# Reload PostgreSQL
sudo systemctl reload postgresql

# Check if database is listening
ss -tlnp | grep 5432  # PostgreSQL
ss -tlnp | grep 3306  # MySQL

# Check firewall
sudo ufw status

# Verify bind address in config
# PostgreSQL: /etc/postgresql/*/main/postgresql.conf
# listen_addresses = '*'

# Check certificate status
sudo certbot certificates

# Check for errors
sudo journalctl -u certbot -n 50

# Verify DNS
dig myapp.example.com

# Wait for propagation (up to 48 hours)

# Check firewall
sudo ufw status

# Allow HTTP
sudo ufw allow 80/tcp

# Test renewal
sudo certbot renew --dry-run

# Force renewal
sudo certbot renew --force-renewal

# Check timer
systemctl status certbot.timer

# Check resource usage
top
htop

# Check disk I/O
iostat -x 1

# Check application logs for slow queries
hop3 app:logs myapp | grep -i slow

hop3 config:set myapp UWSGI_WORKERS=4
hop3 app:restart myapp

# Add Redis
hop3 addons:create redis myapp-cache
hop3 addons:attach myapp-cache --app myapp

# Check slow queries (PostgreSQL)
sudo -u postgres psql -c "SELECT * FROM pg_stat_statements ORDER BY total_time DESC LIMIT 10;"

# Check memory usage
free -h

# Check for OOM killer
dmesg | grep -i oom

# Add swap space
sudo fallocate -l 2G /swapfile
sudo chmod 600 /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile

# Reduce worker memory
hop3 config:set myapp UWSGI_WORKERS=2

# Check disk usage
df -h

# Find large files
du -sh /home/hop3/apps/* | sort -h
du -sh /var/log/* | sort -h

# Clean old logs
sudo journalctl --vacuum-time=7d

# Clean Docker (if used)
docker system prune -a

# Remove old deployments
# (Be careful - only remove apps you don't need)

# Check if app listens on correct interface
ss -tlnp | grep <port>

# Check firewall
sudo ufw status

# Most apps need to bind to 0.0.0.0, not 127.0.0.1
hop3 config:set myapp BIND_ADDRESS=0.0.0.0

sudo ufw allow 80/tcp
sudo ufw allow 443/tcp

# Check DNS
dig myapp.example.com

# Check /etc/hosts
cat /etc/hosts

# Verify DNS A record points to server IP
# Check with your DNS provider

# For testing, add to /etc/hosts on client:
# <server-ip> myapp.example.com

systemctl status hop3-server
journalctl -u hop3-server -n 50

# Validate config
cat /home/hop3/.config/hop3/server.toml

# Check for syntax errors (TOML format)

# Check for stale locks
lsof /home/hop3/.config/hop3/hop3.db

# If locked, stop all services and restart
sudo systemctl stop hop3-server uwsgi-hop3
sudo systemctl start hop3-server uwsgi-hop3

sudo nginx -t
journalctl -u nginx -n 50

# Find the error
sudo nginx -t

# Fix the problematic config file
sudo nano /etc/nginx/sites-enabled/<file>.conf

ss -tlnp | grep :80
# Kill the process using the port, or change Nginx port

systemctl status uwsgi-hop3
journalctl -u uwsgi-hop3 -n 50

sudo mkdir -p /run/uwsgi
sudo chown hop3:www-data /run/uwsgi

# Check all app configs
ls /etc/uwsgi-hop3/
cat /etc/uwsgi-hop3/*.ini

# Full diagnostic dump
hop3 system:info > diagnostic.txt
hop3 system:check --verbose >> diagnostic.txt
hop3 apps >> diagnostic.txt

# Include logs for specific app
hop3 app:logs myapp --lines 200 >> diagnostic.txt