performance-test.sh

#!/bin/bash

# Performance Testing Script
# Validates system performance under various load conditions

set -e

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Configuration
BASE_URL="http://localhost:8080"
CONCURRENT_USERS=10
DURATION=60
RAMP_UP=30

# Test results
RESULTS_DIR="./performance-results/$(date +%Y-%m-%d_%H-%M-%S)"
mkdir -p "$RESULTS_DIR"

log() {
    echo -e "${BLUE}[$(date '+%Y-%m-%d %H:%M:%S')]${NC} $1"
}

success() {
    echo -e "${GREEN}[SUCCESS]${NC} $1"
}

warning() {
    echo -e "${YELLOW}[WARNING]${NC} $1"
}

error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

check_dependencies() {
    log "Checking dependencies..."
    
    if ! command -v curl >/dev/null 2>&1; then
        error "curl is required but not installed"
    fi
    
    if ! command -v ab >/dev/null 2>&1; then
        warning "Apache Bench (ab) not found, installing..."
        sudo apt-get update && sudo apt-get install -y apache2-utils
    fi
    
    if ! command -v wrk >/dev/null 2>&1; then
        warning "wrk not found, some tests will be skipped"
    fi
    
    success "Dependencies checked"
}

test_basic_connectivity() {
    log "Testing basic connectivity..."
    
    # Test load balancer
    if curl -s -f "$BASE_URL/health" >/dev/null; then
        success "Load balancer is accessible"
    else
        error "Load balancer is not accessible"
    fi
    
    # Test read endpoints
    if curl -s -f "$BASE_URL/api/read/health" >/dev/null; then
        success "Read endpoints are accessible"
    else
        warning "Read endpoints may not be accessible"
    fi
}

test_read_performance() {
    log "Testing read performance..."
    
    # Test coupon read performance
    ab -n 1000 -c 10 -g "$RESULTS_DIR/coupon_read.tsv" "$BASE_URL/api/v1/coupons" > "$RESULTS_DIR/coupon_read_results.txt" 2>&1
    
    # Extract key metrics
    local avg_time=$(grep "Time per request:" "$RESULTS_DIR/coupon_read_results.txt" | head -1 | awk '{print $4}')
    local requests_per_sec=$(grep "Requests per second:" "$RESULTS_DIR/coupon_read_results.txt" | awk '{print $4}')
    local failed_requests=$(grep "Failed requests:" "$RESULTS_DIR/coupon_read_results.txt" | awk '{print $3}')
    
    echo "Read Performance Results:" > "$RESULTS_DIR/read_summary.txt"
    echo "Average Response Time: ${avg_time}ms" >> "$RESULTS_DIR/read_summary.txt"
    echo "Requests per Second: $requests_per_sec" >> "$RESULTS_DIR/read_summary.txt"
    echo "Failed Requests: $failed_requests" >> "$RESULTS_DIR/read_summary.txt"
    
    success "Read performance test completed"
}

test_write_performance() {
    log "Testing write performance (will fail with 401 - this is expected)..."
    
    # Create test payload
    local test_payload='{"code":"PERF-TEST","name":"Performance Test","discountValue":10.0}'
    
    # Test write performance (expecting 401 responses)
    ab -n 100 -c 5 -p <(echo "$test_payload") -T "application/json" \
       "$BASE_URL/api/write/coupons" > "$RESULTS_DIR/write_results.txt" 2>&1
    
    # Extract metrics even for failed requests
    local avg_time=$(grep "Time per request:" "$RESULTS_DIR/write_results.txt" | head -1 | awk '{print $4}')
    local requests_per_sec=$(grep "Requests per second:" "$RESULTS_DIR/write_results.txt" | awk '{print $4}')
    
    echo "Write Performance Results (Auth Protected):" > "$RESULTS_DIR/write_summary.txt"
    echo "Average Response Time: ${avg_time}ms" >> "$RESULTS_DIR/write_summary.txt"
    echo "Requests per Second: $requests_per_sec" >> "$RESULTS_DIR/write_summary.txt"
    echo "Note: All requests expected to fail with 401 (authentication required)" >> "$RESULTS_DIR/write_summary.txt"
    
    success "Write performance test completed"
}

test_concurrent_load() {
    log "Testing concurrent load..."
    
    if command -v wrk >/dev/null 2>&1; then
        # Use wrk for more sophisticated load testing
        wrk -t4 -c20 -d30s --timeout 10s "$BASE_URL/api/v1/coupons" > "$RESULTS_DIR/concurrent_load_wrk.txt"
        
        # Extract wrk results
        local latency_avg=$(grep "Latency" "$RESULTS_DIR/concurrent_load_wrk.txt" | awk '{print $2}')
        local req_per_sec=$(grep "Req/Sec" "$RESULTS_DIR/concurrent_load_wrk.txt" | awk '{print $2}')
        local total_requests=$(grep "requests in" "$RESULTS_DIR/concurrent_load_wrk.txt" | awk '{print $1}')
        
        echo "Concurrent Load Results (wrk):" > "$RESULTS_DIR/concurrent_summary.txt"
        echo "Average Latency: $latency_avg" >> "$RESULTS_DIR/concurrent_summary.txt"
        echo "Requests per Second: $req_per_sec" >> "$RESULTS_DIR/concurrent_summary.txt"
        echo "Total Requests: $total_requests" >> "$RESULTS_DIR/concurrent_summary.txt"
        
        success "Concurrent load test with wrk completed"
    else
        # Fallback to ab
        ab -n 500 -c 20 "$BASE_URL/api/v1/coupons" > "$RESULTS_DIR/concurrent_load_ab.txt" 2>&1
        success "Concurrent load test with ab completed"
    fi
}

test_stress_endpoints() {
    log "Testing stress on various endpoints..."
    
    # Health check endpoint stress test
    ab -n 200 -c 10 "$BASE_URL/health" > "$RESULTS_DIR/health_stress.txt" 2>&1
    
    # Keycloak endpoint stress test
    ab -n 100 -c 5 "$BASE_URL/realms/master" > "$RESULTS_DIR/keycloak_stress.txt" 2>&1
    
    success "Stress testing completed"
}

measure_resource_usage() {
    log "Measuring resource usage during load..."
    
    # Start resource monitoring in background
    {
        echo "timestamp,memory_usage,cpu_usage,disk_usage" > "$RESULTS_DIR/resource_usage.csv"
        for i in {1..60}; do
            local memory=$(free | grep Mem | awk '{printf "%.1f", $3/$2 * 100.0}')
            local cpu=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | sed 's/%us,//')
            local disk=$(df / | tail -1 | awk '{print $5}' | sed 's/%//')
            echo "$(date +%s),$memory,$cpu,$disk" >> "$RESULTS_DIR/resource_usage.csv"
            sleep 1
        done
    } &
    
    local monitor_pid=$!
    
    # Run load test while monitoring
    ab -n 1000 -c 25 "$BASE_URL/api/v1/coupons" > "$RESULTS_DIR/load_with_monitoring.txt" 2>&1
    
    # Stop monitoring
    kill $monitor_pid 2>/dev/null || true
    
    success "Resource usage measurement completed"
}

test_database_performance() {
    log "Testing database performance..."
    
    # Test Redis performance
    if docker ps | grep redis >/dev/null; then
        docker exec $(docker ps -q -f name=redis) redis-cli --latency-history -i 1 2>/dev/null | head -20 > "$RESULTS_DIR/redis_latency.txt" &
        local redis_pid=$!
        
        # Run some load to test Redis under pressure
        ab -n 500 -c 10 "$BASE_URL/api/v1/coupons" > /dev/null 2>&1
        
        kill $redis_pid 2>/dev/null || true
        success "Redis performance test completed"
    fi
    
    # Test PostgreSQL connections
    if docker ps | grep postgres >/dev/null; then
        local pg_connections=$(docker exec $(docker ps -q -f name=postgres) psql -U keycloak -d keycloak -t -c "SELECT count(*) FROM pg_stat_activity;" 2>/dev/null | tr -d ' ' || echo "unknown")
        echo "PostgreSQL active connections: $pg_connections" > "$RESULTS_DIR/postgres_stats.txt"
        success "PostgreSQL performance check completed"
    fi
}

generate_performance_report() {
    log "Generating performance report..."
    
    cat > "$RESULTS_DIR/performance_report.md" << EOF
# Performance Test Report

Generated: $(date)

## Test Configuration
- Base URL: $BASE_URL
- Test Duration: ${DURATION}s
- Concurrent Users: $CONCURRENT_USERS

## Summary

### Read Performance
$(cat "$RESULTS_DIR/read_summary.txt" 2>/dev/null || echo "Not available")

### Write Performance  
$(cat "$RESULTS_DIR/write_summary.txt" 2>/dev/null || echo "Not available")

### Concurrent Load
$(cat "$RESULTS_DIR/concurrent_summary.txt" 2>/dev/null || echo "Not available")

## Recommendations

### Performance
- Monitor response times under production load
- Consider caching strategies for frequently accessed data
- Implement connection pooling for databases

### Scaling
- Current setup handles moderate load well
- Consider horizontal scaling for higher loads
- Monitor resource usage trends

### Monitoring
- Set up alerts for response time > 1s
- Monitor error rates > 1%
- Track resource usage > 80%

## Files Generated
- Resource usage: resource_usage.csv
- Detailed results: *_results.txt files
- Load test data: *.tsv files

EOF

    success "Performance report generated: $RESULTS_DIR/performance_report.md"
}

run_performance_tests() {
    log "Starting comprehensive performance testing..."
    
    check_dependencies
    test_basic_connectivity
    test_read_performance
    test_write_performance
    test_concurrent_load
    test_stress_endpoints
    measure_resource_usage
    test_database_performance
    generate_performance_report
    
    success "Performance testing completed!"
    log "Results available in: $RESULTS_DIR"
    log "View report: cat $RESULTS_DIR/performance_report.md"
}

# Main execution
case "${1:-run}" in
    run)
        run_performance_tests
        ;;
    read)
        test_read_performance
        ;;
    write)
        test_write_performance
        ;;
    concurrent)
        test_concurrent_load
        ;;
    stress)
        test_stress_endpoints
        ;;
    *)
        echo "Usage: $0 {run|read|write|concurrent|stress}"
        echo "  run        - Run all performance tests (default)"
        echo "  read       - Test read performance only"
        echo "  write      - Test write performance only"
        echo "  concurrent - Test concurrent load only"
        echo "  stress     - Test stress scenarios only"
        exit 1
        ;;
esac