diff --git a/apps/composer/src/commands/baseCommand.ts b/apps/composer/src/commands/baseCommand.ts index 6134a09c..0d4df134 100644 --- a/apps/composer/src/commands/baseCommand.ts +++ b/apps/composer/src/commands/baseCommand.ts @@ -45,7 +45,7 @@ export abstract class Command { Logger.debug`Running ${this.name} command with debug enabled`; } - Logger.header(`♫ Generating ${this.name} Configurations`); + Logger.header(`\n♫ Generating ${this.name} Configurations`); // Validate input arguments await this.validate(cliOutput); diff --git a/apps/conductor/configs/elasticsearchConfigs/datatable1-mapping.json b/apps/conductor/configs/elasticsearchConfigs/datatable1-mapping.json index bd812525..212a0b31 100644 --- a/apps/conductor/configs/elasticsearchConfigs/datatable1-mapping.json +++ b/apps/conductor/configs/elasticsearchConfigs/datatable1-mapping.json @@ -1,7 +1,5 @@ { - "index_patterns": [ - "datatable1-*" - ], + "index_patterns": ["datatable1-*"], "aliases": { "datatable1_centric": {} }, @@ -117,4 +115,4 @@ "number_of_shards": 1, "number_of_replicas": 0 } -} \ No newline at end of file +} diff --git a/apps/conductor/configs/nginx/default.conf b/apps/conductor/configs/nginx/default.conf new file mode 100644 index 00000000..dbcb280a --- /dev/null +++ b/apps/conductor/configs/nginx/default.conf @@ -0,0 +1,59 @@ +server { + listen 80; + listen [::]:80; + server_name localhost; + + # Stage Frontend + location / { + proxy_pass http://localhost:3000; + include proxy_params; + } + + # General Arranger APIs (for direct access) + location /datatable1-api/ { + proxy_pass http://localhost:5050/; + include proxy_params; + } + + location /datatable2-api/ { + proxy_pass http://localhost:5051/; + include proxy_params; + } + + location /lyric/ { + proxy_pass http://localhost:3030/; + include proxy_params; + } + + location /lectern/ { + proxy_pass http://localhost:3031/; + include proxy_params; + } + + location /song/ { + proxy_pass http://localhost:8080/; + include proxy_params; + } + + location /score/ { + proxy_pass http://localhost:8087/; + include proxy_params; + } + + location /maestro/ { + proxy_pass http://localhost:11235/; + include proxy_params; + } + + # Elasticsearch endpoint + location /es/ { + proxy_pass http://localhost:9200/; + include proxy_params; + } + + # Minio object storage + location /minio/ { + proxy_pass http://localhost:9000/; + include proxy_params; + } +} \ No newline at end of file diff --git a/apps/conductor/configs/nginx/nginx.conf b/apps/conductor/configs/nginx/nginx.conf new file mode 100644 index 00000000..99271cd8 --- /dev/null +++ b/apps/conductor/configs/nginx/nginx.conf @@ -0,0 +1,44 @@ +user www-data; +worker_processes auto; +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; +include /etc/nginx/modules-enabled/*.conf; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + types_hash_max_size 2048; + keepalive_timeout 65; + + # Gzip Settings + gzip on; + gzip_vary on; + gzip_min_length 10240; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain text/css text/xml text/javascript application/x-javascript application/xml application/javascript; + gzip_disable "MSIE [1-6]\."; + + # Security headers + add_header X-Frame-Options SAMEORIGIN; + add_header X-Content-Type-Options nosniff; + add_header X-XSS-Protection "1; mode=block"; + + # Include all .conf files in conf.d directory + include /etc/nginx/conf.d/*.conf; +} \ No newline at end of file diff --git a/apps/conductor/configs/nginx/portal b/apps/conductor/configs/nginx/portal new file mode 100644 index 00000000..6bc583fa --- /dev/null +++ b/apps/conductor/configs/nginx/portal @@ -0,0 +1,89 @@ +# Generic Nginx Site Configuration Template +# This template shows the structure - the actual script generates config dynamically +# Replace {SITE_NAME} and {PORT_*} with actual values + +# Main site +server { + listen 80; + listen [::]:80; + server_name {SITE_NAME}; + + # Frontend application + location / { + proxy_pass http://localhost:{FRONTEND_PORT}; + include proxy_params; + } + + # API endpoints for arranger datasets + # These paths are expected by the frontend + location /api/dataset_1_arranger/ { + proxy_pass http://localhost:{ARRANGER_1_PORT}/; + include proxy_params; + } + + # Add more dataset endpoints as needed: + # location /api/dataset_2_arranger/ { + # proxy_pass http://localhost:{ARRANGER_2_PORT}/; + # include proxy_params; + # } +} + +# Lyric API service +server { + listen 80; + listen [::]:80; + server_name lyric.{SITE_NAME}; + + location / { + proxy_pass http://localhost:{LYRIC_PORT}/; + include proxy_params; + } +} + +# Arranger 1 service +server { + listen 80; + listen [::]:80; + server_name arranger_1.{SITE_NAME}; + + location / { + proxy_pass http://localhost:{ARRANGER_1_PORT}/; + include proxy_params; + } +} + +# Lectern service +server { + listen 80; + listen [::]:80; + server_name lectern.{SITE_NAME}; + + location / { + proxy_pass http://localhost:{LECTERN_PORT}/; + include proxy_params; + } +} + +# Maestro service +server { + listen 80; + listen [::]:80; + server_name maestro.{SITE_NAME}; + + location / { + proxy_pass http://localhost:{MAESTRO_PORT}/; + include proxy_params; + } +} + +# Elasticsearch service +server { + listen 80; + listen [::]:80; + server_name es.{SITE_NAME}; + + location / { + proxy_pass http://localhost:{ES_PORT}/; + include proxy_params; + } +} \ No newline at end of file diff --git a/apps/conductor/configs/nginx/proxy_params b/apps/conductor/configs/nginx/proxy_params new file mode 100644 index 00000000..30f7ebd5 --- /dev/null +++ b/apps/conductor/configs/nginx/proxy_params @@ -0,0 +1,22 @@ +# Basic proxy settings +proxy_connect_timeout 60s; +proxy_send_timeout 60s; +proxy_read_timeout 60s; +proxy_buffering on; +proxy_buffer_size 8k; +proxy_buffers 8 8k; + +# Headers +proxy_set_header Host $host; +proxy_set_header X-Real-IP $remote_addr; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $scheme; + +# Websocket support +proxy_http_version 1.1; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection "upgrade"; + +# Prevent upgrading to https +proxy_hide_header Strict-Transport-Security; +proxy_hide_header Content-Security-Policy; \ No newline at end of file diff --git a/apps/conductor/configs/nginx/readme.md b/apps/conductor/configs/nginx/readme.md new file mode 100644 index 00000000..7364bfc5 --- /dev/null +++ b/apps/conductor/configs/nginx/readme.md @@ -0,0 +1,194 @@ +# Simple HTTP Nginx Configuration for Overture Prelude + +This folder contains a simple nginx configuration to expose your Overture Data Management System services via port 8080 (HTTP only). + +## Files Structure + +``` +nginx-config/ +├── nginx.conf # Main nginx configuration +├── proxy_params # Proxy parameters +├── portal # Site configuration (port 8080) +├── setup.sh # Automated setup script (safe for existing sites) +├── uninstall.sh # Clean removal script +└── README.md # This file +``` + +## Quick Setup + +### Option 1: Automated Setup (Recommended) + +The setup script is designed to be safe when other sites are already configured: + +```bash +chmod +x setup.sh +sudo ./setup.sh +``` + +**The setup script will:** + +- Create timestamped backups of existing files +- Use a unique site name (`overture-prelude`) to avoid conflicts +- Check for port conflicts and warn you +- Ask for confirmation before making changes +- Test the configuration before applying +- Preserve existing nginx configurations + +### Option 2: Manual Setup + +```bash +# Copy main configuration +sudo cp nginx.conf /etc/nginx/nginx.conf +sudo cp proxy_params /etc/nginx/proxy_params + +# Create and enable site +sudo mkdir -p /etc/nginx/sites-available /etc/nginx/sites-enabled +sudo cp portal /etc/nginx/sites-available/portal +sudo ln -sf /etc/nginx/sites-available/portal /etc/nginx/sites-enabled/portal + +# Remove default site (optional) +sudo rm -f /etc/nginx/sites-enabled/default + +# Test and reload +sudo nginx -t +sudo systemctl reload nginx +``` + +## Service Endpoints + +After deployment, your services will be available at: + +- **Frontend (Stage):** `http://your-server:8080/` +- **Arranger APIs (Frontend endpoints):** + - `http://your-server:8080/api/datatable1_arranger/` + - `http://your-server:8080/api/datatable2_arranger/` + - `http://your-server:8080/api/molecular_arranger/` +- **Arranger APIs (Direct access):** + - `http://your-server:8080/datatable1-api/` + - `http://your-server:8080/datatable2-api/` + - `http://your-server:8080/molecular-api/` +- **Data Management Services:** + - **Lyric:** `http://your-server:8080/lyric/` + - **Lectern:** `http://your-server:8080/lectern/` + - **Song:** `http://your-server:8080/song/` + - **Score:** `http://your-server:8080/score/` + - **Maestro:** `http://your-server:8080/maestro/` +- **Infrastructure:** + - **Elasticsearch:** `http://your-server:8080/es/` + - **Minio:** `http://your-server:8080/minio/` + +## Safety Features + +The setup script includes several safety measures: + +- **Automatic backups** with timestamps in `/etc/nginx/backups/` +- **Port conflict detection** - warns if port 8080 is already in use +- **Non-destructive installation** - uses unique site name `overture-prelude` +- **Configuration validation** - tests nginx config before applying +- **Interactive prompts** - asks permission before overwriting files +- **Rollback capability** - backups allow easy restoration + +### Uninstalling + +To cleanly remove the Overture Prelude configuration: + +```bash +chmod +x uninstall.sh +sudo ./uninstall.sh +``` + +This will: + +- Remove only the Overture Prelude site configuration +- Offer to restore from backups +- Leave other sites untouched +- Test configuration before finalizing + +## Port Mapping + +The configuration expects your Docker services on these ports: + +- Stage (Frontend): 3000 +- Arranger DataTable 1: 5050 +- Arranger DataTable 2: 5051 +- Arranger Molecular: 5060 +- Lyric: 3030 +- Lectern: 3031 +- Song: 8080 +- Score: 8087 +- Maestro: 11235 +- Elasticsearch: 9200 +- Minio: 9000 + +## Customization + +### Change Server Name + +Edit `portal` file and replace `localhost` with your domain: + +```nginx +server_name your-domain.com; +``` + +### Change Port + +Edit `portal` file and change the listen directive: + +```nginx +listen 80; # or any other port +listen [::]:80; +``` + +### Add Authentication + +Add basic auth to sensitive endpoints: + +```nginx +location /es/ { + auth_basic "Restricted"; + auth_basic_user_file /etc/nginx/.htpasswd; + proxy_pass http://localhost:9200/; + include proxy_params; +} +``` + +## Troubleshooting + +### Check nginx status: + +```bash +sudo systemctl status nginx +``` + +### View logs: + +```bash +sudo tail -f /var/log/nginx/access.log +sudo tail -f /var/log/nginx/error.log +``` + +### Test configuration: + +```bash +sudo nginx -t +``` + +### Test endpoints: + +```bash +curl -I http://localhost:8080/ +curl -I http://localhost:8080/lyric/ +``` + +### Common Issues + +1. **502 Bad Gateway:** Backend service not running +2. **Permission denied:** Check file permissions and nginx user +3. **Port conflicts:** Ensure port 8080 is available +4. **Path not found:** Verify service ports in Docker Compose + +## Notes + +- This is an HTTP-only configuration (no SSL/HTTPS) +- Based on Ubuntu/Debian nginx structure with sites-available/sites-enabled +- Security headers are minimal for development use diff --git a/apps/conductor/configs/nginx/setup.sh b/apps/conductor/configs/nginx/setup.sh new file mode 100644 index 00000000..17083497 --- /dev/null +++ b/apps/conductor/configs/nginx/setup.sh @@ -0,0 +1,309 @@ +#!/bin/bash + +# ============================================================================ +# Generic Nginx Site Setup Script +# ============================================================================ +# Safely installs a new nginx site configuration with multiple subdomains +# without disrupting existing configurations. +# ============================================================================ + +set -e # Exit on error + +# ─── Config ───────────────────────────────────────────────────────────────── +# Edit these variables to customize your setup +SITE_NAME="${1:-example.com}" # Can be passed as first argument +FRONTEND_PORT="${2:-3000}" # Can be passed as second argument +BACKUP_DIR="/etc/nginx/backups/$(date +%Y%m%d_%H%M%S)" + +# Service port mappings (customize as needed) +declare -A SERVICE_PORTS=( + ["lyric"]="3030" + ["arranger_1"]="5050" + ["lectern"]="3031" + ["maestro"]="11235" + ["es"]="9200" +) + +# ─── Colors ──────────────────────────────────────────────────────────────── +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +RED='\033[0;31m' +MAGENTA='\033[1;35m' +CYAN='\033[1;36m' +NC='\033[0m' # No Color + +# ─── Utilities ───────────────────────────────────────────────────────────── +step() { + echo -e "\n${MAGENTA}[$1/$2]${NC} $3" +} + +error_exit() { + echo -e "${RED}✘ $1${NC}" + exit 1 +} + +backup_file() { + local file_path="$1" + local backup_name="$2" + if [[ -f "$file_path" ]]; then + echo -e "${YELLOW}↳ Backing up $file_path${NC}" + cp "$file_path" "$BACKUP_DIR/$backup_name" + fi +} + +show_usage() { + echo "Usage: $0 [SITE_NAME] [FRONTEND_PORT]" + echo "Example: $0 mysite.com 3000" + echo "Example: $0 pantrack.genomeinformatics.org 3000" + echo "" + echo "This will create nginx configurations for:" + echo " - Main site: SITE_NAME" + echo " - Lyric API: lyric.SITE_NAME" + echo " - Arranger 1: arranger_1.SITE_NAME" + echo " - Lectern: lectern.SITE_NAME" + echo " - Maestro: maestro.SITE_NAME" + echo " - Elasticsearch: es.SITE_NAME" +} + +generate_nginx_config() { + local site_name="$1" + local frontend_port="$2" + + cat > "/tmp/nginx_site_config" << EOF +# Main ${site_name} site +server { + listen 80; + listen [::]:80; + server_name ${site_name}; + + # Frontend + location / { + proxy_pass http://localhost:${frontend_port}; + include proxy_params; + } + + # Specific Arranger dataset endpoints that the frontend expects + location /api/dataset_1_arranger/ { + proxy_pass http://localhost:${SERVICE_PORTS["arranger_1"]}/; + include proxy_params; + } +} + +EOF + + # Generate subdomain configurations + for service in "${!SERVICE_PORTS[@]}"; do + cat >> "/tmp/nginx_site_config" << EOF +# ${service^} service +server { + listen 80; + listen [::]:80; + server_name ${service}.${site_name}; + + location / { + proxy_pass http://localhost:${SERVICE_PORTS[$service]}/; + include proxy_params; + } +} + +EOF + done +} + +# ─── Start ───────────────────────────────────────────────────────────────── +echo -e "\n${CYAN}╔═════════════════════════════════════════════════════════════╗" +echo -e "║ Generic Nginx Site Setup Script ║" +echo -e "╚═════════════════════════════════════════════════════════════╝${NC}" + +# Show usage if requested +if [[ "$1" == "-h" || "$1" == "--help" ]]; then + show_usage + exit 0 +fi + +# Validate site name +if [[ -z "$SITE_NAME" || "$SITE_NAME" == "example.com" ]]; then + echo -e "${RED}Please provide a valid site name.${NC}" + show_usage + exit 1 +fi + +echo -e "${BLUE}Setting up nginx for: ${SITE_NAME}${NC}" +echo -e "${BLUE}Frontend port: ${FRONTEND_PORT}${NC}" + +# ─── Pre-flight Checks ──────────────────────────────────────────────────── +step 1 12 "Checking for prerequisites" + +if [[ $EUID -ne 0 ]]; then + error_exit "This script must be run as root (use sudo)." +fi + +command -v nginx &>/dev/null || error_exit "nginx is not installed." + +# Check if proxy_params exists or will be created +if [[ ! -f "proxy_params" && ! -f "/etc/nginx/proxy_params" ]]; then + error_exit "proxy_params file is missing. Please ensure it exists in current directory or /etc/nginx/" +fi + +mkdir -p "$BACKUP_DIR" +echo -e "${GREEN}✔ Backup directory created: $BACKUP_DIR${NC}" + +# ─── Port Conflicts Check ────────────────────────────────────────────────── +step 2 12 "Checking for port conflicts" +conflicting_ports=() +all_ports=("$FRONTEND_PORT" "${SERVICE_PORTS[@]}") + +for port in "${all_ports[@]}"; do + if netstat -tuln 2>/dev/null | grep -q ":$port "; then + conflicting_ports+=("$port") + fi +done + +if [[ ${#conflicting_ports[@]} -gt 0 ]]; then + echo -e "${YELLOW}⚠ The following ports appear to be in use: ${conflicting_ports[*]}${NC}" + echo -e "${YELLOW} Make sure your services are running on these ports.${NC}" +fi + +# ─── Generate Configuration ─────────────────────────────────────────────── +step 3 12 "Generating nginx configuration" +generate_nginx_config "$SITE_NAME" "$FRONTEND_PORT" +echo -e "${GREEN}✔ Configuration generated${NC}" + +# ─── Confirm Overwrite ──────────────────────────────────────────────────── +step 4 12 "Checking for existing site config" +if [[ -f "/etc/nginx/sites-available/$SITE_NAME" ]]; then + echo -e "${YELLOW}Site '$SITE_NAME' already exists.${NC}" + read -p "Do you want to overwrite it? (y/N): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + error_exit "Aborted by user." + fi +fi + +# ─── Backup Existing ────────────────────────────────────────────────────── +step 5 12 "Backing up existing nginx config" +backup_file "/etc/nginx/nginx.conf" "nginx.conf.backup" + +mkdir -p "$BACKUP_DIR/sites-available" "$BACKUP_DIR/sites-enabled" +cp -a /etc/nginx/sites-available/. "$BACKUP_DIR/sites-available/" 2>/dev/null || true +cp -a /etc/nginx/sites-enabled/. "$BACKUP_DIR/sites-enabled/" 2>/dev/null || true + +# ─── Install nginx.conf ─────────────────────────────────────────────────── +step 6 12 "Ensuring nginx.conf includes sites-enabled" +if ! grep -q "sites-enabled" /etc/nginx/nginx.conf 2>/dev/null; then + echo -e "${YELLOW}nginx.conf does not include sites-enabled.${NC}" + if [[ -f "nginx.conf" ]]; then + read -p "Do you want to replace nginx.conf with the provided one? (y/N): " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + cp nginx.conf /etc/nginx/nginx.conf + echo -e "${GREEN}✔ nginx.conf replaced${NC}" + else + echo -e "${YELLOW}↳ You may need to manually add 'include /etc/nginx/sites-enabled/*;' to nginx.conf${NC}" + fi + else + echo -e "${YELLOW}↳ You may need to manually add 'include /etc/nginx/sites-enabled/*;' to nginx.conf${NC}" + fi +else + echo -e "${GREEN}✔ nginx.conf already includes sites-enabled${NC}" +fi + +# ─── proxy_params ───────────────────────────────────────────────────────── +step 7 12 "Installing proxy_params" +if [[ -f "proxy_params" ]]; then + backup_file "/etc/nginx/proxy_params" "proxy_params.backup" + + if [[ -f "/etc/nginx/proxy_params" ]]; then + if ! cmp -s "proxy_params" "/etc/nginx/proxy_params"; then + echo -e "${YELLOW}proxy_params differs from existing.${NC}" + read -p "Do you want to overwrite it? (y/N): " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + cp proxy_params /etc/nginx/proxy_params + echo -e "${GREEN}✔ proxy_params updated${NC}" + else + echo -e "${YELLOW}↳ Keeping existing proxy_params${NC}" + fi + else + echo -e "${GREEN}✔ proxy_params is identical. No changes needed.${NC}" + fi + else + cp proxy_params /etc/nginx/proxy_params + echo -e "${GREEN}✔ proxy_params installed${NC}" + fi +else + echo -e "${GREEN}✔ Using existing proxy_params${NC}" +fi + +# ─── Install Site Configuration ─────────────────────────────────────────── +step 8 12 "Installing site configuration" +cp "/tmp/nginx_site_config" "/etc/nginx/sites-available/$SITE_NAME" +echo -e "${GREEN}✔ Site configuration installed${NC}" + +# ─── Enable Site ────────────────────────────────────────────────────────── +step 9 12 "Enabling site '$SITE_NAME'" +ln -sf "/etc/nginx/sites-available/$SITE_NAME" "/etc/nginx/sites-enabled/$SITE_NAME" +echo -e "${GREEN}✔ Site $SITE_NAME enabled${NC}" + +# ─── Check for Conflicts ────────────────────────────────────────────────── +step 10 12 "Checking for conflicting site configurations" +conflicting_sites=$(grep -l "server_name.*$SITE_NAME" /etc/nginx/sites-enabled/* 2>/dev/null | grep -v "$SITE_NAME" || true) +if [[ -n "$conflicting_sites" ]]; then + echo -e "${YELLOW}⚠ Found other sites with similar server names:${NC}" + for site in $conflicting_sites; do + echo " - $(basename "$site")" + done +fi + +# ─── Default Site ───────────────────────────────────────────────────────── +step 11 12 "Checking default site" +if [[ -f "/etc/nginx/sites-enabled/default" ]]; then + echo -e "${YELLOW}Default site is enabled.${NC}" + read -p "Do you want to disable it? (y/N): " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + rm -f /etc/nginx/sites-enabled/default + echo -e "${GREEN}✔ Default site disabled${NC}" + fi +fi + +# ─── Test and Reload ────────────────────────────────────────────────────── +step 12 12 "Testing nginx configuration" +if nginx -t; then + echo -e "${GREEN}✔ Configuration test passed${NC}" + read -p "Reload nginx now? (Y/n): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Nn]$ ]]; then + systemctl reload nginx + echo -e "${GREEN}✔ nginx reloaded${NC}" + else + echo "↳ Reload skipped" + fi +else + error_exit "nginx configuration test failed. See above for details." +fi + +# ─── Cleanup ────────────────────────────────────────────────────────────── +rm -f "/tmp/nginx_site_config" + +# ─── Done ───────────────────────────────────────────────────────────────── +echo -e "\n${CYAN}╔══════════════════════════╗" +echo "║ Nginx Setup Complete ║" +echo -e "╚══════════════════════════╝${NC}" + +echo -e "\n${GREEN}📂 Backups saved to:${NC} $BACKUP_DIR" +echo -e "\n${BLUE}The following domains will be served:${NC}" +echo -e " • ${SITE_NAME} (main site)" +for service in "${!SERVICE_PORTS[@]}"; do + echo -e " • ${service}.${SITE_NAME} (${service} service)" +done + +echo -e "\n${YELLOW}📋 DNS Configuration Required:${NC}" +echo -e "Make sure the following DNS records point to this server:" +echo -e " A ${SITE_NAME} → [SERVER_IP]" +for service in "${!SERVICE_PORTS[@]}"; do + echo -e " A ${service}.${SITE_NAME} → [SERVER_IP]" +done + +echo -e "\n${BLUE}To undo this setup, restore files from the backup directory above.${NC}\n" \ No newline at end of file diff --git a/apps/conductor/configs/nginx/uninstall.sh b/apps/conductor/configs/nginx/uninstall.sh new file mode 100644 index 00000000..6afd95d3 --- /dev/null +++ b/apps/conductor/configs/nginx/uninstall.sh @@ -0,0 +1,146 @@ +#!/bin/bash + +# Uninstall script for Overture Prelude nginx configuration + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +SITE_NAME="overture-prelude" + +echo -e "${BLUE}Overture Prelude nginx configuration removal script${NC}" +echo "" + +# Check if running as root or with sudo +if [[ $EUID -ne 0 ]]; then + echo -e "${RED}This script must be run as root or with sudo${NC}" + exit 1 +fi + +# Function to list available backups +list_backups() { + if [[ -d "/etc/nginx/backups" ]]; then + echo -e "${BLUE}Available backups:${NC}" + ls -la /etc/nginx/backups/ | grep "^d" | awk '{print $9}' | grep -v "^\.$\|^\.\.$" | sort -r + return 0 + else + echo -e "${YELLOW}No backup directory found${NC}" + return 1 + fi +} + +# Check what's currently installed +echo -e "${BLUE}Checking current installation...${NC}" + +# Check if our site exists +if [[ -f "/etc/nginx/sites-available/$SITE_NAME" ]]; then + echo -e "${GREEN}Found Overture Prelude site configuration${NC}" + SITE_EXISTS=true +else + echo -e "${YELLOW}Overture Prelude site configuration not found${NC}" + SITE_EXISTS=false +fi + +# Check if site is enabled +if [[ -L "/etc/nginx/sites-enabled/$SITE_NAME" ]]; then + echo -e "${GREEN}Site is currently enabled${NC}" + SITE_ENABLED=true +else + SITE_ENABLED=false +fi + +if [[ "$SITE_EXISTS" == false ]]; then + echo -e "${YELLOW}Nothing to uninstall${NC}" + exit 0 +fi + +echo "" +echo -e "${YELLOW}This will remove:${NC}" +echo " - /etc/nginx/sites-available/$SITE_NAME" +if [[ "$SITE_ENABLED" == true ]]; then + echo " - /etc/nginx/sites-enabled/$SITE_NAME (symlink)" +fi + +echo "" +read -p "Are you sure you want to continue? (y/N): " -n 1 -r +echo +if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo -e "${RED}Aborted by user${NC}" + exit 1 +fi + +# Disable site if enabled +if [[ "$SITE_ENABLED" == true ]]; then + echo -e "${BLUE}Disabling site...${NC}" + rm -f "/etc/nginx/sites-enabled/$SITE_NAME" +fi + +# Remove site configuration +echo -e "${BLUE}Removing site configuration...${NC}" +rm -f "/etc/nginx/sites-available/$SITE_NAME" + +# Ask about restoring backups +echo "" +if list_backups; then + echo "" + read -p "Do you want to restore from a backup? (y/N): " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + echo "Enter backup directory name (from list above):" + read -r backup_dir + + if [[ -d "/etc/nginx/backups/$backup_dir" ]]; then + echo -e "${BLUE}Restoring from backup: $backup_dir${NC}" + + # Restore nginx.conf if backup exists + if [[ -f "/etc/nginx/backups/$backup_dir/nginx.conf.backup" ]]; then + read -p "Restore nginx.conf? (y/N): " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + cp "/etc/nginx/backups/$backup_dir/nginx.conf.backup" "/etc/nginx/nginx.conf" + echo -e "${GREEN}nginx.conf restored${NC}" + fi + fi + + # Restore proxy_params if backup exists + if [[ -f "/etc/nginx/backups/$backup_dir/proxy_params.backup" ]]; then + read -p "Restore proxy_params? (y/N): " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + cp "/etc/nginx/backups/$backup_dir/proxy_params.backup" "/etc/nginx/proxy_params" + echo -e "${GREEN}proxy_params restored${NC}" + fi + fi + else + echo -e "${RED}Backup directory not found: $backup_dir${NC}" + fi + fi +fi + +# Test nginx configuration +echo -e "${BLUE}Testing nginx configuration...${NC}" +if nginx -t; then + echo -e "${GREEN}Configuration test successful!${NC}" + + read -p "Reload nginx now? (Y/n): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Nn]$ ]]; then + systemctl reload nginx + echo -e "${GREEN}nginx reloaded${NC}" + fi +else + echo -e "${RED}Configuration test failed!${NC}" + echo "You may need to manually fix the nginx configuration" + exit 1 +fi + +echo "" +echo -e "${GREEN}Overture Prelude nginx configuration removed successfully${NC}" +echo "" +echo -e "${BLUE}Note: This script only removes the Overture Prelude site configuration.${NC}" +echo -e "${BLUE}Other files like nginx.conf and proxy_params were left as-is unless restored from backup.${NC}" \ No newline at end of file diff --git a/apps/conductor/src/cli/commandOptions.ts b/apps/conductor/src/cli/commandOptions.ts new file mode 100644 index 00000000..99129444 --- /dev/null +++ b/apps/conductor/src/cli/commandOptions.ts @@ -0,0 +1,608 @@ +/** + * CLI Options Module - Enhanced with ErrorFactory patterns + * + * This module configures the command-line options for the Conductor CLI. + * Updated to reflect the refactored SONG/Score services and enhanced error handling. + */ + +import { Command } from "commander"; +import { Profiles } from "../types/constants"; +import { CLIOutput } from "../types/cli"; +import { Logger } from "../utils/logger"; +import { ErrorFactory } from "../utils/errors"; + +/** + * Configures the command-line options for the Conductor CLI + * Enhanced with ErrorFactory patterns for better error handling + * @param program - The Commander.js program instance + */ +export function configureCommandOptions(program: Command): void { + try { + // Global options with enhanced error handling + program + .version("1.0.0") + .description("Conductor: Data Processing Pipeline") + .option("--debug", "Enable debug mode") + // Add a custom action for the help option + .addHelpCommand("help [command]", "Display help for a specific command") + .on("--help", () => { + // Call the reference commands after the default help + Logger.showReferenceCommands(); + }); + + // Enhanced command configuration with validation + configureUploadCommand(program); + configureLecternUploadCommand(program); + configureLyricRegisterCommand(program); + configureLyricUploadCommand(program); + configureMaestroIndexCommand(program); + configureSongUploadSchemaCommand(program); + configureSongCreateStudyCommand(program); + configureSongSubmitAnalysisCommand(program); + configureSongPublishAnalysisCommand(program); + + Logger.debugString("CLI command options configured successfully"); + } catch (error) { + throw ErrorFactory.validation( + "Failed to configure CLI command options", + { error: error instanceof Error ? error.message : String(error) }, + [ + "CLI configuration may be corrupted", + "Check for conflicting command definitions", + "Try restarting the application", + "Contact support if the problem persists", + ] + ); + } +} + +/** + * Configure upload command with enhanced validation + */ +function configureUploadCommand(program: Command): void { + program + .command("upload") + .description("Upload data to Elasticsearch") + .option("-f, --file ", "Input files to process") + .option("-i, --index ", "Elasticsearch index name") + .option("-b, --batch-size ", "Batch size for uploads") + .option("--delimiter ", "CSV delimiter character") + .option("-o, --output ", "Output directory for generated files") + .option("--force", "Force overwrite of existing files") + .option("--url ", "Elasticsearch URL") + .option("--user ", "Elasticsearch username", "elastic") + .option( + "--password ", + "Elasticsearch password", + "myelasticpassword" + ) + .action(() => { + /* Handled by main.ts */ + }); +} + +/** + * Configure Lectern upload command with enhanced validation + */ +function configureLecternUploadCommand(program: Command): void { + program + .command("lecternUpload") + .description("Upload schema to Lectern server") + .option("-s, --schema-file ", "Schema JSON file to upload") + .option( + "-u, --lectern-url ", + "Lectern server URL", + process.env.LECTERN_URL || "http://localhost:3031" + ) + .option("-t, --auth-token ", "Authentication token", "") + .option("-o, --output ", "Output directory for response logs") + .option("--force", "Force overwrite of existing files") + .action(() => { + /* Handled by main.ts */ + }); +} + +/** + * Configure Lyric dictionary registration command with enhanced validation + */ +function configureLyricRegisterCommand(program: Command): void { + program + .command("lyricRegister") + .description("Register a dictionary with Lyric service") + .option( + "-u, --lyric-url ", + "Lyric server URL", + process.env.LYRIC_URL || "http://localhost:3030" + ) + .option("-c, --category-name ", "Category name") + .option("--dict-name ", "Dictionary name") + .option("-v, --dictionary-version ", "Dictionary version") + .option("-e, --default-centric-entity ", "Default centric entity") + .option("-o, --output ", "Output directory for response logs") + .option("--force", "Force overwrite of existing files") + .action(() => { + /* Handled by main.ts */ + }); +} + +/** + * Configure Lyric data loading command with enhanced validation + */ +function configureLyricUploadCommand(program: Command): void { + program + .command("lyricUpload") + .description("Load data into Lyric service") + .option( + "-u, --lyric-url ", + "Lyric server URL", + process.env.LYRIC_URL || "http://localhost:3030" + ) + .option( + "-l, --lectern-url ", + "Lectern server URL", + process.env.LECTERN_URL || "http://localhost:3031" + ) + .option( + "-d, --data-directory ", + "Directory containing CSV data files", + process.env.LYRIC_DATA + ) + .option( + "-c, --category-id ", + "Category ID", + process.env.CATEGORY_ID || "1" + ) + .option( + "-g, --organization ", + "Organization name", + process.env.ORGANIZATION || "OICR" + ) + .option( + "-m, --max-retries ", + "Maximum number of retry attempts", + process.env.MAX_RETRIES || "10" + ) + .option( + "-r, --retry-delay ", + "Delay between retry attempts in milliseconds", + process.env.RETRY_DELAY || "1000" + ) + .option("-o, --output ", "Output directory for response logs") + .option("--force", "Force overwrite of existing files") + .action(() => { + /* Handled by main.ts */ + }); +} + +/** + * Configure repository indexing command with enhanced validation + */ +function configureMaestroIndexCommand(program: Command): void { + program + .command("maestroIndex") + .description("Index a repository with optional filtering") + .option( + "--index-url ", + "Indexing service URL", + process.env.INDEX_URL || "http://localhost:11235" + ) + .option( + "--repository-code ", + "Repository code to index", + process.env.REPOSITORY_CODE + ) + .option( + "--organization ", + "Organization name filter", + process.env.ORGANIZATION + ) + .option("--id ", "Specific ID to index", process.env.ID) + .option("-o, --output ", "Output directory for response logs") + .option("--force", "Skip confirmation prompts") + .option("--debug", "Enable detailed debug logging") + .action(() => { + /* Handled by main.ts */ + }); +} + +/** + * Configure SONG schema upload command with enhanced validation + */ +function configureSongUploadSchemaCommand(program: Command): void { + program + .command("songUploadSchema") + .description("Upload schema to SONG server") + .option("-s, --schema-file ", "Schema JSON file to upload") + .option( + "-u, --song-url ", + "SONG server URL", + process.env.SONG_URL || "http://localhost:8080" + ) + .option( + "-t, --auth-token ", + "Authentication token", + process.env.AUTH_TOKEN || "123" + ) + .option("-o, --output ", "Output directory for response logs") + .option("--force", "Force overwrite of existing files") + .action(() => { + /* Handled by main.ts */ + }); +} + +/** + * Configure SONG study creation command with enhanced validation + */ +function configureSongCreateStudyCommand(program: Command): void { + program + .command("songCreateStudy") + .description("Create study in SONG server") + .option( + "-u, --song-url ", + "SONG server URL", + process.env.SONG_URL || "http://localhost:8080" + ) + .option("-i, --study-id ", "Study ID", process.env.STUDY_ID || "demo") + .option( + "-n, --study-name ", + "Study name", + process.env.STUDY_NAME || "string" + ) + .option( + "-g, --organization ", + "Organization name", + process.env.ORGANIZATION || "string" + ) + .option( + "--description ", + "Study description", + process.env.DESCRIPTION || "string" + ) + .option( + "-t, --auth-token ", + "Authentication token", + process.env.AUTH_TOKEN || "123" + ) + .option("-o, --output ", "Output directory for response logs") + .option("--force", "Force creation even if study exists", false) + .action(() => { + /* Handled by main.ts */ + }); +} + +/** + * Configure SONG analysis submission command with enhanced validation + */ +function configureSongSubmitAnalysisCommand(program: Command): void { + program + .command("songSubmitAnalysis") + .description("Submit analysis to SONG and upload files to Score") + .option("-a, --analysis-file ", "Analysis JSON file to submit") + .option( + "-u, --song-url ", + "SONG server URL", + process.env.SONG_URL || "http://localhost:8080" + ) + .option( + "-s, --score-url ", + "Score server URL", + process.env.SCORE_URL || "http://localhost:8087" + ) + .option("-i, --study-id ", "Study ID", process.env.STUDY_ID || "demo") + .option("--allow-duplicates", "Allow duplicate analysis submissions", false) + .option( + "-d, --data-dir ", + "Directory containing data files", + process.env.DATA_DIR || "./data" + ) + .option( + "--output-dir ", + "Directory for manifest file output", + process.env.OUTPUT_DIR || "./output" + ) + .option( + "-m, --manifest-file ", + "Path for manifest file", + process.env.MANIFEST_FILE + ) + .option( + "-t, --auth-token ", + "Authentication token", + process.env.AUTH_TOKEN || "123" + ) + .option( + "--ignore-undefined-md5", + "Ignore files with undefined MD5 checksums", + false + ) + .option("-o, --output ", "Output directory for response logs") + .option( + "--force", + "Force studyId from command line instead of from file", + false + ) + .action(() => { + /* Handled by main.ts */ + }); +} + +/** + * Configure SONG publish analysis command with enhanced validation + */ +function configureSongPublishAnalysisCommand(program: Command): void { + program + .command("songPublishAnalysis") + .description("Publish analysis in SONG server") + .option("-a, --analysis-id ", "Analysis ID to publish") + .option("-i, --study-id ", "Study ID", process.env.STUDY_ID || "demo") + .option( + "-u, --song-url ", + "SONG server URL", + process.env.SONG_URL || "http://localhost:8080" + ) + .option( + "-t, --auth-token ", + "Authentication token", + process.env.AUTH_TOKEN || "123" + ) + .option( + "--ignore-undefined-md5", + "Ignore files with undefined MD5 checksums", + false + ) + .option("-o, --output ", "Output directory for response logs") + .action(() => { + /* Handled by main.ts */ + }); +} + +/** + * Parses command-line arguments into a standardized CLIOutput object + * Enhanced with ErrorFactory patterns for better error handling + * + * @param options - Parsed command-line options + * @returns A CLIOutput object for command execution + */ +export function parseCommandLineArgs(options: any): CLIOutput { + try { + // Enhanced logging for debugging + Logger.debugString(`Raw options: ${JSON.stringify(options, null, 2)}`); + Logger.debugString(`Process argv: ${process.argv.join(" ")}`); + + // Enhanced profile determination with validation + let profile = options.profile || (Profiles.UPLOAD as any); + + if (!profile || typeof profile !== "string") { + throw ErrorFactory.args("Invalid or missing command profile", undefined, [ + "Ensure a valid command is specified", + "Use 'conductor --help' for available commands", + "Check command spelling and syntax", + ]); + } + + // Enhanced file path parsing with validation + const filePaths = parseFilePaths(options); + + // Enhanced configuration creation with validation + const config = createConfigFromOptions(options); + + Logger.debugString(`Parsed profile: ${profile}`); + Logger.debugString(`Parsed file paths: ${filePaths.join(", ")}`); + + // Build the standardized CLI output + return { + profile, + filePaths, + outputPath: options.output, + config, + options, + envConfig: createEnvConfig(config), + }; + } catch (error) { + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + throw ErrorFactory.validation( + "Failed to parse command line arguments", + { + options: Object.keys(options), + error: error instanceof Error ? error.message : String(error), + }, + [ + "Check command syntax and parameters", + "Verify all required arguments are provided", + "Use --debug flag for detailed error information", + "Try 'conductor --help' for command-specific help", + ] + ); + } +} + +/** + * Enhanced file path parsing with validation + */ +function parseFilePaths(options: any): string[] { + const filePaths: string[] = []; + + // Parse main file paths + if (options.file) { + if (Array.isArray(options.file)) { + filePaths.push(...options.file); + } else if (typeof options.file === "string") { + filePaths.push(options.file); + } else { + throw ErrorFactory.args("Invalid file parameter format", undefined, [ + "File parameter must be a string or array of strings", + "Example: -f data.csv", + "Example: -f file1.csv file2.csv", + "Check file parameter syntax", + ]); + } + } + + // Add template file to filePaths if present + if (options.templateFile && !filePaths.includes(options.templateFile)) { + filePaths.push(options.templateFile); + } + + // Add schema file to filePaths if present for Lectern or SONG upload + if (options.schemaFile && !filePaths.includes(options.schemaFile)) { + filePaths.push(options.schemaFile); + } + + // Add analysis file to filePaths if present for SONG analysis submission + if (options.analysisFile && !filePaths.includes(options.analysisFile)) { + filePaths.push(options.analysisFile); + } + + return filePaths; +} + +/** + * Enhanced configuration creation from options with validation + */ +function createConfigFromOptions(options: any) { + try { + return { + elasticsearch: { + url: + options.url || + process.env.ELASTICSEARCH_URL || + "http://localhost:9200", + user: options.user || process.env.ELASTICSEARCH_USER, + password: options.password || process.env.ELASTICSEARCH_PASSWORD, + index: options.index || options.indexName || "conductor-data", + templateFile: options.templateFile, + templateName: options.templateName, + alias: options.aliasName, + }, + lectern: { + url: + options.lecternUrl || + process.env.LECTERN_URL || + "http://localhost:3031", + authToken: options.authToken || process.env.LECTERN_AUTH_TOKEN || "", + }, + lyric: { + url: + options.lyricUrl || process.env.LYRIC_URL || "http://localhost:3030", + categoryName: options.categoryName || process.env.CATEGORY_NAME, + dictionaryName: options.dictName || process.env.DICTIONARY_NAME, + dictionaryVersion: + options.dictionaryVersion || process.env.DICTIONARY_VERSION, + defaultCentricEntity: + options.defaultCentricEntity || process.env.DEFAULT_CENTRIC_ENTITY, + // Data loading specific options + dataDirectory: options.dataDirectory || process.env.LYRIC_DATA, + categoryId: options.categoryId || process.env.CATEGORY_ID, + organization: options.organization || process.env.ORGANIZATION, + maxRetries: parseIntegerOption( + options.maxRetries, + process.env.MAX_RETRIES, + 10 + ), + retryDelay: parseIntegerOption( + options.retryDelay, + process.env.RETRY_DELAY, + 20000 + ), + }, + song: { + url: options.songUrl || process.env.SONG_URL || "http://localhost:8080", + authToken: options.authToken || process.env.AUTH_TOKEN || "123", + schemaFile: options.schemaFile || process.env.SONG_SCHEMA, + studyId: options.studyId || process.env.STUDY_ID || "demo", + studyName: options.studyName || process.env.STUDY_NAME || "string", + organization: + options.organization || process.env.ORGANIZATION || "string", + description: options.description || process.env.DESCRIPTION || "string", + analysisFile: options.analysisFile || process.env.ANALYSIS_FILE, + allowDuplicates: + options.allowDuplicates || + process.env.ALLOW_DUPLICATES === "true" || + false, + ignoreUndefinedMd5: + options.ignoreUndefinedMd5 || + process.env.IGNORE_UNDEFINED_MD5 === "true" || + false, + // Combined Score functionality (now part of song config) + scoreUrl: + options.scoreUrl || process.env.SCORE_URL || "http://localhost:8087", + dataDir: options.dataDir || process.env.DATA_DIR || "./data", + outputDir: options.outputDir || process.env.OUTPUT_DIR || "./output", + manifestFile: options.manifestFile || process.env.MANIFEST_FILE, + }, + maestroIndex: { + url: + options.indexUrl || process.env.INDEX_URL || "http://localhost:11235", + repositoryCode: options.repositoryCode || process.env.REPOSITORY_CODE, + organization: options.organization || process.env.ORGANIZATION, + id: options.id || process.env.ID, + }, + batchSize: parseIntegerOption(options.batchSize, undefined, 1000), + delimiter: options.delimiter || ",", + }; + } catch (error) { + throw ErrorFactory.config( + "Failed to create configuration from options", + "config", + [ + "Check all configuration parameters", + "Verify environment variables are set correctly", + "Ensure numeric values are valid integers", + "Use --debug flag for detailed configuration information", + ] + ); + } +} + +/** + * Enhanced integer parsing with validation + */ +function parseIntegerOption( + optionValue: any, + envValue: string | undefined, + defaultValue: number +): number { + const value = optionValue || envValue; + + if (value === undefined || value === null) { + return defaultValue; + } + + const parsed = parseInt(String(value)); + + if (isNaN(parsed)) { + throw ErrorFactory.validation( + `Invalid integer value: ${value}`, + { value, type: typeof value }, + [ + "Provide a valid integer number", + "Check numeric parameters and environment variables", + "Remove any non-numeric characters", + `Using default value: ${defaultValue}`, + ] + ); + } + + return parsed; +} + +/** + * Create environment configuration object from main config + */ +function createEnvConfig(config: any) { + return { + elasticsearchUrl: config.elasticsearch.url, + esUser: config.elasticsearch.user, + esPassword: config.elasticsearch.password, + indexName: config.elasticsearch.index, + lecternUrl: config.lectern.url, + lyricUrl: config.lyric.url, + songUrl: config.song.url, + lyricData: config.lyric.dataDirectory, + categoryId: config.lyric.categoryId, + organization: config.lyric.organization, + }; +} diff --git a/apps/conductor/src/config/environment.ts b/apps/conductor/src/cli/environment.ts similarity index 52% rename from apps/conductor/src/config/environment.ts rename to apps/conductor/src/cli/environment.ts index b7e2ebf8..d307475e 100644 --- a/apps/conductor/src/config/environment.ts +++ b/apps/conductor/src/cli/environment.ts @@ -4,10 +4,7 @@ * Replaces scattered process.env reads throughout the codebase */ -/** - * Centralized environment variable management - * Replaces scattered process.env reads throughout the codebase - */ +import { ErrorFactory } from "../utils/errors"; interface ServiceEndpoints { elasticsearch: { @@ -135,12 +132,23 @@ export class Environment { /** * Validate that required environment variables are set + * Enhanced with ErrorFactory for better user guidance */ static validateRequired(requiredVars: string[]): void { const missing = requiredVars.filter((varName) => !process.env[varName]); if (missing.length > 0) { - throw new Error( - `Missing required environment variables: ${missing.join(", ")}` + // UPDATED: Use ErrorFactory instead of generic Error + throw ErrorFactory.config( + `Missing required environment variables: ${missing.join(", ")}`, + "environment", + [ + `Set missing variables: ${missing.join(", ")}`, + "Check your .env file or environment configuration", + "Ensure all required services are configured", + "Use export VARIABLE_NAME=value to set variables", + "Example: export ELASTICSEARCH_URL=http://localhost:9200", + "Restart the application after setting variables", + ] ); } } @@ -161,6 +169,110 @@ export class Environment { }; } + /** + * Validate URL format for a service + * Enhanced with ErrorFactory for better error messages + */ + static validateServiceUrl(serviceName: string, url: string): void { + if (!url) { + throw ErrorFactory.config( + `${serviceName} service URL not configured`, + `${serviceName.toLowerCase()}Url`, + [ + `Set ${serviceName.toUpperCase()}_URL environment variable`, + `Use --${serviceName.toLowerCase()}-url parameter`, + "Verify the service is running and accessible", + "Check network connectivity", + `Example: export ${serviceName.toUpperCase()}_URL=http://localhost:8080`, + ] + ); + } + + try { + const parsedUrl = new URL(url); + if (!["http:", "https:"].includes(parsedUrl.protocol)) { + throw ErrorFactory.config( + `Invalid ${serviceName} URL protocol - must be http or https`, + `${serviceName.toLowerCase()}Url`, + [ + "Use http:// or https:// protocol", + `Check URL format: http://localhost:8080`, + "Verify the service URL is correct", + "Include protocol in the URL", + ] + ); + } + } catch (error) { + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + throw ErrorFactory.config( + `Invalid ${serviceName} URL format: ${url}`, + `${serviceName.toLowerCase()}Url`, + [ + "Use a valid URL format: http://localhost:8080", + "Include protocol (http:// or https://)", + "Check for typos in the URL", + "Verify port number is correct", + "Ensure no extra spaces or characters", + ] + ); + } + } + + /** + * Validate numeric environment variable + * Enhanced with ErrorFactory for better error messages + */ + static validateNumericEnv( + varName: string, + value: string, + min?: number, + max?: number + ): number { + const parsed = parseInt(value); + + if (isNaN(parsed)) { + throw ErrorFactory.config( + `Invalid numeric value for ${varName}: ${value}`, + varName.toLowerCase(), + [ + `Set ${varName} to a valid number`, + "Check environment variable format", + "Use only numeric values (no letters or symbols)", + `Example: export ${varName}=1000`, + ] + ); + } + + if (min !== undefined && parsed < min) { + throw ErrorFactory.config( + `${varName} value ${parsed} is below minimum ${min}`, + varName.toLowerCase(), + [ + `Set ${varName} to ${min} or higher`, + "Check the value meets minimum requirements", + `Example: export ${varName}=${min}`, + ] + ); + } + + if (max !== undefined && parsed > max) { + throw ErrorFactory.config( + `${varName} value ${parsed} exceeds maximum ${max}`, + varName.toLowerCase(), + [ + `Set ${varName} to ${max} or lower`, + "Check the value meets maximum requirements", + `Example: export ${varName}=${max}`, + ] + ); + } + + return parsed; + } + /** * Reset cached values (useful for testing) */ diff --git a/apps/conductor/src/cli/index.ts b/apps/conductor/src/cli/index.ts index 488a0de8..4079818e 100644 --- a/apps/conductor/src/cli/index.ts +++ b/apps/conductor/src/cli/index.ts @@ -1,16 +1,16 @@ -// src/cli/index.ts - Simplified CLI setup using new configuration system +// src/cli/index.ts import { Command } from "commander"; import { Config, CLIOutput } from "../types/cli"; -import { parseCommandLineArgs } from "./options"; -import { configureCommandOptions } from "./options"; -import { ServiceConfigManager } from "../config/serviceConfigManager"; -import { validateEnvironment } from "../validations/environment"; +import { parseCommandLineArgs } from "./commandOptions"; +import { configureCommandOptions } from "./commandOptions"; +import { validateEnvironment } from "../validations/environmentValidator"; +import { ServiceConfigManager } from "./serviceConfigManager"; import { Logger } from "../utils/logger"; +import { ErrorFactory } from "../utils/errors"; /** * Type definition for supported CLI profiles. - * This should match the CommandRegistry command names exactly. */ type CLIprofile = | "upload" @@ -27,128 +27,279 @@ type CLIprofile = * Standardized output from the CLI parsing process. */ interface CLIOutputInternal { - /** Configuration settings for the command */ config: Config; - - /** List of input file paths specified by the user */ filePaths: string[]; - - /** The selected profile/command to execute */ profile: CLIprofile; - - /** Optional output directory path */ outputPath?: string; - - /** Environment configuration */ envConfig: any; - - /** Raw command options for command-specific handling */ options: any; } /** * Sets up the CLI environment and parses command-line arguments. - * Now uses the simplified configuration system. + * Fixed to apply defaults BEFORE validation. */ export async function setupCLI(): Promise { const program = new Command(); try { - Logger.debug("Conductor CLI"); + Logger.debugString("Conductor CLI setup starting"); // Configure command options configureCommandOptions(program); - - Logger.debug("Raw arguments:", process.argv); program.parse(process.argv); - // Get the command const commandName = program.args[0]; + if (!commandName) { + throw ErrorFactory.args("No command specified", undefined, [ + "Provide a command to execute", + "Use 'conductor --help' to see available commands", + "Example: conductor upload -f data.csv", + ]); + } - // Get the specific command const command = program.commands.find((cmd) => cmd.name() === commandName); - - // Extract options for the specific command - const options = command ? command.opts() : {}; - - Logger.debug("Parsed options:", options); - Logger.debug("Remaining arguments:", program.args); - - // Determine the profile based on the command name - let profile: CLIprofile = "upload"; // Default to upload - switch (commandName) { - case "upload": - profile = "upload"; - break; - case "lecternUpload": - profile = "lecternUpload"; - break; - case "lyricRegister": - profile = "lyricRegister"; - break; - case "lyricUpload": - profile = "lyricUpload"; - break; - case "maestroIndex": - profile = "maestroIndex"; - break; - case "songUploadSchema": - profile = "songUploadSchema"; - break; - case "songCreateStudy": - profile = "songCreateStudy"; - break; - case "songSubmitAnalysis": - profile = "songSubmitAnalysis"; - break; - case "songPublishAnalysis": - profile = "songPublishAnalysis"; - break; + if (!command) { + throw ErrorFactory.args( + `Command '${commandName}' not found`, + commandName, + [ + "Check command spelling and case", + "Use 'conductor --help' for available commands", + ] + ); } - // Validate environment for services that need it - // Skip validation for services that don't use Elasticsearch - const skipElasticsearchValidation: CLIprofile[] = [ - "lecternUpload", - "lyricRegister", - "lyricUpload", - "songUploadSchema", - "songCreateStudy", - "songSubmitAnalysis", - "songPublishAnalysis", - ]; - - if (!skipElasticsearchValidation.includes(profile)) { - const esConfig = ServiceConfigManager.createElasticsearchConfig({ - url: options.url || undefined, - }); - await validateEnvironment({ - elasticsearchUrl: esConfig.url, - }); - } + const options = command.opts(); + const profile = determineProfile(commandName); + + Logger.debug`Parsed options: ${JSON.stringify(options)}`; + + // Create configuration with proper defaults FIRST + const config = createConfigWithDefaults(options); - // Create simplified configuration using new system - const config = createSimplifiedConfig(options); + // Then validate environment with defaults applied + await validateEnvironmentForProfile(profile, config); - // Parse command-line arguments into CLIOutput + // Parse CLI output with the proper config const cliOutput = parseCommandLineArgs({ ...options, profile, - // Ensure schema file is added to filePaths for relevant uploads ...(options.schemaFile ? { file: options.schemaFile } : {}), - // Ensure analysis file is added to filePaths for SONG analysis submission ...(options.analysisFile ? { file: options.analysisFile } : {}), }); - // Override with simplified config + // Override with the config that has defaults properly applied cliOutput.config = config; - Logger.debug("CLI setup completed successfully"); + Logger.debugString("CLI setup completed successfully"); return cliOutput; } catch (error) { - console.error("Error during CLI setup:", error); - throw error; + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + throw ErrorFactory.validation( + "CLI setup failed", + { + error: error instanceof Error ? error.message : String(error), + args: process.argv, + }, + [ + "Check command syntax and parameters", + "Verify all required services are configured", + "Use --debug flag for detailed error information", + "Try 'conductor --help' for usage information", + ] + ); + } +} + +function determineProfile(commandName: string): CLIprofile { + const profileMap: Record = { + upload: "upload", + lecternUpload: "lecternUpload", + lyricRegister: "lyricRegister", + lyricUpload: "lyricUpload", + maestroIndex: "maestroIndex", + songUploadSchema: "songUploadSchema", + songCreateStudy: "songCreateStudy", + songSubmitAnalysis: "songSubmitAnalysis", + songPublishAnalysis: "songPublishAnalysis", + }; + + const profile = profileMap[commandName]; + if (!profile) { + const availableProfiles = Object.keys(profileMap).join(", "); + throw ErrorFactory.args( + `Unknown command profile: ${commandName}`, + commandName, + [`Available commands: ${availableProfiles}`] + ); } + + return profile; +} + +/** + * Create configuration with proper defaults applied + */ +function createConfigWithDefaults(options: any): Config { + // Apply defaults first, then override with provided options + const defaultConfig = { + elasticsearch: { + url: process.env.ELASTICSEARCH_URL || "http://localhost:9200", + user: process.env.ELASTICSEARCH_USER || "elastic", + password: process.env.ELASTICSEARCH_PASSWORD || "myelasticpassword", + index: process.env.ELASTICSEARCH_INDEX || "conductor-data", + }, + lectern: { + url: process.env.LECTERN_URL || "http://localhost:3031", + authToken: process.env.LECTERN_AUTH_TOKEN || "", + }, + lyric: { + url: process.env.LYRIC_URL || "http://localhost:3030", + categoryId: process.env.CATEGORY_ID || "1", + organization: process.env.ORGANIZATION || "OICR", + maxRetries: parseInt(process.env.MAX_RETRIES || "10"), + retryDelay: parseInt(process.env.RETRY_DELAY || "20000"), + }, + song: { + url: process.env.SONG_URL || "http://localhost:8080", + authToken: process.env.AUTH_TOKEN || "123", + studyId: process.env.STUDY_ID || "demo", + studyName: process.env.STUDY_NAME || "string", + organization: process.env.ORGANIZATION || "string", + description: process.env.DESCRIPTION || "string", + allowDuplicates: process.env.ALLOW_DUPLICATES === "true" || false, + ignoreUndefinedMd5: process.env.IGNORE_UNDEFINED_MD5 === "true" || false, + scoreUrl: process.env.SCORE_URL || "http://localhost:8087", + dataDir: process.env.DATA_DIR || "./data", + outputDir: process.env.OUTPUT_DIR || "./output", + }, + maestroIndex: { + url: process.env.INDEX_URL || "http://localhost:11235", + repositoryCode: process.env.REPOSITORY_CODE, + organization: process.env.ORGANIZATION, + id: process.env.ID, + }, + batchSize: parseInt(process.env.BATCH_SIZE || "1000"), + delimiter: process.env.CSV_DELIMITER || ",", + }; + + // Now override with command line options + return { + elasticsearch: { + url: options.url || defaultConfig.elasticsearch.url, + user: options.user || defaultConfig.elasticsearch.user, + password: options.password || defaultConfig.elasticsearch.password, + index: + options.index || options.indexName || defaultConfig.elasticsearch.index, + templateFile: options.templateFile, + templateName: options.templateName, + alias: options.aliasName, + }, + lectern: { + url: options.lecternUrl || defaultConfig.lectern.url, + authToken: options.authToken || defaultConfig.lectern.authToken, + }, + lyric: { + url: options.lyricUrl || defaultConfig.lyric.url, + categoryName: options.categoryName, + dictionaryName: options.dictName, + dictionaryVersion: options.dictionaryVersion, + defaultCentricEntity: options.defaultCentricEntity, + dataDirectory: options.dataDirectory, + categoryId: options.categoryId || defaultConfig.lyric.categoryId, + organization: options.organization || defaultConfig.lyric.organization, + maxRetries: options.maxRetries + ? parseInt(options.maxRetries) + : defaultConfig.lyric.maxRetries, + retryDelay: options.retryDelay + ? parseInt(options.retryDelay) + : defaultConfig.lyric.retryDelay, + }, + song: { + url: options.songUrl || defaultConfig.song.url, + authToken: options.authToken || defaultConfig.song.authToken, + schemaFile: options.schemaFile, + studyId: options.studyId || defaultConfig.song.studyId, + studyName: options.studyName || defaultConfig.song.studyName, + organization: options.organization || defaultConfig.song.organization, + description: options.description || defaultConfig.song.description, + analysisFile: options.analysisFile, + allowDuplicates: + options.allowDuplicates !== undefined + ? options.allowDuplicates + : defaultConfig.song.allowDuplicates, + ignoreUndefinedMd5: + options.ignoreUndefinedMd5 !== undefined + ? options.ignoreUndefinedMd5 + : defaultConfig.song.ignoreUndefinedMd5, + scoreUrl: options.scoreUrl || defaultConfig.song.scoreUrl, + dataDir: options.dataDir || defaultConfig.song.dataDir, + outputDir: options.outputDir || defaultConfig.song.outputDir, + manifestFile: options.manifestFile, + }, + maestroIndex: { + url: options.indexUrl || defaultConfig.maestroIndex.url, + repositoryCode: + options.repositoryCode || defaultConfig.maestroIndex.repositoryCode, + organization: + options.organization || defaultConfig.maestroIndex.organization, + id: options.id || defaultConfig.maestroIndex.id, + }, + batchSize: options.batchSize + ? parseInt(options.batchSize) + : defaultConfig.batchSize, + delimiter: options.delimiter || defaultConfig.delimiter, + }; +} + +/** + * Enhanced environment validation for specific profiles + */ +async function validateEnvironmentForProfile( + profile: CLIprofile, + config: Config +): Promise { + // Skip validation for services that don't use Elasticsearch + const skipElasticsearchValidation: CLIprofile[] = [ + "lecternUpload", + "lyricRegister", + "lyricUpload", + "songUploadSchema", + "songCreateStudy", + "songSubmitAnalysis", + "songPublishAnalysis", + ]; + + if (!skipElasticsearchValidation.includes(profile)) { + try { + await validateEnvironment({ + elasticsearchUrl: config.elasticsearch.url, + }); + } catch (error) { + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + throw ErrorFactory.config( + "Environment validation failed for Elasticsearch", + "elasticsearch", + [ + "Check Elasticsearch configuration and connectivity", + "Verify ELASTICSEARCH_URL environment variable", + "Ensure Elasticsearch service is running", + "Use --url parameter to specify Elasticsearch URL", + ] + ); + } + } + + Logger.debugString( + `Environment validation completed for profile: ${profile}` + ); } /** diff --git a/apps/conductor/src/cli/options.ts b/apps/conductor/src/cli/options.ts deleted file mode 100644 index 09779028..00000000 --- a/apps/conductor/src/cli/options.ts +++ /dev/null @@ -1,451 +0,0 @@ -/** - * CLI Options Module - Complete Updated Version - * - * This module configures the command-line options for the Conductor CLI. - * Updated to reflect the refactored SONG/Score services and removed commands. - */ - -import { Command } from "commander"; -import { Profiles } from "../types/constants"; -import { CLIOutput } from "../types/cli"; -import { Logger } from "../utils/logger"; - -/** - * Configures the command-line options for the Conductor CLI - * @param program - The Commander.js program instance - */ -export function configureCommandOptions(program: Command): void { - // Global options - program - .version("1.0.0") - .description("Conductor: Data Processing Pipeline") - .option("--debug", "Enable debug mode") - // Add a custom action for the help option - .addHelpCommand("help [command]", "Display help for a specific command") - .on("--help", () => { - // Call the reference commands after the default help - Logger.showReferenceCommands(); - }); - - // Upload command - program - .command("upload") - .description("Upload data to Elasticsearch") - .option("-f, --file ", "Input files to process") - .option("-i, --index ", "Elasticsearch index name") - .option("-b, --batch-size ", "Batch size for uploads") - .option("--delimiter ", "CSV delimiter character") - .option("-o, --output ", "Output directory for generated files") - .option("--force", "Force overwrite of existing files") - .option("--url ", "Elasticsearch URL") - .option("--user ", "Elasticsearch username", "elastic") - .option( - "--password ", - "Elasticsearch password", - "myelasticpassword" - ) - .action(() => { - /* Handled by main.ts */ - }); - - // Lectern schema upload command - program - .command("lecternUpload") - .description("Upload schema to Lectern server") - .option("-s, --schema-file ", "Schema JSON file to upload") - .option( - "-u, --lectern-url ", - "Lectern server URL", - process.env.LECTERN_URL || "http://localhost:3031" - ) - .option("-t, --auth-token ", "Authentication token", "") - .option("-o, --output ", "Output directory for response logs") - .option("--force", "Force overwrite of existing files") - .action(() => { - /* Handled by main.ts */ - }); - - // Lyric dictionary registration command - program - .command("lyricRegister") - .description("Register a dictionary with Lyric service") - .option( - "-u, --lyric-url ", - "Lyric server URL", - process.env.LYRIC_URL || "http://localhost:3030" - ) - .option("-c, --category-name ", "Category name") - .option("--dict-name ", "Dictionary name") - .option("-v, --dictionary-version ", "Dictionary version") - .option("-e, --default-centric-entity ", "Default centric entity") - .option("-o, --output ", "Output directory for response logs") - .option("--force", "Force overwrite of existing files") - .action(() => { - /* Handled by main.ts */ - }); - - // Lyric data loading command - program - .command("lyricUpload") - .description("Load data into Lyric service") - .option( - "-u, --lyric-url ", - "Lyric server URL", - process.env.LYRIC_URL || "http://localhost:3030" - ) - .option( - "-l, --lectern-url ", - "Lectern server URL", - process.env.LECTERN_URL || "http://localhost:3031" - ) - .option( - "-d, --data-directory ", - "Directory containing CSV data files", - process.env.LYRIC_DATA - ) - .option( - "-c, --category-id ", - "Category ID", - process.env.CATEGORY_ID || "1" - ) - .option( - "-g, --organization ", - "Organization name", - process.env.ORGANIZATION || "OICR" - ) - .option( - "-m, --max-retries ", - "Maximum number of retry attempts", - process.env.MAX_RETRIES || "10" - ) - .option( - "-r, --retry-delay ", - "Delay between retry attempts in milliseconds", - process.env.RETRY_DELAY || "1000" - ) - .option("-o, --output ", "Output directory for response logs") - .option("--force", "Force overwrite of existing files") - .action(() => { - /* Handled by main.ts */ - }); - - // Repository indexing command - program - .command("maestroIndex") - .description("Index a repository with optional filtering") - .option( - "--index-url ", - "Indexing service URL", - process.env.INDEX_URL || "http://localhost:11235" - ) - .option( - "--repository-code ", - "Repository code to index", - process.env.REPOSITORY_CODE - ) - .option( - "--organization ", - "Organization name filter", - process.env.ORGANIZATION - ) - .option("--id ", "Specific ID to index", process.env.ID) - .option("-o, --output ", "Output directory for response logs") - .option("--force", "Skip confirmation prompts") - .option("--debug", "Enable detailed debug logging") - .action(() => { - /* Handled by main.ts */ - }); - - // SONG schema upload command - program - .command("songUploadSchema") - .description("Upload schema to SONG server") - .option("-s, --schema-file ", "Schema JSON file to upload") - .option( - "-u, --song-url ", - "SONG server URL", - process.env.SONG_URL || "http://localhost:8080" - ) - .option( - "-t, --auth-token ", - "Authentication token", - process.env.AUTH_TOKEN || "123" - ) - .option("-o, --output ", "Output directory for response logs") - .option("--force", "Force overwrite of existing files") - .action(() => { - /* Handled by main.ts */ - }); - - // SONG study creation command - program - .command("songCreateStudy") - .description("Create study in SONG server") - .option( - "-u, --song-url ", - "SONG server URL", - process.env.SONG_URL || "http://localhost:8080" - ) - .option("-i, --study-id ", "Study ID", process.env.STUDY_ID || "demo") - .option( - "-n, --study-name ", - "Study name", - process.env.STUDY_NAME || "string" - ) - .option( - "-g, --organization ", - "Organization name", - process.env.ORGANIZATION || "string" - ) - .option( - "--description ", - "Study description", - process.env.DESCRIPTION || "string" - ) - .option( - "-t, --auth-token ", - "Authentication token", - process.env.AUTH_TOKEN || "123" - ) - .option("-o, --output ", "Output directory for response logs") - .option("--force", "Force creation even if study exists", false) - .action(() => { - /* Handled by main.ts */ - }); - - // SONG analysis submission command (now includes Score file upload) - program - .command("songSubmitAnalysis") - .description("Submit analysis to SONG and upload files to Score") - .option("-a, --analysis-file ", "Analysis JSON file to submit") - .option( - "-u, --song-url ", - "SONG server URL", - process.env.SONG_URL || "http://localhost:8080" - ) - .option( - "-s, --score-url ", - "Score server URL", - process.env.SCORE_URL || "http://localhost:8087" - ) - .option("-i, --study-id ", "Study ID", process.env.STUDY_ID || "demo") - .option("--allow-duplicates", "Allow duplicate analysis submissions", false) - .option( - "-d, --data-dir ", - "Directory containing data files", - process.env.DATA_DIR || "./data" - ) - .option( - "--output-dir ", - "Directory for manifest file output", - process.env.OUTPUT_DIR || "./output" - ) - .option( - "-m, --manifest-file ", - "Path for manifest file", - process.env.MANIFEST_FILE - ) - .option( - "-t, --auth-token ", - "Authentication token", - process.env.AUTH_TOKEN || "123" - ) - .option( - "--ignore-undefined-md5", - "Ignore files with undefined MD5 checksums", - false - ) - .option("-o, --output ", "Output directory for response logs") - .option( - "--force", - "Force studyId from command line instead of from file", - false - ) - .action(() => { - /* Handled by main.ts */ - }); - - // SONG publish analysis command - program - .command("songPublishAnalysis") - .description("Publish analysis in SONG server") - .option("-a, --analysis-id ", "Analysis ID to publish") - .option("-i, --study-id ", "Study ID", process.env.STUDY_ID || "demo") - .option( - "-u, --song-url ", - "SONG server URL", - process.env.SONG_URL || "http://localhost:8080" - ) - .option( - "-t, --auth-token ", - "Authentication token", - process.env.AUTH_TOKEN || "123" - ) - .option( - "--ignore-undefined-md5", - "Ignore files with undefined MD5 checksums", - false - ) - .option("-o, --output ", "Output directory for response logs") - .action(() => { - /* Handled by main.ts */ - }); - - // Note: scoreManifestUpload and songScoreSubmit commands have been removed - // Their functionality is now integrated into songSubmitAnalysis -} - -/** - * Parses command-line arguments into a standardized CLIOutput object - * Updated to handle the combined SONG/Score workflow - * - * @param options - Parsed command-line options - * @returns A CLIOutput object for command execution - */ -export function parseCommandLineArgs(options: any): CLIOutput { - // Log raw options for debugging - Logger.debug(`Raw options: ${JSON.stringify(options)}`); - Logger.debug(`Process argv: ${process.argv.join(" ")}`); - - // Determine the profile from options - let profile = options.profile || Profiles.UPLOAD; - - // Special handling for lyricData command to ensure data directory is captured - if (profile === Profiles.LYRIC_DATA) { - // Check for a positional argument that might be the data directory - const positionalArgs = process.argv - .slice(3) - .filter((arg) => !arg.startsWith("-")); - - if (positionalArgs.length > 0 && !options.dataDirectory) { - options.dataDirectory = positionalArgs[0]; - Logger.debug( - `Captured data directory from positional argument: ${options.dataDirectory}` - ); - } - } - - // Parse file paths - const filePaths = Array.isArray(options.file) - ? options.file - : options.file - ? [options.file] - : []; - - // Add template file to filePaths if present - if (options.templateFile && !filePaths.includes(options.templateFile)) { - filePaths.push(options.templateFile); - } - - // Add schema file to filePaths if present for Lectern or SONG upload - if (options.schemaFile && !filePaths.includes(options.schemaFile)) { - filePaths.push(options.schemaFile); - } - - // Add analysis file to filePaths if present for SONG analysis submission - if (options.analysisFile && !filePaths.includes(options.analysisFile)) { - filePaths.push(options.analysisFile); - } - - Logger.debug(`Parsed profile: ${profile}`); - Logger.debug(`Parsed file paths: ${filePaths.join(", ")}`); - - // Create config object with support for all services - const config = { - elasticsearch: { - url: - options.url || process.env.ELASTICSEARCH_URL || "http://localhost:9200", - user: options.user || process.env.ELASTICSEARCH_USER, - password: options.password || process.env.ELASTICSEARCH_PASSWORD, - index: options.index || options.indexName || "conductor-data", - templateFile: options.templateFile, - templateName: options.templateName, - alias: options.aliasName, - }, - lectern: { - url: - options.lecternUrl || - process.env.LECTERN_URL || - "http://localhost:3031", - authToken: options.authToken || process.env.LECTERN_AUTH_TOKEN || "", - }, - lyric: { - url: options.lyricUrl || process.env.LYRIC_URL || "http://localhost:3030", - categoryName: options.categoryName || process.env.CATEGORY_NAME, - dictionaryName: options.dictName || process.env.DICTIONARY_NAME, - dictionaryVersion: - options.dictionaryVersion || process.env.DICTIONARY_VERSION, - defaultCentricEntity: - options.defaultCentricEntity || process.env.DEFAULT_CENTRIC_ENTITY, - // Data loading specific options - dataDirectory: options.dataDirectory || process.env.LYRIC_DATA, - categoryId: options.categoryId || process.env.CATEGORY_ID, - organization: options.organization || process.env.ORGANIZATION, - maxRetries: options.maxRetries - ? parseInt(options.maxRetries) - : process.env.MAX_RETRIES - ? parseInt(process.env.MAX_RETRIES) - : 10, - retryDelay: options.retryDelay - ? parseInt(options.retryDelay) - : process.env.RETRY_DELAY - ? parseInt(process.env.RETRY_DELAY) - : 20000, - }, - song: { - url: options.songUrl || process.env.SONG_URL || "http://localhost:8080", - authToken: options.authToken || process.env.AUTH_TOKEN || "123", - schemaFile: options.schemaFile || process.env.SONG_SCHEMA, - studyId: options.studyId || process.env.STUDY_ID || "demo", - studyName: options.studyName || process.env.STUDY_NAME || "string", - organization: - options.organization || process.env.ORGANIZATION || "string", - description: options.description || process.env.DESCRIPTION || "string", - analysisFile: options.analysisFile || process.env.ANALYSIS_FILE, - allowDuplicates: - options.allowDuplicates || - process.env.ALLOW_DUPLICATES === "true" || - false, - ignoreUndefinedMd5: - options.ignoreUndefinedMd5 || - process.env.IGNORE_UNDEFINED_MD5 === "true" || - false, - // Combined Score functionality (now part of song config) - scoreUrl: - options.scoreUrl || process.env.SCORE_URL || "http://localhost:8087", - dataDir: options.dataDir || process.env.DATA_DIR || "./data", - outputDir: options.outputDir || process.env.OUTPUT_DIR || "./output", - manifestFile: options.manifestFile || process.env.MANIFEST_FILE, - }, - maestroIndex: { - url: - options.indexUrl || process.env.INDEX_URL || "http://localhost:11235", - repositoryCode: options.repositoryCode || process.env.REPOSITORY_CODE, - organization: options.organization || process.env.ORGANIZATION, - id: options.id || process.env.ID, - }, - batchSize: options.batchSize ? parseInt(options.batchSize, 10) : 1000, - delimiter: options.delimiter || ",", - }; - - // Build the standardized CLI output - return { - profile, - filePaths, - outputPath: options.output, - config, - options, - envConfig: { - elasticsearchUrl: config.elasticsearch.url, - esUser: config.elasticsearch.user, - esPassword: config.elasticsearch.password, - indexName: config.elasticsearch.index, - lecternUrl: config.lectern.url, - lyricUrl: config.lyric.url, - songUrl: config.song.url, - lyricData: config.lyric.dataDirectory, - categoryId: config.lyric.categoryId, - organization: config.lyric.organization, - }, - }; -} diff --git a/apps/conductor/src/cli/serviceConfigManager.ts b/apps/conductor/src/cli/serviceConfigManager.ts new file mode 100644 index 00000000..89fb4a7e --- /dev/null +++ b/apps/conductor/src/cli/serviceConfigManager.ts @@ -0,0 +1,390 @@ +// src/cli/ServiceConfigManager.ts +/** + * Unified service configuration management + * Replaces scattered config objects throughout commands and services + */ + +import { Environment } from "./environment"; +import { ServiceConfig } from "../services/base/types"; +import { ErrorFactory } from "../utils/errors"; // ADDED: Import ErrorFactory + +interface StandardServiceConfig extends ServiceConfig { + name: string; + retries: number; + retryDelay: number; +} + +interface ElasticsearchConfig extends StandardServiceConfig { + user: string; + password: string; + index: string; + batchSize: number; + delimiter: string; +} + +interface FileServiceConfig extends StandardServiceConfig { + dataDir: string; + outputDir: string; + manifestFile?: string; +} + +interface LyricConfig extends StandardServiceConfig { + categoryId: string; + organization: string; + maxRetries: number; + retryDelay: number; +} + +export class ServiceConfigManager { + /** + * Create Elasticsearch configuration + */ + static createElasticsearchConfig( + overrides: Partial = {} + ): ElasticsearchConfig { + const env = Environment.services.elasticsearch; + const defaults = Environment.defaults.elasticsearch; + + return { + name: "Elasticsearch", + url: env.url, + authToken: undefined, // ES uses user/password + timeout: Environment.defaults.timeouts.default, + retries: 3, + retryDelay: 1000, + user: env.user, + password: env.password, + index: defaults.index, + batchSize: defaults.batchSize, + delimiter: defaults.delimiter, + ...overrides, + }; + } + + /** + * Create Lectern service configuration + */ + static createLecternConfig( + overrides: Partial = {} + ): StandardServiceConfig { + const env = Environment.services.lectern; + + return { + name: "Lectern", + url: env.url, + authToken: env.authToken, + timeout: Environment.defaults.timeouts.default, + retries: 3, + retryDelay: 1000, + ...overrides, + }; + } + + /** + * Create Lyric service configuration + */ + static createLyricConfig(overrides: Partial = {}): LyricConfig { + const env = Environment.services.lyric; + const defaults = Environment.defaults.lyric; + + return { + name: "Lyric", + url: env.url, + authToken: undefined, + timeout: Environment.defaults.timeouts.upload, // Longer timeout for uploads + retries: 3, + retryDelay: defaults.retryDelay, // Use the environment default + categoryId: env.categoryId, + organization: env.organization, + maxRetries: defaults.maxRetries, + ...overrides, + }; + } + + /** + * Create SONG service configuration + */ + static createSongConfig( + overrides: Partial = {} + ): StandardServiceConfig { + const env = Environment.services.song; + + return { + name: "SONG", + url: env.url, + authToken: env.authToken, + timeout: Environment.defaults.timeouts.upload, + retries: 3, + retryDelay: 1000, + ...overrides, + }; + } + + /** + * Create Score service configuration + */ + static createScoreConfig( + overrides: Partial = {} + ): StandardServiceConfig { + const env = Environment.services.score; + + return { + name: "Score", + url: env.url, + authToken: env.authToken, + timeout: Environment.defaults.timeouts.upload, + retries: 2, // Lower retries for file uploads + retryDelay: 2000, + ...overrides, + }; + } + + /** + * Create Maestro service configuration + */ + static createMaestroConfig( + overrides: Partial = {} + ): StandardServiceConfig { + const env = Environment.services.maestro; + + return { + name: "Maestro", + url: env.url, + authToken: undefined, + timeout: Environment.defaults.timeouts.default, + retries: 3, + retryDelay: 1000, + ...overrides, + }; + } + + /** + * Create file service configuration (for commands that handle files) + */ + static createFileServiceConfig( + baseConfig: StandardServiceConfig, + fileOptions: Partial = {} + ): FileServiceConfig { + return { + ...baseConfig, + dataDir: fileOptions.dataDir || "./data", + outputDir: fileOptions.outputDir || "./output", + manifestFile: fileOptions.manifestFile, + ...fileOptions, + }; + } + + /** + * Validate service configuration + * UPDATED: Enhanced with ErrorFactory for better error messages + */ + static validateConfig(config: StandardServiceConfig): void { + if (!config.url) { + // UPDATED: Use ErrorFactory instead of generic Error + throw ErrorFactory.config( + `Missing URL for ${config.name} service`, + "serviceUrl", + [ + `Set ${config.name.toUpperCase()}_URL environment variable`, + `Use --${config.name.toLowerCase()}-url parameter`, + "Verify service is running and accessible", + "Check network connectivity", + `Example: export ${config.name.toUpperCase()}_URL=http://localhost:8080`, + ] + ); + } + + if (config.timeout && config.timeout < 1000) { + // UPDATED: Use ErrorFactory instead of generic Error + throw ErrorFactory.config( + `Timeout too low for ${config.name} service (minimum 1000ms)`, + "timeout", + [ + "Set timeout to 1000ms or higher", + "Use reasonable timeout values (5000-30000ms recommended)", + "Consider network latency and service response times", + `Example: --timeout 10000 for ${config.name}`, + ] + ); + } + + if (config.retries && config.retries < 0) { + // UPDATED: Use ErrorFactory instead of generic Error + throw ErrorFactory.config( + `Invalid retries value for ${config.name} service`, + "retries", + [ + "Use a positive integer for retries (0-10 recommended)", + "Set retries to 0 to disable retry logic", + "Consider service reliability when setting retry count", + `Example: --retries 3 for ${config.name}`, + ] + ); + } + + // Additional validation for URL format + if (config.url) { + try { + const parsedUrl = new URL(config.url); + if (!["http:", "https:"].includes(parsedUrl.protocol)) { + throw ErrorFactory.config( + `Invalid ${config.name} URL protocol - must be http or https`, + "serviceUrl", + [ + "Use http:// or https:// protocol", + `Check URL format: http://localhost:8080`, + "Verify the service URL is correct", + "Include protocol in the URL", + ] + ); + } + } catch (error) { + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + throw ErrorFactory.config( + `Invalid ${config.name} URL format: ${config.url}`, + "serviceUrl", + [ + "Use a valid URL format: http://localhost:8080", + "Include protocol (http:// or https://)", + "Check for typos in the URL", + "Verify port number is correct", + "Ensure no extra spaces or characters", + ] + ); + } + } + } + + /** + * Validate Elasticsearch-specific configuration + * ADDED: New validation method for ES-specific settings + */ + static validateElasticsearchConfig(config: ElasticsearchConfig): void { + this.validateConfig(config); + + if ( + config.batchSize && + (config.batchSize < 1 || config.batchSize > 10000) + ) { + throw ErrorFactory.config( + `Invalid batch size for Elasticsearch: ${config.batchSize}`, + "batchSize", + [ + "Use batch size between 1 and 10000", + "Recommended values: 500-2000 for most files", + "Smaller batches for large documents, larger for simple data", + "Example: --batch-size 1000", + ] + ); + } + + if (config.delimiter && config.delimiter.length !== 1) { + throw ErrorFactory.config( + `Invalid CSV delimiter: ${config.delimiter}`, + "delimiter", + [ + "Use a single character delimiter", + "Common delimiters: ',' (comma), '\\t' (tab), ';' (semicolon)", + "Example: --delimiter ';'", + "Ensure delimiter matches your CSV file format", + ] + ); + } + + if (!config.index || config.index.trim() === "") { + throw ErrorFactory.config( + "Elasticsearch index name is required", + "index", + [ + "Provide an index name with --index parameter", + "Use lowercase names with hyphens or underscores", + "Example: --index my-data-index", + "Ensure index exists in Elasticsearch", + ] + ); + } + } + + /** + * Validate Lyric-specific configuration + * ADDED: New validation method for Lyric-specific settings + */ + static validateLyricConfig(config: LyricConfig): void { + this.validateConfig(config); + + if ( + config.maxRetries && + (config.maxRetries < 1 || config.maxRetries > 50) + ) { + throw ErrorFactory.config( + `Invalid max retries for Lyric: ${config.maxRetries}`, + "maxRetries", + [ + "Use max retries between 1 and 50", + "Recommended: 5-15 for most use cases", + "Higher values for unstable connections", + "Example: --max-retries 10", + ] + ); + } + + if ( + config.retryDelay && + (config.retryDelay < 1000 || config.retryDelay > 300000) + ) { + throw ErrorFactory.config( + `Invalid retry delay for Lyric: ${config.retryDelay}ms`, + "retryDelay", + [ + "Use retry delay between 1000ms (1s) and 300000ms (5min)", + "Recommended: 10000-30000ms for most use cases", + "Longer delays for heavily loaded services", + "Example: --retry-delay 20000", + ] + ); + } + + if (!config.categoryId || config.categoryId.trim() === "") { + throw ErrorFactory.config("Lyric category ID is required", "categoryId", [ + "Provide category ID with --category-id parameter", + "Set CATEGORY_ID environment variable", + "Category ID should match your registered dictionary", + "Contact administrator for valid category IDs", + ]); + } + + if (!config.organization || config.organization.trim() === "") { + throw ErrorFactory.config( + "Lyric organization is required", + "organization", + [ + "Provide organization with --organization parameter", + "Set ORGANIZATION environment variable", + "Use your institution or organization name", + "Organization should match your Lyric configuration", + ] + ); + } + } + + /** + * Get all configured services status + */ + static getServicesOverview() { + const env = Environment.services; + return { + elasticsearch: { + url: env.elasticsearch.url, + configured: !!env.elasticsearch.url, + }, + lectern: { url: env.lectern.url, configured: !!env.lectern.url }, + lyric: { url: env.lyric.url, configured: !!env.lyric.url }, + song: { url: env.song.url, configured: !!env.song.url }, + score: { url: env.score.url, configured: !!env.score.url }, + maestro: { url: env.maestro.url, configured: !!env.maestro.url }, + }; + } +} diff --git a/apps/conductor/src/commands/baseCommand.ts b/apps/conductor/src/commands/baseCommand.ts index e739afc2..965c8fff 100644 --- a/apps/conductor/src/commands/baseCommand.ts +++ b/apps/conductor/src/commands/baseCommand.ts @@ -3,6 +3,8 @@ * * Provides the base abstract class and interfaces for all command implementations. * Commands follow the Command Pattern for encapsulating operations. + * Enhanced with ErrorFactory patterns for consistent error handling. + * Updated to use centralized file utilities. */ import { CLIOutput } from "../types/cli"; @@ -10,28 +12,14 @@ import * as fs from "fs"; import * as path from "path"; import * as readline from "readline"; import { Logger } from "../utils/logger"; -import { ConductorError, ErrorCodes } from "../utils/errors"; - -/** - * Command execution result - */ -export interface CommandResult { - /** Whether the command succeeded */ - success: boolean; - - /** Optional error message if the command failed */ - errorMessage?: string; - - /** Optional error code if the command failed */ - errorCode?: string; - - /** Additional result details */ - details?: Record; -} +import { ErrorFactory } from "../utils/errors"; +import { validateFileAccess } from "../utils/fileUtils"; /** * Abstract base class for all CLI commands in the conductor service. * Provides common functionality for command execution, validation, and file handling. + * Enhanced with ErrorFactory patterns for better error messages and user guidance. + * Updated to match composer pattern - throws errors instead of returning CommandResult. */ export abstract class Command { /** Default directory where output files will be stored if not specified by user */ @@ -53,176 +41,114 @@ export abstract class Command { /** * Main method to run the command with the provided CLI arguments. * Handles validation, output path resolution, and error handling. + * Updated to throw errors directly like composer instead of returning CommandResult. * * @param cliOutput - The parsed command line arguments - * @returns A promise that resolves to a CommandResult object + * @returns A promise that resolves when command execution is complete */ - async run(cliOutput: CLIOutput): Promise { + async run(cliOutput: CLIOutput): Promise { const startTime = Date.now(); - try { - // Enable debug logging if requested - if (cliOutput.debug) { - Logger.enableDebug(); - Logger.debug(`Running ${this.name} command with debug enabled`); - } + // Enable debug logging if requested + if (cliOutput.debug) { + Logger.enableDebug(); + Logger.debugString(`Running ${this.name} command with debug enabled`); + } - // Validate input arguments - directly throws errors - try { - await this.validate(cliOutput); - } catch (validationError) { - Logger.debug(`Validation error: ${validationError}`); - if (validationError instanceof Error) { - throw validationError; - } - throw new ConductorError( - String(validationError), - ErrorCodes.VALIDATION_FAILED - ); - } + Logger.header(`♫ Running ${this.name} Command`); - Logger.debug(`Output path before check: ${cliOutput.outputPath}`); + // Enhanced validation with ErrorFactory + await this.validate(cliOutput); - let usingDefaultPath = false; + Logger.debugString(`Output path before check: ${cliOutput.outputPath}`); - // If no output path specified, use the default - if (!cliOutput.outputPath?.trim()) { - Logger.debug("No output directory specified."); - usingDefaultPath = true; - cliOutput.outputPath = path.join(this.defaultOutputPath); - } + let usingDefaultPath = false; - const isDefaultPath = this.isUsingDefaultPath(cliOutput); + // If no output path specified, use the default + if (!cliOutput.outputPath?.trim()) { + Logger.debugString("No output directory specified. Using default."); + usingDefaultPath = true; + cliOutput.outputPath = path.join(this.defaultOutputPath); + } - // Inform user about output path - if (isDefaultPath || usingDefaultPath) { - Logger.info( - `Using default output path: ${cliOutput.outputPath}`, - "Use -o or --output to specify a different location" - ); - } else { - Logger.info(`Output directory set to: ${cliOutput.outputPath}`); - } + const isDefaultPath = this.isUsingDefaultPath(cliOutput); - // Check for existing files and confirm overwrite if needed - // Skip confirmation if force flag is set in options - const forceFlag = cliOutput.options?.force === true; - if (cliOutput.outputPath && !forceFlag) { - const shouldContinue = await this.checkForExistingFiles( - cliOutput.outputPath - ); - if (!shouldContinue) { - Logger.info("Operation cancelled by user."); - return { - success: false, - errorMessage: "Operation cancelled by user", - errorCode: "USER_CANCELLED", - }; - } - } else if (forceFlag) { - Logger.debug("Force flag enabled, skipping overwrite confirmation"); - } + // Inform user about output path + if (isDefaultPath || usingDefaultPath) { + Logger.warn`Using default output path: ${cliOutput.outputPath}`; + Logger.tipString( + "Use -o or --output to specify a different location" + ); + } else { + Logger.info`Output directory set to: ${cliOutput.outputPath}`; + } - Logger.info(`Starting execution of ${this.name} command`); + // Check for existing files and confirm overwrite if needed + // Skip confirmation if force flag is set in options + const forceFlag = cliOutput.options?.force === true; + if (cliOutput.outputPath && !forceFlag) { + const shouldContinue = await this.checkForExistingFiles( + cliOutput.outputPath + ); + if (!shouldContinue) { + Logger.infoString("Operation cancelled by user."); + // Throw error instead of returning result + throw ErrorFactory.validation("Operation cancelled by user", { + command: this.name, + }); + } + } else if (forceFlag) { + Logger.debugString("Force flag enabled, skipping overwrite confirmation"); + } - // Execute the specific command implementation - const result = await this.execute(cliOutput); + Logger.debug`Starting execution of ${this.name} command`; - // Calculate and log execution time - const endTime = Date.now(); - const executionTime = (endTime - startTime) / 1000; + // Execute the specific command implementation + await this.execute(cliOutput); - if (result.success) { - Logger.info( - `${ - this.name - } command completed successfully in ${executionTime.toFixed(2)}s` - ); - } else { - Logger.debug( - `${this.name} command failed after ${executionTime.toFixed(2)}s: ${ - result.errorMessage - }` - ); - } + // Calculate and log execution time + const endTime = Date.now(); + const executionTime = (endTime - startTime) / 1000; - return result; - } catch (error: unknown) { - // Use Logger instead of console.error - Logger.debug(`ERROR IN ${this.name} COMMAND:`, error); - - const errorMessage = - error instanceof Error ? error.message : String(error); - Logger.debug(`Unexpected error in ${this.name} command: ${errorMessage}`); - - return { - success: false, - errorMessage, - errorCode: - error instanceof ConductorError - ? error.code - : ErrorCodes.UNKNOWN_ERROR, - details: { - error, - stack: error instanceof Error ? error.stack : undefined, - }, - }; - } + Logger.debug`${ + this.name + } baseCommand: command completed successfully in ${executionTime.toFixed( + 2 + )}s`; } /** * Abstract method that must be implemented by derived classes. * Contains the specific logic for each command. + * Updated to throw errors directly instead of returning CommandResult. * * @param cliOutput - The parsed command line arguments - * @returns A promise that resolves to a CommandResult + * @returns A promise that resolves when execution is complete */ - protected abstract execute(cliOutput: CLIOutput): Promise; + protected abstract execute(cliOutput: CLIOutput): Promise; /** * Validates command line arguments. * This base implementation checks for required input files. * Derived classes should override to add additional validation. + * Enhanced with ErrorFactory for better error messages. * * @param cliOutput - The parsed command line arguments - * @throws ConductorError if validation fails + * @throws Enhanced ConductorError if validation fails */ protected async validate(cliOutput: CLIOutput): Promise { if (!cliOutput.filePaths?.length) { - throw new ConductorError( - "No input files provided", - ErrorCodes.INVALID_ARGS - ); + throw ErrorFactory.args("No input files provided", this.name, [ + "Provide input files with -f or --file parameter", + "Example: conductor upload -f data.csv", + "Use wildcards for multiple files: -f *.csv", + "Specify multiple files: -f file1.csv file2.csv", + ]); } - // Validate each input file exists + // Enhanced file validation with detailed feedback using centralized utilities for (const filePath of cliOutput.filePaths) { - if (!fs.existsSync(filePath)) { - throw new ConductorError( - `Input file not found: ${filePath}`, - ErrorCodes.FILE_NOT_FOUND - ); - } - - // Check if file is readable - try { - fs.accessSync(filePath, fs.constants.R_OK); - } catch (error) { - throw new ConductorError( - `File '${filePath}' is not readable`, - ErrorCodes.INVALID_FILE, - error - ); - } - - // Check if file has content - const stats = fs.statSync(filePath); - if (stats.size === 0) { - throw new ConductorError( - `File '${filePath}' is empty`, - ErrorCodes.INVALID_FILE - ); - } + validateFileAccess(filePath, "input file"); } } @@ -230,72 +156,86 @@ export abstract class Command { * Checks if the current output path is the default one. * * @param cliOutput - The parsed command line arguments - * @returns true if using the default output path, false otherwise + * @returns True if using default output path */ protected isUsingDefaultPath(cliOutput: CLIOutput): boolean { - return ( - cliOutput.outputPath === this.defaultOutputPath || - cliOutput.outputPath === - path.join(this.defaultOutputPath, this.defaultOutputFileName) - ); + if (!cliOutput.outputPath) return true; + const normalizedOutput = path.normalize(cliOutput.outputPath); + const normalizedDefault = path.normalize(this.defaultOutputPath); + return normalizedOutput === normalizedDefault; } /** - * Creates a directory if it doesn't already exist. + * Enhanced method to check for existing files in the output directory + * and prompt user for confirmation if files would be overwritten. * - * @param dirPath - Path to the directory to create + * @param directoryPath - Path to the output directory + * @param outputFileName - Optional specific filename to check + * @returns Promise resolving to true if user confirms or no conflicts exist */ - protected createDirectoryIfNotExists(dirPath: string): void { - if (!fs.existsSync(dirPath)) { - fs.mkdirSync(dirPath, { recursive: true }); - Logger.info(`Created directory: ${dirPath}`); + protected async checkForExistingFiles( + directoryPath: string, + outputFileName?: string + ): Promise { + // Create directory if it doesn't exist + if (!fs.existsSync(directoryPath)) { + try { + fs.mkdirSync(directoryPath, { recursive: true }); + Logger.debug`Created output directory: ${directoryPath}`; + return true; // No existing files to worry about + } catch (error) { + throw ErrorFactory.file( + `Cannot create output directory: ${path.basename(directoryPath)}`, + directoryPath, + [ + "Check directory permissions", + "Ensure parent directory is writable", + "Verify disk space is available", + "Try using a different output directory", + ] + ); + } } - } - /** - * Checks if files in the output directory would be overwritten. - * Prompts the user for confirmation if files would be overwritten. - * - * @param outputPath - Path where output files will be written - * @returns A promise that resolves to true if execution should continue, false otherwise - */ - protected async checkForExistingFiles(outputPath: string): Promise { - let directoryPath = outputPath; - let outputFileName: string | undefined; - - // Determine if outputPath is a file or directory - if (path.extname(outputPath)) { - Logger.debug(`Output path appears to be a file: ${outputPath}`); - directoryPath = path.dirname(outputPath); - outputFileName = path.basename(outputPath); - Logger.debug( - `Using directory: ${directoryPath}, fileName: ${outputFileName}` + // Get existing files in directory + let existingEntries: string[]; + try { + existingEntries = fs.existsSync(directoryPath) + ? fs.readdirSync(directoryPath) + : []; + } catch (error) { + throw ErrorFactory.file( + `Cannot read output directory: ${path.basename(directoryPath)}`, + directoryPath, + [ + "Check directory permissions", + "Ensure directory is accessible", + "Verify directory is not corrupted", + "Try using a different output directory", + ] ); } - // Create the output directory if it doesn't exist - this.createDirectoryIfNotExists(directoryPath); - - // Get existing entries in the directory - const existingEntries = fs.existsSync(directoryPath) - ? fs.readdirSync(directoryPath) - : []; - // Filter existing files that would be overwritten const filesToOverwrite = existingEntries.filter((entry) => { const fullPath = path.join(directoryPath, entry); - // If specific file name is given, only check that exact file - if (outputFileName) { - return entry === outputFileName && fs.statSync(fullPath).isFile(); - } + try { + // If specific file name is given, only check that exact file + if (outputFileName) { + return entry === outputFileName && fs.statSync(fullPath).isFile(); + } - // If no specific file name, check if entry is a file and would match generated output - return ( - fs.statSync(fullPath).isFile() && - (entry.endsWith(".json") || - entry.startsWith(this.defaultOutputFileName.split(".")[0])) - ); + // If no specific file name, check if entry is a file and would match generated output + return ( + fs.statSync(fullPath).isFile() && + (entry.endsWith(".json") || + entry.startsWith(this.defaultOutputFileName.split(".")[0])) + ); + } catch (error) { + // Skip entries we can't stat + return false; + } }); // If no files would be overwritten, continue without prompting @@ -304,10 +244,8 @@ export abstract class Command { } // Display list of files that would be overwritten - Logger.info( - "The following file(s) in the output directory will be overwritten:" - ); - filesToOverwrite.forEach((file) => Logger.info(`- ${file}`)); + Logger.info`The following file(s) in the output directory will be overwritten:`; + filesToOverwrite.forEach((file) => Logger.info`- ${file}`); // Create readline interface for user input const rl = readline.createInterface({ @@ -330,6 +268,49 @@ export abstract class Command { * @param filePath - Path to the generated file */ protected logGeneratedFile(filePath: string): void { - Logger.info(`Generated file: ${filePath}`); + Logger.info`Generated file: ${filePath}`; + } + + /** + * Enhanced utility method for validating required parameters + */ + protected validateRequired( + params: Record, + requiredFields: string[], + context?: string + ): void { + const missingFields = requiredFields.filter( + (field) => + params[field] === undefined || + params[field] === null || + params[field] === "" + ); + + if (missingFields.length > 0) { + const contextMsg = context ? ` for ${context}` : ""; + + throw ErrorFactory.validation( + `Missing required parameters${contextMsg}`, + { + missingFields, + provided: Object.keys(params), + context, + command: this.name, + }, + [ + `Provide values for: ${missingFields.join(", ")}`, + "Check command line arguments and options", + "Verify all required parameters are included", + `Use 'conductor ${this.name} --help' for parameter information`, + ] + ); + } + } + + /** + * Enhanced utility method for validating file existence - now uses centralized utils + */ + protected validateFileExists(filePath: string, fileType?: string): void { + validateFileAccess(filePath, fileType); } } diff --git a/apps/conductor/src/commands/commandRegistry.ts b/apps/conductor/src/commands/commandRegistry.ts index 1fc5594d..cc290211 100644 --- a/apps/conductor/src/commands/commandRegistry.ts +++ b/apps/conductor/src/commands/commandRegistry.ts @@ -1,11 +1,9 @@ -// src/commands/CommandRegistry.ts -/** - * Simplified command registry to replace the complex factory pattern - * Much cleaner than the current commandFactory.ts approach - */ +// src/commands/CommandRegistry.ts - Enhanced with ErrorFactory patterns import { Command } from "./baseCommand"; import { Logger } from "../utils/logger"; +import { ErrorFactory } from "../utils/errors"; +import { CLIOutput } from "../types/cli"; // Import all command classes import { UploadCommand } from "./uploadCsvCommand"; @@ -30,6 +28,7 @@ interface CommandInfo { /** * Registry of all available commands with metadata + * Enhanced with ErrorFactory for better error handling */ export class CommandRegistry { private static commands = new Map([ @@ -116,21 +115,81 @@ export class CommandRegistry { ], ]); + /** + * Execute a command by name (like composer) + * Enhanced with ErrorFactory for better error messages + */ + static async execute( + commandName: string, + cliOutput: CLIOutput + ): Promise { + Logger.debugString(`Executing command: ${commandName}`); + + // Create and run the command + const command = this.createCommand(commandName); + + // The command will throw errors directly if it fails (like composer) + // Otherwise it completes successfully + await command.run(cliOutput); + + // If we get here, command succeeded + Logger.debug`Command '${commandName}' completed successfully`; + } + /** * Create a command instance by name + * Enhanced with ErrorFactory for better error messages */ static createCommand(commandName: string): Command { + if (!commandName || typeof commandName !== "string") { + throw ErrorFactory.args("Command name is required", undefined, [ + "Provide a valid command name", + "Use 'conductor --help' to see available commands", + "Check command spelling and syntax", + ]); + } + const commandInfo = this.commands.get(commandName); if (!commandInfo) { const availableCommands = Array.from(this.commands.keys()).join(", "); - throw new Error( - `Unknown command: ${commandName}. Available commands: ${availableCommands}` + const similarCommands = this.findSimilarCommands(commandName); + + const suggestions = [ + `Available commands: ${availableCommands}`, + "Use 'conductor --help' for command documentation", + "Check command spelling and syntax", + ]; + + if (similarCommands.length > 0) { + suggestions.unshift(`Did you mean: ${similarCommands.join(", ")}?`); + } + + throw ErrorFactory.args( + `Unknown command: ${commandName}`, + commandName, + suggestions ); } - Logger.debug(`Creating command: ${commandInfo.name}`); - return new commandInfo.constructor(); + try { + Logger.debugString(`Creating command: ${commandInfo.name}`); + return new commandInfo.constructor(); + } catch (error) { + throw ErrorFactory.validation( + `Failed to create command '${commandName}'`, + { + commandName, + error: error instanceof Error ? error.message : String(error), + }, + [ + "Command may have initialization issues", + "Check system requirements and dependencies", + "Try restarting the application", + "Contact support if the problem persists", + ] + ); + } } /** @@ -184,32 +243,63 @@ export class CommandRegistry { } Logger.generic(""); } + + Logger.generic("For detailed command help:"); + Logger.generic(" conductor --help"); + Logger.generic(""); + Logger.generic("For general options:"); + Logger.generic(" conductor --help"); } /** * Display help for a specific command + * Enhanced with ErrorFactory for unknown commands */ static displayCommandHelp(commandName: string): void { + if (!commandName) { + throw ErrorFactory.args("Command name required for help", undefined, [ + "Specify a command to get help for", + "Example: conductor upload --help", + "Use 'conductor --help' for general help", + ]); + } + const commandInfo = this.getCommandInfo(commandName); if (!commandInfo) { - Logger.error(`Unknown command: ${commandName}`); - this.displayHelp(); - return; + const availableCommands = this.getCommandNames().join(", "); + const similarCommands = this.findSimilarCommands(commandName); + + const suggestions = [ + `Available commands: ${availableCommands}`, + "Use 'conductor --help' for all commands", + "Check command spelling", + ]; + + if (similarCommands.length > 0) { + suggestions.unshift(`Did you mean: ${similarCommands.join(", ")}?`); + } + + throw ErrorFactory.args( + `Unknown command: ${commandName}`, + commandName, + suggestions + ); } Logger.header(`Command: ${commandInfo.name}`); - Logger.info(commandInfo.description); - Logger.info(`Category: ${commandInfo.category}`); + Logger.info`${commandInfo.description}`; + Logger.info`Category: ${commandInfo.category}`; // You could extend this to show command-specific options - Logger.tip( + Logger.tipString( `Use 'conductor ${commandName} --help' for command-specific options` ); } /** * Register a new command (useful for plugins or extensions) + * Enhanced with validation */ static registerCommand( name: string, @@ -217,8 +307,58 @@ export class CommandRegistry { category: string, constructor: CommandConstructor ): void { + if (!name || typeof name !== "string") { + throw ErrorFactory.args( + "Valid command name required for registration", + undefined, + [ + "Provide a non-empty string as command name", + "Use lowercase names with hyphens for consistency", + "Example: 'my-custom-command'", + ] + ); + } + + if (!description || typeof description !== "string") { + throw ErrorFactory.args( + "Command description required for registration", + undefined, + [ + "Provide a descriptive string for the command", + "Describe what the command does briefly", + "Example: 'Upload data to custom service'", + ] + ); + } + + if (!category || typeof category !== "string") { + throw ErrorFactory.args( + "Command category required for registration", + undefined, + [ + "Provide a category for organizing commands", + "Use existing categories or create meaningful new ones", + "Examples: 'Data Upload', 'Schema Management'", + ] + ); + } + + if (!constructor || typeof constructor !== "function") { + throw ErrorFactory.validation( + "Valid command constructor required for registration", + { name, constructor: typeof constructor }, + [ + "Provide a class constructor that extends Command", + "Ensure the constructor is properly imported", + "Check that the command class is valid", + ] + ); + } + if (this.commands.has(name)) { - Logger.warn(`Command '${name}' is already registered. Overwriting.`); + Logger.warnString( + `Command '${name}' is already registered. Overwriting.` + ); } this.commands.set(name, { @@ -228,13 +368,138 @@ export class CommandRegistry { constructor, }); - Logger.debug(`Registered command: ${name}`); + Logger.debugString(`Registered command: ${name}`); } /** * Unregister a command */ static unregisterCommand(name: string): boolean { - return this.commands.delete(name); + if (!name || typeof name !== "string") { + throw ErrorFactory.args( + "Valid command name required for unregistration", + undefined, + [ + "Provide the name of the command to unregister", + "Check the command name spelling", + "Use getCommandNames() to see registered commands", + ] + ); + } + + const wasRemoved = this.commands.delete(name); + + if (wasRemoved) { + Logger.debugString(`Unregistered command: ${name}`); + } else { + Logger.warnString( + `Command '${name}' was not registered, nothing to unregister` + ); + } + + return wasRemoved; + } + + /** + * Enhanced command validation + */ + static validateCommandName(commandName: string): boolean { + if (!commandName || typeof commandName !== "string") { + return false; + } + + // Check for valid command name format + if (!/^[a-zA-Z][a-zA-Z0-9-]*$/.test(commandName)) { + return false; + } + + return true; + } + + /** + * Find commands with similar names (for typo suggestions) + */ + private static findSimilarCommands(commandName: string): string[] { + const allCommands = this.getCommandNames(); + const similar: string[] = []; + + for (const command of allCommands) { + // Simple similarity check - starts with same letters or contains the input + if ( + command.toLowerCase().startsWith(commandName.toLowerCase()) || + command.toLowerCase().includes(commandName.toLowerCase()) || + commandName.toLowerCase().includes(command.toLowerCase()) + ) { + similar.push(command); + } + } + + return similar.slice(0, 3); // Return max 3 suggestions + } + + /** + * Get command statistics + */ + static getStats(): { + totalCommands: number; + categoryCounts: Record; + commandsByCategory: Map; + } { + const categories = this.getCommandsByCategory(); + const categoryCounts: Record = {}; + + for (const [category, commands] of categories) { + categoryCounts[category] = commands.length; + } + + return { + totalCommands: this.commands.size, + categoryCounts, + commandsByCategory: categories, + }; + } + + /** + * Validate all registered commands (useful for testing) + */ + static validateAllCommands(): { + valid: boolean; + errors: string[]; + } { + const errors: string[] = []; + + for (const [name, info] of this.commands) { + try { + // Basic validation + if (!this.validateCommandName(name)) { + errors.push(`Invalid command name format: ${name}`); + } + + if (!info.description || info.description.trim().length === 0) { + errors.push(`Command '${name}' missing description`); + } + + if (!info.category || info.category.trim().length === 0) { + errors.push(`Command '${name}' missing category`); + } + + // Try to instantiate (this might catch constructor issues) + const instance = new info.constructor(); + if (!(instance instanceof Command)) { + errors.push(`Command '${name}' constructor does not extend Command`); + } + } catch (error) { + errors.push( + `Command '${name}' validation failed: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + + return { + valid: errors.length === 0, + errors, + }; } } diff --git a/apps/conductor/src/commands/lecternUploadCommand.ts b/apps/conductor/src/commands/lecternUploadCommand.ts index cf402863..7404f98b 100644 --- a/apps/conductor/src/commands/lecternUploadCommand.ts +++ b/apps/conductor/src/commands/lecternUploadCommand.ts @@ -1,17 +1,18 @@ -// src/commands/lecternUploadCommand.ts - Updated to use new configuration system -import { Command, CommandResult } from "./baseCommand"; +// src/commands/lecternUploadCommand.ts - Enhanced with ErrorFactory patterns +import { Command } from "./baseCommand"; import { CLIOutput } from "../types/cli"; import { Logger } from "../utils/logger"; import chalk from "chalk"; -import { ConductorError, ErrorCodes } from "../utils/errors"; +import { ErrorFactory } from "../utils/errors"; import { LecternService } from "../services/lectern"; import { LecternSchemaUploadParams } from "../services/lectern/types"; -import { ServiceConfigManager } from "../config/serviceConfigManager"; +import { ServiceConfigManager } from "../cli/serviceConfigManager"; import * as fs from "fs"; +import * as path from "path"; /** * Command for uploading schemas to the Lectern service - * Now uses the simplified configuration system! + * Enhanced with ErrorFactory patterns and improved user feedback */ export class LecternUploadCommand extends Command { constructor() { @@ -24,170 +25,372 @@ export class LecternUploadCommand extends Command { protected async validate(cliOutput: CLIOutput): Promise { const { options } = cliOutput; - // Get schema file from various sources + // Enhanced schema file validation const schemaFile = this.getSchemaFile(options); if (!schemaFile) { - throw new ConductorError( - "Schema file not specified. Use --schema-file or set LECTERN_SCHEMA environment variable.", - ErrorCodes.INVALID_ARGS + throw ErrorFactory.args( + "Schema file not specified for Lectern upload", + "lecternUpload", + [ + "Provide a schema file: conductor lecternUpload --schema-file dictionary.json", + "Set LECTERN_SCHEMA environment variable", + "Use -s or --schema-file parameter", + "Ensure the file contains a valid Lectern dictionary schema", + ] ); } - // Validate file exists and is readable - if (!fs.existsSync(schemaFile)) { - throw new ConductorError( - `Schema file not found: ${schemaFile}`, - ErrorCodes.FILE_NOT_FOUND + Logger.debug`Validating schema file: ${schemaFile}`; + + // Enhanced file validation + this.validateSchemaFile(schemaFile); + + // Enhanced service URL validation + const lecternUrl = options.lecternUrl || process.env.LECTERN_URL; + if (!lecternUrl) { + throw ErrorFactory.config( + "Lectern service URL not configured", + "lecternUrl", + [ + "Set Lectern URL: conductor lecternUpload --lectern-url http://localhost:3031", + "Set LECTERN_URL environment variable", + "Verify Lectern service is running and accessible", + "Check network connectivity to Lectern service", + ] ); } + + Logger.debug`Using Lectern URL: ${lecternUrl}`; } /** - * Executes the Lectern schema upload process - * Much simpler now with the new configuration system! + * Executes the Lectern schema upload process with enhanced error handling */ - protected async execute(cliOutput: CLIOutput): Promise { + protected async execute(cliOutput: CLIOutput): Promise { const { options } = cliOutput; - try { - // Extract configuration using the new simplified system - const schemaFile = this.getSchemaFile(options)!; - - // Use the new ServiceConfigManager - much cleaner! - const serviceConfig = ServiceConfigManager.createLecternConfig({ - url: options.lecternUrl, - authToken: options.authToken, - }); - - // Validate the configuration - ServiceConfigManager.validateConfig(serviceConfig); - - const uploadParams = this.extractUploadParams(schemaFile); - - // Create service instance - const lecternService = new LecternService(serviceConfig); - - // Check service health - const healthResult = await lecternService.checkHealth(); - if (!healthResult.healthy) { - throw new ConductorError( - `Lectern service is not healthy: ${ - healthResult.message || "Unknown error" - }`, - ErrorCodes.CONNECTION_ERROR, - { healthResult } - ); - } + // Extract configuration using the new simplified system + const schemaFile = this.getSchemaFile(options)!; + const fileName = path.basename(schemaFile); - // Log upload info - this.logUploadInfo(schemaFile, serviceConfig.url); + Logger.debug`Starting Lectern schema upload for: ${fileName}`; - // Upload schema - const result = await lecternService.uploadSchema(uploadParams); + // Use the new ServiceConfigManager + const serviceConfig = ServiceConfigManager.createLecternConfig({ + url: options.lecternUrl, + authToken: options.authToken, + }); - // Log success - this.logSuccess(result); + // Validate the configuration + ServiceConfigManager.validateConfig(serviceConfig); - return { - success: true, - details: result, - }; - } catch (error) { - return this.handleExecutionError(error); + // Parse and validate schema content + const uploadParams = this.extractUploadParams(schemaFile); + + // Create service instance with enhanced error handling + const lecternService = new LecternService(serviceConfig); + + // Enhanced health check with specific feedback + Logger.debug`Checking Lectern service health...`; + const healthResult = await lecternService.checkHealth(); + if (!healthResult.healthy) { + throw ErrorFactory.connection( + "Lectern service health check failed", + "Lectern", + serviceConfig.url, + [ + "Check that Lectern service is running", + `Verify service URL: ${serviceConfig.url}`, + "Check network connectivity", + "Review Lectern service logs for errors", + `Test manually: curl ${serviceConfig.url}/health`, + healthResult.message + ? `Health check message: ${healthResult.message}` + : "", + ].filter(Boolean) + ); } + + // Log upload info with enhanced context + this.logUploadInfo(fileName, serviceConfig.url, uploadParams); + + // Upload schema with enhanced error context + Logger.info`Uploading schema to Lectern service...`; + const result = await lecternService.uploadSchema(uploadParams); + + // Enhanced success logging + this.logSuccess(result, fileName); + + // Success - method completes normally } /** - * Get schema file from various sources + * Get schema file from various sources with enhanced validation */ private getSchemaFile(options: any): string | undefined { - return options.schemaFile || process.env.LECTERN_SCHEMA; + const schemaFile = options.schemaFile || process.env.LECTERN_SCHEMA; + + if (schemaFile) { + Logger.debug`Schema file source: ${ + options.schemaFile ? "command line" : "environment variable" + }`; + } + + return schemaFile; } /** - * Extract upload parameters from schema file + * Enhanced schema file validation + */ + private validateSchemaFile(schemaFile: string): void { + const fileName = path.basename(schemaFile); + + // Check file existence + if (!fs.existsSync(schemaFile)) { + throw ErrorFactory.file( + `Schema file not found: ${fileName}`, + schemaFile, + [ + "Check that the file path is correct", + "Ensure the file exists at the specified location", + "Verify file permissions allow read access", + `Current directory: ${process.cwd()}`, + "Use absolute path if relative path is not working", + ] + ); + } + + // Check file extension + const ext = path.extname(schemaFile).toLowerCase(); + if (ext !== ".json") { + Logger.warn`Schema file extension is '${ext}' (expected '.json')`; + Logger.tipString("Lectern schemas should typically be JSON files"); + } + + // Check file readability + try { + fs.accessSync(schemaFile, fs.constants.R_OK); + } catch (error) { + throw ErrorFactory.file( + `Schema file is not readable: ${fileName}`, + schemaFile, + [ + "Check file permissions", + "Ensure the file is not locked by another process", + "Verify you have read access to the file", + ] + ); + } + + // Check file size + const stats = fs.statSync(schemaFile); + if (stats.size === 0) { + throw ErrorFactory.file(`Schema file is empty: ${fileName}`, schemaFile, [ + "Ensure the file contains a valid schema definition", + "Check if the file was properly created", + "Verify the file is not corrupted", + ]); + } + + if (stats.size > 10 * 1024 * 1024) { + // 10MB + Logger.warn`Schema file is quite large: ${( + stats.size / + 1024 / + 1024 + ).toFixed(1)}MB`; + Logger.tipString( + "Large schema files may take longer to upload and process" + ); + } + } + + /** + * Extract and validate upload parameters from schema file */ private extractUploadParams(schemaFile: string): LecternSchemaUploadParams { + const fileName = path.basename(schemaFile); + try { - Logger.info(`Reading schema file: ${schemaFile}`); + Logger.debug`Reading and parsing schema file: ${fileName}`; const schemaContent = fs.readFileSync(schemaFile, "utf-8"); + // Enhanced JSON validation + let parsedSchema; + try { + parsedSchema = JSON.parse(schemaContent); + } catch (error) { + throw ErrorFactory.file( + `Invalid JSON format in schema file: ${fileName}`, + schemaFile, + [ + "Check JSON syntax for errors (missing commas, brackets, quotes)", + "Validate JSON structure using a JSON validator", + "Ensure file encoding is UTF-8", + "Try viewing the file in a JSON editor", + error instanceof Error ? `JSON error: ${error.message}` : "", + ].filter(Boolean) + ); + } + + // Enhanced schema structure validation + this.validateSchemaStructure(parsedSchema, fileName, schemaFile); + return { schemaContent, }; } catch (error) { - throw new ConductorError( - `Error reading schema file: ${ - error instanceof Error ? error.message : String(error) - }`, - ErrorCodes.FILE_ERROR, - error + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + throw ErrorFactory.file( + `Error reading schema file: ${fileName}`, + schemaFile, + [ + "Check file permissions and accessibility", + "Verify file is not corrupted", + "Ensure file encoding is UTF-8", + "Try opening the file manually to inspect content", + ] + ); + } + } + + /** + * Enhanced schema structure validation + */ + private validateSchemaStructure( + schema: any, + fileName: string, + filePath: string + ): void { + if (!schema || typeof schema !== "object") { + throw ErrorFactory.validation( + `Invalid schema structure in ${fileName}`, + { schema, file: filePath }, + [ + "Schema must be a valid JSON object", + "Check that the file contains proper schema definition", + "Ensure the schema follows Lectern format requirements", + "Review Lectern documentation for schema structure", + ] + ); + } + + // Check for required Lectern schema fields + const requiredFields = ["name", "schemas"]; + const missingFields = requiredFields.filter((field) => !schema[field]); + + if (missingFields.length > 0) { + throw ErrorFactory.validation( + `Missing required fields in schema ${fileName}`, + { missingFields, schema, file: filePath }, + [ + `Add missing fields: ${missingFields.join(", ")}`, + "Lectern schemas require 'name' and 'schemas' fields", + "Check schema format against Lectern documentation", + "Ensure all required properties are present", + ] + ); + } + + // Validate schema name + if (typeof schema.name !== "string" || schema.name.trim() === "") { + throw ErrorFactory.validation( + `Invalid schema name in ${fileName}`, + { name: schema.name, file: filePath }, + [ + "Schema 'name' must be a non-empty string", + "Use a descriptive name for the schema", + "Avoid special characters in schema names", + ] ); } + + // Validate schemas array + if (!Array.isArray(schema.schemas)) { + throw ErrorFactory.validation( + `Invalid 'schemas' field in ${fileName}`, + { schemas: schema.schemas, file: filePath }, + [ + "'schemas' field must be an array", + "Include at least one schema definition", + "Check array syntax and structure", + ] + ); + } + + if (schema.schemas.length === 0) { + throw ErrorFactory.validation( + `Empty schemas array in ${fileName}`, + { file: filePath }, + [ + "Include at least one schema definition", + "Add schema objects to the 'schemas' array", + "Check if schemas were properly defined", + ] + ); + } + + Logger.debug`Schema validation passed for ${fileName}: name="${schema.name}", schemas=${schema.schemas.length}`; } /** - * Log upload information + * Enhanced upload information logging */ - private logUploadInfo(schemaFile: string, serviceUrl: string): void { - Logger.info(`${chalk.bold.cyan("Uploading Schema to Lectern:")}`); - Logger.info(`URL: ${serviceUrl}/dictionaries`); - Logger.info(`Schema File: ${schemaFile}`); + private logUploadInfo( + fileName: string, + serviceUrl: string, + params: LecternSchemaUploadParams + ): void { + Logger.generic(`${chalk.bold.cyan("Lectern Schema Upload Details:\n")}`); + Logger.generic(` ▸ File: ${fileName}`); + Logger.debug` ▸ Target: ${serviceUrl}/dictionaries`; + + // Parse schema for additional info + try { + const schema = JSON.parse(params.schemaContent); + Logger.generic(` ▸ Schema Name: ${schema.name || "Unnamed"}`); + Logger.debugString( + ` ▸ Schema Count: ${ + Array.isArray(schema.schemas) ? schema.schemas.length : 0 + }` + ); + + if (schema.version) { + Logger.generic(` ▸ Version: ${schema.version}\n`); + } + } catch (error) { + Logger.debug`Could not parse schema for logging: ${error}`; + } } /** - * Log successful upload + * Enhanced success logging with detailed information */ - private logSuccess(result: any): void { - Logger.success("Schema uploaded successfully"); + private logSuccess(result: any, fileName: string): void { + Logger.success`Schema uploaded successfully to Lectern`; Logger.generic(" "); - Logger.generic(chalk.gray(` - Schema ID: ${result.id || "N/A"}`)); + Logger.generic(chalk.gray(` ✓ File: ${fileName}`)); Logger.generic( - chalk.gray(` - Schema Name: ${result.name || "Unnamed"}`) + chalk.gray(` ✓ Schema ID: ${result.id || "Generated by Lectern"}`) ); Logger.generic( - chalk.gray(` - Schema Version: ${result.version || "N/A"}`) + chalk.gray(` ✓ Schema Name: ${result.name || "As specified in file"}`) + ); + Logger.generic( + chalk.gray(` ✓ Version: ${result.version || "As specified in file"}`) ); - Logger.generic(" "); - } - - /** - * Handle execution errors with helpful user feedback - */ - private handleExecutionError(error: unknown): CommandResult { - if (error instanceof ConductorError) { - // Add context-specific help for common Lectern errors - if (error.code === ErrorCodes.VALIDATION_FAILED) { - Logger.info("\nSchema validation failed. Check your schema structure."); - Logger.tip( - 'Ensure your schema has required fields: "name" and "schema"' - ); - } else if (error.code === ErrorCodes.FILE_NOT_FOUND) { - Logger.info("\nSchema file not found. Check the file path."); - } else if (error.code === ErrorCodes.CONNECTION_ERROR) { - Logger.info("\nConnection error. Check Lectern service availability."); - } - - if (error.details?.suggestion) { - Logger.tip(error.details.suggestion); - } - return { - success: false, - errorMessage: error.message, - errorCode: error.code, - details: error.details, - }; + if (result.created_at) { + Logger.generic(chalk.gray(` ✓ Created: ${result.created_at}`)); } - // Handle unexpected errors - const errorMessage = error instanceof Error ? error.message : String(error); - return { - success: false, - errorMessage: `Schema upload failed: ${errorMessage}`, - errorCode: ErrorCodes.CONNECTION_ERROR, - details: { originalError: error }, - }; + Logger.generic(" "); + Logger.tipString( + "Schema is now available for use in Lectern-compatible services" + ); } } diff --git a/apps/conductor/src/commands/lyricRegistrationCommand.ts b/apps/conductor/src/commands/lyricRegistrationCommand.ts index 1080de91..eb7268a2 100644 --- a/apps/conductor/src/commands/lyricRegistrationCommand.ts +++ b/apps/conductor/src/commands/lyricRegistrationCommand.ts @@ -1,111 +1,308 @@ -// src/commands/lyricRegistrationCommand.ts -import { Command, CommandResult } from "./baseCommand"; +// src/commands/lyricRegistrationCommand.ts - Enhanced with ErrorFactory patterns +import { Command } from "./baseCommand"; import { CLIOutput } from "../types/cli"; import { Logger } from "../utils/logger"; import chalk from "chalk"; -import { ConductorError, ErrorCodes } from "../utils/errors"; -import { LyricRegistrationService } from "../services/lyric/LyricRegistrationService"; // Fixed import +import { ErrorFactory } from "../utils/errors"; +import { LyricRegistrationService } from "../services/lyric/LyricRegistrationService"; import { DictionaryRegistrationParams } from "../services/lyric/types"; /** * Command for registering a dictionary with the Lyric service + * Enhanced with ErrorFactory patterns and comprehensive validation */ export class LyricRegistrationCommand extends Command { constructor() { super("Lyric Dictionary Registration"); } + /** + * Validates command line arguments with enhanced error messages + */ + protected async validate(cliOutput: CLIOutput): Promise { + const { options } = cliOutput; + + Logger.debug`Validating Lyric registration parameters`; + + // Enhanced validation with specific guidance for each parameter + this.validateLyricUrl(options); + this.validateDictionaryName(options); + this.validateCategoryName(options); + this.validateDictionaryVersion(options); + this.validateCentricEntity(options); + + Logger.debugString("Lyric registration parameters validated"); + } + /** * Executes the Lyric dictionary registration process */ - protected async execute(cliOutput: CLIOutput): Promise { + protected async execute(cliOutput: CLIOutput): Promise { const { options } = cliOutput; - try { - // Extract configuration - much cleaner now - const registrationParams = this.extractRegistrationParams(options); - const serviceConfig = this.extractServiceConfig(options); - - // Create service instance using new pattern - fixed variable name - const lyricService = new LyricRegistrationService(serviceConfig); - - // Check service health first - const healthResult = await lyricService.checkHealth(); - if (!healthResult.healthy) { - throw new ConductorError( - `Lyric service is not healthy: ${ - healthResult.message || "Unknown error" - }`, - ErrorCodes.CONNECTION_ERROR, - { healthResult } - ); - } + // Extract configuration with enhanced validation + const registrationParams = this.extractRegistrationParams(options); + const serviceConfig = this.extractServiceConfig(options); - // Optional: Validate centric entity against Lectern - if (options.lecternUrl) { - await this.validateCentricEntity( - registrationParams.defaultCentricEntity, - registrationParams.dictionaryName, - registrationParams.dictionaryVersion, - options.lecternUrl - ); - } + Logger.debug`Starting Lyric dictionary registration`; + Logger.debug`Dictionary: ${registrationParams.dictionaryName} v${registrationParams.dictionaryVersion}`; + Logger.debug`Category: ${registrationParams.categoryName}`; + Logger.debug`Centric Entity: ${registrationParams.defaultCentricEntity}`; + + // Create service instance with enhanced error handling + const lyricService = new LyricRegistrationService(serviceConfig); + + // Enhanced health check with specific feedback + Logger.debug`Checking Lyric service health...`; + const healthResult = await lyricService.checkHealth(); + if (!healthResult.healthy) { + throw ErrorFactory.connection( + "Lyric service health check failed", + "Lyric", + serviceConfig.url, + [ + "Check that Lyric service is running", + `Verify service URL: ${serviceConfig.url}`, + "Check network connectivity and firewall settings", + "Review Lyric service logs for errors", + `Test manually: curl ${serviceConfig.url}/health`, + healthResult.message + ? `Health check message: ${healthResult.message}` + : "", + ].filter(Boolean) + ); + } + + // Optional: Validate centric entity against Lectern if URL provided + if (options.lecternUrl) { + await this.validateCentricEntityAgainstLectern( + registrationParams, + options.lecternUrl + ); + } + + // Register dictionary with enhanced context + this.logRegistrationInfo(registrationParams, serviceConfig.url); + + Logger.debug`Submitting dictionary registration to Lyric...`; + const result = await lyricService.registerDictionary(registrationParams); + + // Enhanced success logging + this.logSuccess(registrationParams, result); - // Register dictionary - much simpler now! - this.logRegistrationInfo(registrationParams, serviceConfig.url); + // Success - method completes normally + } - const result = await lyricService.registerDictionary(registrationParams); + /** + * Enhanced Lyric URL validation + */ + private validateLyricUrl(options: any): void { + const lyricUrl = options.lyricUrl || process.env.LYRIC_URL; - // Log success - this.logSuccess(registrationParams); + if (!lyricUrl) { + throw ErrorFactory.config( + "Lyric service URL not configured", + "lyricUrl", + [ + "Set Lyric URL: conductor lyricRegister --lyric-url http://localhost:3030", + "Set LYRIC_URL environment variable", + "Verify Lyric service is running and accessible", + "Check network connectivity to Lyric service", + ] + ); + } - return { - success: true, - details: result, - }; + // Basic URL format validation + try { + new URL(lyricUrl); + Logger.debug`Using Lyric URL: ${lyricUrl}`; } catch (error) { - return this.handleExecutionError(error); + throw ErrorFactory.config( + `Invalid Lyric URL format: ${lyricUrl}`, + "lyricUrl", + [ + "Use a valid URL format: http://localhost:3030", + "Include protocol (http:// or https://)", + "Check for typos in the URL", + "Verify port number is correct", + ] + ); } } /** - * Validates command line arguments + * Enhanced dictionary name validation */ - protected async validate(cliOutput: CLIOutput): Promise { - const { options } = cliOutput; + private validateDictionaryName(options: any): void { + const dictName = options.dictName || process.env.DICTIONARY_NAME; - // Validate required parameters exist - const requiredParams = [ - { key: "lyricUrl", name: "Lyric URL", envVar: "LYRIC_URL" }, - { key: "dictName", name: "Dictionary name", envVar: "DICTIONARY_NAME" }, - { key: "categoryName", name: "Category name", envVar: "CATEGORY_NAME" }, - { - key: "dictionaryVersion", - name: "Dictionary version", - envVar: "DICTIONARY_VERSION", - }, - { - key: "defaultCentricEntity", - name: "Default centric entity", - envVar: "DEFAULT_CENTRIC_ENTITY", - }, - ]; - - for (const param of requiredParams) { - const value = options[param.key] || process.env[param.envVar]; - if (!value) { - throw new ConductorError( - `${param.name} is required. Use --${param.key - .replace(/([A-Z])/g, "-$1") - .toLowerCase()} or set ${param.envVar} environment variable.`, - ErrorCodes.INVALID_ARGS - ); - } + if (!dictName) { + throw ErrorFactory.args( + "Dictionary name not specified", + "lyricRegister", + [ + "Provide dictionary name: conductor lyricRegister --dict-name my-dictionary", + "Set DICTIONARY_NAME environment variable", + "Use a descriptive name for the dictionary", + "Ensure the name matches your Lectern schema", + ] + ); } + + if (typeof dictName !== "string" || dictName.trim() === "") { + throw ErrorFactory.validation( + "Invalid dictionary name format", + { dictName }, + [ + "Dictionary name must be a non-empty string", + "Use descriptive names like 'clinical-data' or 'genomic-metadata'", + "Avoid special characters and spaces", + "Use lowercase with hyphens or underscores", + ] + ); + } + + // Validate name format + if (!/^[a-zA-Z0-9_-]+$/.test(dictName)) { + throw ErrorFactory.validation( + `Dictionary name contains invalid characters: ${dictName}`, + { dictName }, + [ + "Use only letters, numbers, hyphens, and underscores", + "Avoid spaces and special characters", + "Example: 'clinical-data-v1' or 'genomic_metadata'", + "Keep names concise but descriptive", + ] + ); + } + + Logger.debug`Dictionary name validated: ${dictName}`; } /** - * Extract registration parameters from options + * Enhanced category name validation + */ + private validateCategoryName(options: any): void { + const categoryName = options.categoryName || process.env.CATEGORY_NAME; + + if (!categoryName) { + throw ErrorFactory.args("Category name not specified", "lyricRegister", [ + "Provide category name: conductor lyricRegister --category-name my-category", + "Set CATEGORY_NAME environment variable", + "Categories organize related dictionaries", + "Use descriptive category names like 'clinical' or 'genomics'", + ]); + } + + if (typeof categoryName !== "string" || categoryName.trim() === "") { + throw ErrorFactory.validation( + "Invalid category name format", + { categoryName }, + [ + "Category name must be a non-empty string", + "Use descriptive names that group related dictionaries", + "Examples: 'clinical', 'genomics', 'metadata'", + "Keep names simple and memorable", + ] + ); + } + + Logger.debug`Category name validated: ${categoryName}`; + } + + /** + * Enhanced dictionary version validation + */ + private validateDictionaryVersion(options: any): void { + const version = options.dictionaryVersion || process.env.DICTIONARY_VERSION; + + if (!version) { + throw ErrorFactory.args( + "Dictionary version not specified", + "lyricRegister", + [ + "Provide version: conductor lyricRegister --dictionary-version 1.0", + "Set DICTIONARY_VERSION environment variable", + "Use semantic versioning: major.minor.patch", + "Examples: '1.0', '2.1.3', '1.0.0-beta'", + ] + ); + } + + if (typeof version !== "string" || version.trim() === "") { + throw ErrorFactory.validation( + "Invalid dictionary version format", + { version }, + [ + "Version must be a non-empty string", + "Use semantic versioning format: major.minor or major.minor.patch", + "Examples: '1.0', '2.1.3', '1.0.0-beta'", + "Increment versions when schema changes", + ] + ); + } + + // Basic version format validation + if (!/^\d+(\.\d+)*(-[a-zA-Z0-9]+)?$/.test(version)) { + Logger.warn`Version format '${version}' doesn't follow semantic versioning`; + Logger.tipString("Consider using semantic versioning: major.minor.patch"); + } + + Logger.debug`Dictionary version validated: ${version}`; + } + + /** + * Enhanced centric entity validation + */ + private validateCentricEntity(options: any): void { + const centricEntity = + options.defaultCentricEntity || process.env.DEFAULT_CENTRIC_ENTITY; + + if (!centricEntity) { + throw ErrorFactory.args( + "Default centric entity not specified", + "lyricRegister", + [ + "Provide centric entity: conductor lyricRegister --default-centric-entity donor", + "Set DEFAULT_CENTRIC_ENTITY environment variable", + "Centric entity must exist in your dictionary schema", + "Common entities: 'donor', 'specimen', 'sample', 'file'", + ] + ); + } + + if (typeof centricEntity !== "string" || centricEntity.trim() === "") { + throw ErrorFactory.validation( + "Invalid centric entity format", + { centricEntity }, + [ + "Centric entity must be a non-empty string", + "Use entity names from your dictionary schema", + "Examples: 'donor', 'specimen', 'sample', 'file'", + "Entity must be defined in your Lectern schema", + ] + ); + } + + // Basic entity name validation + if (!/^[a-zA-Z][a-zA-Z0-9_]*$/.test(centricEntity)) { + throw ErrorFactory.validation( + `Invalid centric entity format: ${centricEntity}`, + { centricEntity }, + [ + "Entity names must start with a letter", + "Use only letters, numbers, and underscores", + "Follow your schema's entity naming conventions", + "Examples: 'donor', 'specimen_data', 'sample_metadata'", + ] + ); + } + + Logger.debug`Centric entity validated: ${centricEntity}`; + } + + /** + * Extract registration parameters with validation */ private extractRegistrationParams( options: any @@ -121,141 +318,120 @@ export class LyricRegistrationCommand extends Command { } /** - * Extract service configuration from options + * Extract service configuration with enhanced defaults */ private extractServiceConfig(options: any) { + const url = options.lyricUrl || process.env.LYRIC_URL!; + return { - url: options.lyricUrl || process.env.LYRIC_URL!, - timeout: 10000, + url, + timeout: 15000, // Longer timeout for registration operations retries: 3, authToken: options.authToken || process.env.AUTH_TOKEN, }; } /** - * Validate centric entity against Lectern dictionary + * Enhanced centric entity validation against Lectern */ - private async validateCentricEntity( - centricEntity: string, - dictionaryName: string, - dictionaryVersion: string, + private async validateCentricEntityAgainstLectern( + params: DictionaryRegistrationParams, lecternUrl: string ): Promise { try { - Logger.info("Validating centric entity against Lectern dictionary..."); + Logger.info`Validating centric entity '${params.defaultCentricEntity}' against Lectern dictionary...`; - // This is a simplified version - you'd import and use LecternService here - // For now, just showing the pattern + // This would use LecternService to validate the entity + // For now, just showing the pattern with helpful logging const entities = await this.fetchDictionaryEntities( lecternUrl, - dictionaryName, - dictionaryVersion + params.dictionaryName, + params.dictionaryVersion ); - if (!entities.includes(centricEntity)) { - throw new ConductorError( - `Entity '${centricEntity}' does not exist in dictionary '${dictionaryName}'`, - ErrorCodes.VALIDATION_FAILED, + if (!entities.includes(params.defaultCentricEntity)) { + throw ErrorFactory.validation( + `Centric entity '${params.defaultCentricEntity}' not found in dictionary '${params.dictionaryName}'`, { + centricEntity: params.defaultCentricEntity, availableEntities: entities, - suggestion: `Available entities: ${entities.join(", ")}`, - } + dictionaryName: params.dictionaryName, + dictionaryVersion: params.dictionaryVersion, + }, + [ + `Available entities in ${params.dictionaryName}: ${entities.join( + ", " + )}`, + "Check the spelling of the centric entity name", + "Verify the entity exists in your Lectern schema", + "Update your schema if the entity is missing", + `Use one of: ${entities.slice(0, 3).join(", ")}${ + entities.length > 3 ? "..." : "" + }`, + ] ); } - Logger.info(`✓ Entity '${centricEntity}' validated against dictionary`); + Logger.success`Centric entity '${params.defaultCentricEntity}' validated against dictionary`; } catch (error) { - if (error instanceof ConductorError) { + if (error instanceof Error && error.name === "ConductorError") { throw error; } - Logger.warn( - `Could not validate centric entity: ${ - error instanceof Error ? error.message : String(error) - }` + Logger.warn`Could not validate centric entity against Lectern: ${ + error instanceof Error ? error.message : String(error) + }`; + Logger.tipString( + "Proceeding without Lectern validation - ensure entity exists in your schema" ); - Logger.warn("Proceeding without validation..."); } } /** * Fetch available entities from Lectern dictionary - * TODO: Replace with LecternService when refactored */ private async fetchDictionaryEntities( lecternUrl: string, dictionaryName: string, dictionaryVersion: string ): Promise { - // Placeholder - would use LecternService here - // This is just to show the pattern for validation - return ["donor", "specimen", "sample"]; // Example entities + // Placeholder implementation - would use LecternService in practice + // Return common entities for now + Logger.debug`Fetching entities from Lectern for ${dictionaryName} v${dictionaryVersion}`; + + // This would be replaced with actual LecternService calls + return ["donor", "specimen", "sample", "file", "analysis"]; } /** - * Log registration information + * Enhanced registration information logging */ private logRegistrationInfo( params: DictionaryRegistrationParams, url: string ): void { - Logger.info(`${chalk.bold.cyan("Registering Dictionary:")}`); - Logger.info(`URL: ${url}/dictionary/register`); - Logger.info(`Category: ${params.categoryName}`); - Logger.info(`Dictionary: ${params.dictionaryName}`); - Logger.info(`Version: ${params.dictionaryVersion}`); - Logger.info(`Centric Entity: ${params.defaultCentricEntity}`); - } - - /** - * Log successful registration - */ - private logSuccess(params: DictionaryRegistrationParams): void { - Logger.success("Dictionary registered successfully"); - Logger.generic(" "); - Logger.generic(chalk.gray(` - Category: ${params.categoryName}`)); - Logger.generic(chalk.gray(` - Dictionary: ${params.dictionaryName}`)); - Logger.generic(chalk.gray(` - Version: ${params.dictionaryVersion}`)); Logger.generic( - chalk.gray(` - Centric Entity: ${params.defaultCentricEntity}`) + `${chalk.bold.cyan("Lyric Dictionary Registration Details:\n")}` ); - Logger.generic(" "); + Logger.generic(` ▸ Service: ${url}/dictionary/register`); + Logger.generic(` ▸ Category: ${params.categoryName}`); + Logger.generic(` ▸ Dictionary: ${params.dictionaryName}`); + Logger.generic(` ▸ Version: ${params.dictionaryVersion}`); + Logger.generic(` ▸ Centric Entity: ${params.defaultCentricEntity}`); } /** - * Handle execution errors with helpful user feedback + * Enhanced success logging with detailed information */ - private handleExecutionError(error: unknown): CommandResult { - if (error instanceof ConductorError) { - // Handle specific error types with helpful messages - if ( - error.code === ErrorCodes.VALIDATION_FAILED && - error.details?.availableEntities - ) { - Logger.info( - `\nAvailable entities: ${error.details.availableEntities.join(", ")}` - ); - } + private logSuccess(params: DictionaryRegistrationParams, result: any): void { + Logger.success`Dictionary registered successfully with Lyric`; - if (error.details?.suggestion) { - Logger.tip(error.details.suggestion); - } - - return { - success: false, - errorMessage: error.message, - errorCode: error.code, - details: error.details, - }; + if (result.id) { + Logger.generic(chalk.gray(` ✓ Registration ID: ${result.id}`)); } - // Handle unexpected errors - const errorMessage = error instanceof Error ? error.message : String(error); - return { - success: false, - errorMessage: `Dictionary registration failed: ${errorMessage}`, - errorCode: ErrorCodes.CONNECTION_ERROR, - details: { originalError: error }, - }; + if (result.created_at) { + Logger.generic(chalk.gray(` ✓ Created: ${result.created_at}`)); + } } } diff --git a/apps/conductor/src/commands/lyricUploadCommand.ts b/apps/conductor/src/commands/lyricUploadCommand.ts index bbaac7a5..98c491f3 100644 --- a/apps/conductor/src/commands/lyricUploadCommand.ts +++ b/apps/conductor/src/commands/lyricUploadCommand.ts @@ -1,134 +1,422 @@ -// src/commands/lyricUploadCommand.ts -import { Command, CommandResult } from "./baseCommand"; +// src/commands/lyricUploadCommand.ts - Enhanced with ErrorFactory patterns +import { Command } from "./baseCommand"; import { CLIOutput } from "../types/cli"; import { Logger } from "../utils/logger"; import chalk from "chalk"; -import { ConductorError, ErrorCodes } from "../utils/errors"; +import { ErrorFactory } from "../utils/errors"; import { DataSubmissionResult, LyricSubmissionService, } from "../services/lyric/LyricSubmissionService"; import { DataSubmissionParams } from "../services/lyric/LyricSubmissionService"; +import * as fs from "fs"; +import * as path from "path"; /** * Command for loading data into Lyric - * Much simpler now with workflow extracted to service layer + * Enhanced with ErrorFactory patterns and comprehensive validation */ export class LyricUploadCommand extends Command { constructor() { super("Lyric Data Loading"); } + /** + * Validates command line arguments with enhanced error messages + */ + protected async validate(cliOutput: CLIOutput): Promise { + // Ensure config exists + if (!cliOutput.config) { + throw ErrorFactory.config("Configuration is missing", "config", [ + "Internal configuration error", + "Restart the application", + "Check command line arguments", + "Use --debug for detailed information", + ]); + } + + Logger.debug`Validating Lyric data upload parameters`; + + // Enhanced validation for required parameters + this.validateLyricUrl(cliOutput); + this.validateDataDirectory(cliOutput); + this.validateCategoryId(cliOutput); + this.validateOrganization(cliOutput); + this.validateRetrySettings(cliOutput); + + // Validate data directory contents + await this.validateDataDirectoryContents(cliOutput); + + Logger.successString("Lyric data upload parameters validated"); + } + /** * Executes the Lyric data loading process */ - protected async execute(cliOutput: CLIOutput): Promise { - try { - // Extract and validate configuration - const submissionParams = this.extractSubmissionParams(cliOutput); - const serviceConfig = this.extractServiceConfig(cliOutput); - - // Create service - const lyricSubmissionService = new LyricSubmissionService(serviceConfig); - - // Check service health - const healthResult = await lyricSubmissionService.checkHealth(); - if (!healthResult.healthy) { - throw new ConductorError( - `Lyric service is not healthy: ${ - healthResult.message || "Unknown error" - }`, - ErrorCodes.CONNECTION_ERROR - ); - } + protected async execute(cliOutput: CLIOutput): Promise { + // Extract and validate configuration + const submissionParams = this.extractSubmissionParams(cliOutput); + const serviceConfig = this.extractServiceConfig(cliOutput); + + Logger.info`Starting Lyric data loading process`; + Logger.info`Data Directory: ${submissionParams.dataDirectory}`; + Logger.info`Category ID: ${submissionParams.categoryId}`; + Logger.info`Organization: ${submissionParams.organization}`; - // Log submission info - this.logSubmissionInfo(submissionParams, serviceConfig.url); + // Create service with enhanced error handling + const lyricSubmissionService = new LyricSubmissionService(serviceConfig); - // Execute the complete workflow - const result = await lyricSubmissionService.submitDataWorkflow( - submissionParams + // Enhanced health check with specific feedback + Logger.debug`Checking Lyric service health...`; + const healthResult = await lyricSubmissionService.checkHealth(); + if (!healthResult.healthy) { + throw ErrorFactory.connection( + "Lyric service health check failed", + "Lyric", + serviceConfig.url, + [ + "Check that Lyric service is running", + `Verify service URL: ${serviceConfig.url}`, + "Check network connectivity and firewall settings", + "Review Lyric service logs for errors", + `Test manually: curl ${serviceConfig.url}/health`, + healthResult.message + ? `Health check message: ${healthResult.message}` + : "", + ].filter(Boolean) ); + } + + // Log submission info with enhanced context + this.logSubmissionInfo(submissionParams, serviceConfig.url); + + // Execute the complete workflow with enhanced progress tracking + Logger.info`Starting data submission workflow...`; + const result = await lyricSubmissionService.submitDataWorkflow( + submissionParams + ); + + // Enhanced success logging + this.logSuccess(result); - // Log success - this.logSuccess(result); + // Success - method completes normally + } + + /** + * Enhanced Lyric URL validation + */ + private validateLyricUrl(cliOutput: CLIOutput): void { + const lyricUrl = this.getLyricUrl(cliOutput); + + if (!lyricUrl) { + throw ErrorFactory.config( + "Lyric service URL not configured", + "lyricUrl", + [ + "Set Lyric URL: conductor lyricUpload --lyric-url http://localhost:3030", + "Set LYRIC_URL environment variable", + "Verify Lyric service is running and accessible", + "Check network connectivity to Lyric service", + ] + ); + } - return { - success: true, - details: result, - }; + // Basic URL format validation + try { + new URL(lyricUrl); + Logger.debug`Using Lyric URL: ${lyricUrl}`; } catch (error) { - return this.handleExecutionError(error); + throw ErrorFactory.config( + `Invalid Lyric URL format: ${lyricUrl}`, + "lyricUrl", + [ + "Use a valid URL format: http://localhost:3030", + "Include protocol (http:// or https://)", + "Check for typos in the URL", + "Verify port number is correct (usually 3030 for Lyric)", + ] + ); } } /** - * Validates command line arguments + * Enhanced data directory validation */ - protected async validate(cliOutput: CLIOutput): Promise { - // Ensure config exists - if (!cliOutput.config) { - throw new ConductorError( - "Configuration is missing", - ErrorCodes.INVALID_ARGS + private validateDataDirectory(cliOutput: CLIOutput): void { + const dataDirectory = this.getDataDirectory(cliOutput); + + if (!dataDirectory) { + throw ErrorFactory.args("Data directory not specified", "lyricUpload", [ + "Provide data directory: conductor lyricUpload --data-directory ./data", + "Set LYRIC_DATA environment variable", + "Ensure directory contains CSV files to upload", + "Use absolute or relative path to data directory", + ]); + } + + if (!fs.existsSync(dataDirectory)) { + throw ErrorFactory.file( + `Data directory not found: ${dataDirectory}`, + dataDirectory, + [ + "Check that the directory path is correct", + "Ensure the directory exists", + "Verify permissions allow access", + `Current directory: ${process.cwd()}`, + "Use absolute path if relative path is not working", + ] ); } - // Validate required parameters - const requiredParams = [ - { - value: this.getLyricUrl(cliOutput), - name: "Lyric URL", - suggestion: - "Use --lyric-url option or set LYRIC_URL environment variable", - }, - { - value: this.getDataDirectory(cliOutput), - name: "Data directory", - suggestion: - "Use --data-directory (-d) option or set LYRIC_DATA environment variable", - }, - ]; - - for (const param of requiredParams) { - if (!param.value) { - throw new ConductorError( - `${param.name} is required. ${param.suggestion}`, - ErrorCodes.INVALID_ARGS - ); - } + const stats = fs.statSync(dataDirectory); + if (!stats.isDirectory()) { + throw ErrorFactory.file( + `Path is not a directory: ${dataDirectory}`, + dataDirectory, + [ + "Provide a directory path, not a file path", + "Check the path points to a directory", + "Ensure the path is correct", + ] + ); + } + + Logger.debug`Data directory validated: ${dataDirectory}`; + } + + /** + * Enhanced category ID validation + */ + private validateCategoryId(cliOutput: CLIOutput): void { + const categoryId = + cliOutput.config.lyric?.categoryId || + cliOutput.options?.categoryId || + process.env.CATEGORY_ID; + + if (!categoryId) { + throw ErrorFactory.args("Category ID not specified", "lyricUpload", [ + "Provide category ID: conductor lyricUpload --category-id 1", + "Set CATEGORY_ID environment variable", + "Category ID should match your registered dictionary", + "Contact administrator for valid category IDs", + ]); + } + + // Validate category ID format + const categoryIdNum = parseInt(categoryId); + if (isNaN(categoryIdNum) || categoryIdNum <= 0) { + throw ErrorFactory.validation( + `Invalid category ID format: ${categoryId}`, + { categoryId }, + [ + "Category ID must be a positive integer", + "Examples: 1, 2, 3, etc.", + "Check with Lyric administrator for valid IDs", + "Ensure the category exists in Lyric", + ] + ); + } + + Logger.debug`Category ID validated: ${categoryId}`; + } + + /** + * Enhanced organization validation + */ + private validateOrganization(cliOutput: CLIOutput): void { + const organization = + cliOutput.config.lyric?.organization || + cliOutput.options?.organization || + process.env.ORGANIZATION; + + if (!organization) { + throw ErrorFactory.args("Organization not specified", "lyricUpload", [ + "Provide organization: conductor lyricUpload --organization OICR", + "Set ORGANIZATION environment variable", + "Use your institution or organization name", + "Organization should match your Lyric configuration", + ]); + } + + if (typeof organization !== "string" || organization.trim() === "") { + throw ErrorFactory.validation( + "Invalid organization format", + { organization }, + [ + "Organization must be a non-empty string", + "Use your institution's identifier", + "Examples: 'OICR', 'NIH', 'University-Toronto'", + "Check with Lyric administrator for valid organizations", + ] + ); + } + + Logger.debug`Organization validated: ${organization}`; + } + + /** + * Enhanced retry settings validation + */ + private validateRetrySettings(cliOutput: CLIOutput): void { + const maxRetries = + cliOutput.config.lyric?.maxRetries || + (cliOutput.options?.maxRetries + ? parseInt(cliOutput.options.maxRetries) + : undefined) || + 10; + + const retryDelay = + cliOutput.config.lyric?.retryDelay || + (cliOutput.options?.retryDelay + ? parseInt(cliOutput.options.retryDelay) + : undefined) || + 20000; + + if (maxRetries < 1 || maxRetries > 50) { + throw ErrorFactory.validation( + `Invalid max retries value: ${maxRetries}`, + { maxRetries }, + [ + "Max retries must be between 1 and 50", + "Recommended: 5-15 for most use cases", + "Higher values for unstable connections", + "Example: conductor lyricUpload --max-retries 10", + ] + ); } - // Validate data directory exists + if (retryDelay < 1000 || retryDelay > 300000) { + throw ErrorFactory.validation( + `Invalid retry delay value: ${retryDelay}ms`, + { retryDelay }, + [ + "Retry delay must be between 1000ms (1s) and 300000ms (5min)", + "Recommended: 10000-30000ms for most use cases", + "Longer delays for heavily loaded services", + "Example: conductor lyricUpload --retry-delay 20000", + ] + ); + } + + Logger.debug`Retry settings validated: ${maxRetries} retries, ${retryDelay}ms delay`; + } + + /** + * Enhanced data directory contents validation + */ + private async validateDataDirectoryContents( + cliOutput: CLIOutput + ): Promise { const dataDirectory = this.getDataDirectory(cliOutput)!; - if (!require("fs").existsSync(dataDirectory)) { - throw new ConductorError( - `Data directory not found: ${dataDirectory}`, - ErrorCodes.FILE_NOT_FOUND + + try { + const files = fs.readdirSync(dataDirectory); + const csvFiles = files.filter((file) => + file.toLowerCase().endsWith(".csv") + ); + + if (csvFiles.length === 0) { + throw ErrorFactory.file( + `No CSV files found in data directory: ${path.basename( + dataDirectory + )}`, + dataDirectory, + [ + "Ensure the directory contains CSV files", + "Check file extensions are .csv", + "Verify files are not in subdirectories", + `Directory contains: ${files.slice(0, 5).join(", ")}${ + files.length > 5 ? "..." : "" + }`, + "Only CSV files are supported for Lyric upload", + ] + ); + } + + // Validate each CSV file + const invalidFiles = []; + for (const csvFile of csvFiles) { + const filePath = path.join(dataDirectory, csvFile); + try { + const stats = fs.statSync(filePath); + if (stats.size === 0) { + invalidFiles.push(`${csvFile} (empty file)`); + } else if (stats.size > 100 * 1024 * 1024) { + // 100MB + Logger.warn`Large CSV file detected: ${csvFile} (${( + stats.size / + 1024 / + 1024 + ).toFixed(1)}MB)`; + Logger.tipString("Large files may take longer to process"); + } + } catch (error) { + invalidFiles.push(`${csvFile} (cannot read)`); + } + } + + if (invalidFiles.length > 0) { + throw ErrorFactory.file( + `Invalid CSV files found in data directory`, + dataDirectory, + [ + `Fix these files: ${invalidFiles.join(", ")}`, + "Ensure all CSV files contain data", + "Check file permissions", + "Remove or fix empty or corrupted files", + ] + ); + } + + Logger.success`Found ${csvFiles.length} valid CSV file(s) for upload`; + csvFiles.forEach((file) => Logger.debug` - ${file}`); + } catch (error) { + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + throw ErrorFactory.file( + `Error reading data directory: ${ + error instanceof Error ? error.message : String(error) + }`, + dataDirectory, + [ + "Check directory permissions", + "Ensure directory is accessible", + "Verify directory is not corrupted", + ] ); } } /** - * Extract submission parameters from CLI output + * Extract submission parameters with validation */ private extractSubmissionParams(cliOutput: CLIOutput): DataSubmissionParams { return { categoryId: - cliOutput.config.lyric?.categoryId || process.env.CATEGORY_ID || "1", + cliOutput.config.lyric?.categoryId || + cliOutput.options?.categoryId || + process.env.CATEGORY_ID || + "1", organization: cliOutput.config.lyric?.organization || + cliOutput.options?.organization || process.env.ORGANIZATION || "OICR", dataDirectory: this.getDataDirectory(cliOutput)!, maxRetries: parseInt( String( - cliOutput.config.lyric?.maxRetries || process.env.MAX_RETRIES || "10" + cliOutput.config.lyric?.maxRetries || + cliOutput.options?.maxRetries || + process.env.MAX_RETRIES || + "10" ) ), retryDelay: parseInt( String( cliOutput.config.lyric?.retryDelay || + cliOutput.options?.retryDelay || process.env.RETRY_DELAY || "20000" ) @@ -137,12 +425,12 @@ export class LyricUploadCommand extends Command { } /** - * Extract service configuration from CLI output + * Extract service configuration with enhanced defaults */ private extractServiceConfig(cliOutput: CLIOutput) { return { url: this.getLyricUrl(cliOutput)!, - timeout: 30000, // Longer timeout for file uploads + timeout: 60000, // Longer timeout for file uploads (1 minute) retries: 3, }; } @@ -170,76 +458,46 @@ export class LyricUploadCommand extends Command { } /** - * Log submission information + * Enhanced submission information logging */ private logSubmissionInfo( params: DataSubmissionParams, serviceUrl: string ): void { - Logger.info(`${chalk.bold.cyan("Starting Data Loading Process:")}`); - Logger.info(`Lyric URL: ${serviceUrl}`); - Logger.info(`Data Directory: ${params.dataDirectory}`); - Logger.info(`Category ID: ${params.categoryId}`); - Logger.info(`Organization: ${params.organization}`); - Logger.info(`Max Retries: ${params.maxRetries}`); + Logger.info`${chalk.bold.cyan("Lyric Data Loading Details:")}`; + Logger.generic(` Service: ${serviceUrl}`); + Logger.generic(` Data Directory: ${params.dataDirectory}`); + Logger.generic(` Category ID: ${params.categoryId}`); + Logger.generic(` Organization: ${params.organization}`); + Logger.generic(` Max Retries: ${params.maxRetries}`); + Logger.generic(` Retry Delay: ${params.retryDelay}ms`); } /** - * Log successful submission + * Enhanced success logging with detailed information */ private logSuccess(result: DataSubmissionResult): void { - Logger.success("Data loading completed successfully"); + Logger.success`Data loading completed successfully`; Logger.generic(" "); - Logger.generic(chalk.gray(` - Submission ID: ${result.submissionId}`)); - Logger.generic(chalk.gray(` - Status: ${result.status}`)); + Logger.generic(chalk.gray(` ✓ Submission ID: ${result.submissionId}`)); + Logger.generic(chalk.gray(` ✓ Status: ${result.status}`)); Logger.generic( - chalk.gray(` - Files Submitted: ${result.filesSubmitted.join(", ")}`) + chalk.gray(` ✓ Files Submitted: ${result.filesSubmitted.length}`) ); - Logger.generic(" "); - } - /** - * Handle execution errors with helpful user feedback - */ - private handleExecutionError(error: unknown): CommandResult { - if (error instanceof ConductorError) { - // Add context-specific help - if (error.code === ErrorCodes.FILE_NOT_FOUND) { - Logger.info( - "\nFile or directory issue detected. Check paths and permissions." - ); - } else if (error.code === ErrorCodes.VALIDATION_FAILED) { - Logger.info( - "\nSubmission validation failed. Check your data files for errors." - ); - if (error.details?.submissionId) { - Logger.info(`Submission ID: ${error.details.submissionId}`); - } - } else if (error.code === ErrorCodes.CONNECTION_ERROR) { - Logger.info( - "\nConnection error. Check network and service availability." - ); - } - - if (error.details?.suggestion) { - Logger.tip(error.details.suggestion); - } + if (result.filesSubmitted.length > 0) { + Logger.generic( + chalk.gray(` ✓ Files: ${result.filesSubmitted.join(", ")}`) + ); + } - return { - success: false, - errorMessage: error.message, - errorCode: error.code, - details: error.details, - }; + if (result.message) { + Logger.generic(chalk.gray(` ✓ Message: ${result.message}`)); } - // Handle unexpected errors - const errorMessage = error instanceof Error ? error.message : String(error); - return { - success: false, - errorMessage: `Data loading failed: ${errorMessage}`, - errorCode: ErrorCodes.CONNECTION_ERROR, - details: { originalError: error }, - }; + Logger.generic(" "); + Logger.tipString( + "Data is now available in Lyric for analysis and querying" + ); } } diff --git a/apps/conductor/src/commands/maestroIndexCommand.ts b/apps/conductor/src/commands/maestroIndexCommand.ts index 09111a22..3157c8f9 100644 --- a/apps/conductor/src/commands/maestroIndexCommand.ts +++ b/apps/conductor/src/commands/maestroIndexCommand.ts @@ -1,207 +1,371 @@ -import axios from "axios"; -import { Command, CommandResult } from "./baseCommand"; -import { CLIOutput } from "../types/cli"; -import { Logger } from "../utils/logger"; -import chalk from "chalk"; -import { ConductorError, ErrorCodes } from "../utils/errors"; - /** - * Response from index repository request + * Maestro Index Command + * + * Command for indexing data using the Maestro service. + * Enhanced with ErrorFactory patterns for consistent error handling. */ -interface IndexRepositoryResponse { - message?: string; - status?: string; - [key: string]: unknown; -} + +import { Command } from "./baseCommand"; +import { CLIOutput } from "../types/cli"; +import { Logger } from "../utils/logger"; +import { ErrorFactory } from "../utils/errors"; +import axios, { AxiosResponse } from "axios"; /** - * Command for indexing a repository with optional organization and ID filters + * Command for indexing data using Maestro service + * Enhanced with comprehensive validation and error handling */ export class MaestroIndexCommand extends Command { - private readonly TIMEOUT = 30000; // 30 seconds - constructor() { - super("maestroIndex"); - this.defaultOutputFileName = "index-repository-results.json"; + super("Maestro Index"); } /** - * Executes the repository indexing process - * @param cliOutput The CLI configuration and inputs - * @returns A CommandResult indicating success or failure + * Enhanced validation with specific error messages for each parameter */ - protected async execute(cliOutput: CLIOutput): Promise { + protected async validate(cliOutput: CLIOutput): Promise { const { options } = cliOutput; + Logger.debug`Validating Maestro indexing parameters`; + + // Enhanced validation for each required parameter + this.validateIndexUrl(options); + this.validateRepositoryCode(options); + + Logger.successString("Maestro indexing parameters validated"); + } + + /** + * Enhanced execution with detailed logging and error handling + */ + protected async execute(cliOutput: CLIOutput): Promise { + const { options } = cliOutput; + + // Extract validated parameters + const indexUrl = options.indexUrl || process.env.INDEX_URL; + const repositoryCode = + options.repositoryCode || process.env.REPOSITORY_CODE; + const organization = options.organization || process.env.ORGANIZATION; + const id = options.id || process.env.ID; + + // Construct the URL based on provided parameters + const requestUrl = this.buildRequestUrl( + indexUrl, + repositoryCode, + organization, + id + ); + + // Log indexing information with enhanced context + this.logIndexingInfo(requestUrl, repositoryCode, organization, id); + + // Make the request with enhanced error handling + Logger.info`Sending indexing request to Maestro...`; + const response = await this.makeIndexRequest(requestUrl); + + // Enhanced success logging + this.logSuccess(response.data, repositoryCode, organization, id); + + // Command completed successfully - no return needed + } + + /** + * Enhanced repository code validation + */ + private validateRepositoryCode(options: any): void { + const repositoryCode = + options.repositoryCode || process.env.REPOSITORY_CODE; + + if (!repositoryCode) { + throw ErrorFactory.args( + "Repository code required for indexing operation", + "maestroIndex", + [ + "Provide repository code: conductor maestroIndex --repository-code lyric.overture", + "Set REPOSITORY_CODE environment variable", + "Repository codes identify data sources in the system", + "Contact system administrator for valid repository codes", + "Common examples: 'lyric.overture', 'song.overture'", + ] + ); + } + + if (typeof repositoryCode !== "string" || repositoryCode.trim() === "") { + throw ErrorFactory.validation( + "Invalid repository code format", + { repositoryCode }, + [ + "Repository code must be a non-empty string", + "Use format like 'service.instance' (e.g., 'lyric.overture')", + "Check for typos or extra whitespace", + "Verify the repository code with your administrator", + ] + ); + } + + // Basic format validation + if (!/^[a-zA-Z0-9._-]+$/.test(repositoryCode)) { + throw ErrorFactory.validation( + `Repository code contains invalid characters: ${repositoryCode}`, + { repositoryCode }, + [ + "Use only letters, numbers, dots, hyphens, and underscores", + "Example format: 'lyric.overture' or 'song_instance'", + "Avoid spaces and special characters", + "Check with administrator for valid naming conventions", + ] + ); + } + + Logger.debug`Repository code validated: ${repositoryCode}`; + } + + /** + * Enhanced index URL validation + */ + private validateIndexUrl(options: any): void { + const indexUrl = options.indexUrl || process.env.INDEX_URL; + + if (!indexUrl) { + throw ErrorFactory.config( + "Maestro index URL not configured", + "indexUrl", + [ + "Provide index URL: conductor maestroIndex --index-url http://localhost:11235", + "Set INDEX_URL environment variable", + "Verify Maestro service is running and accessible", + "Check network connectivity to Maestro service", + "Default Maestro port is usually 11235", + ] + ); + } + try { - // Extract configuration from options or environment variables - const indexUrl = - options.indexUrl || process.env.INDEX_URL || "http://localhost:11235"; - const repositoryCode = - options.repositoryCode || process.env.REPOSITORY_CODE; - const organization = options.organization || process.env.ORGANIZATION; - const id = options.id || process.env.ID; - - // Validate required parameters - if (!repositoryCode) { - throw new ConductorError( - "Repository code not specified. Use --repository-code or set REPOSITORY_CODE environment variable.", - ErrorCodes.INVALID_ARGS + const url = new URL(indexUrl); + if (!["http:", "https:"].includes(url.protocol)) { + throw ErrorFactory.validation( + `Invalid protocol in Maestro URL: ${url.protocol}`, + { indexUrl, protocol: url.protocol }, + [ + "Protocol must be http or https", + "Use format: http://localhost:11235 or https://maestro.example.com", + "Check for typos in the URL", + "Verify the correct protocol with your administrator", + ] ); } - - // Construct the URL based on provided parameters - let url = `${indexUrl}/index/repository/${repositoryCode}`; - if (organization) { - url += `/organization/${organization}`; - if (id) { - url += `/id/${id}`; - } + Logger.debug`Using Maestro URL: ${indexUrl}`; + } catch (error) { + if (error instanceof Error && error.name === "ConductorError") { + throw error; // Re-throw enhanced errors } - // Log indexing information - Logger.info(`\x1b[1;36mIndexing Repository:\x1b[0m`); - Logger.info(`URL: ${url}`); - Logger.info(`Repository Code: ${repositoryCode}`); - if (organization) Logger.info(`Organization: ${organization}`); - if (id) Logger.info(`ID: ${id}`); - - // Make the request - Logger.info("Sending indexing request..."); - const response = await axios.post(url, "", { - headers: { - accept: "application/json", - }, - timeout: this.TIMEOUT, - }); - - // Process response - const responseData = response.data as IndexRepositoryResponse; - - // Log success message - Logger.success(`Repository indexing request successful`); - Logger.generic(" "); - Logger.generic(chalk.gray(` - Repository: ${repositoryCode}`)); - if (organization) - Logger.generic(chalk.gray(` - Organization: ${organization}`)); - if (id) Logger.generic(chalk.gray(` - ID: ${id}`)); - if (responseData && responseData.message) { - Logger.generic(chalk.gray(` - Message: ${responseData.message}`)); - } - Logger.generic(" "); - - return { - success: true, - details: { - repository: repositoryCode, - organization: organization || "All", - id: id || "All", - response: responseData, - }, - }; - } catch (error: unknown) { - // Handle errors and return failure result - if (error instanceof ConductorError) { - throw error; - } + throw ErrorFactory.config( + `Invalid Maestro URL format: ${indexUrl}`, + "indexUrl", + [ + "Use a valid URL format: http://localhost:11235", + "Include protocol (http:// or https://)", + "Check for typos in the URL", + "Verify port number is correct (usually 11235 for Maestro)", + "Ensure proper URL encoding for special characters", + ] + ); + } + } + + /** + * Build the complete request URL with query parameters + */ + private buildRequestUrl( + baseUrl: string, + repositoryCode: string, + organization?: string, + id?: string + ): string { + const url = new URL(`${baseUrl}/index`); + + url.searchParams.append("repositoryCode", repositoryCode); + + if (organization) { + url.searchParams.append("organization", organization); + } + + if (id) { + url.searchParams.append("id", id); + } + + return url.toString(); + } + + /** + * Enhanced logging for indexing operation details + */ + private logIndexingInfo( + requestUrl: string, + repositoryCode: string, + organization?: string, + id?: string + ): void { + Logger.section("Maestro Indexing Operation"); + Logger.info`Repository Code: ${repositoryCode}`; + + if (organization) { + Logger.info`Organization: ${organization}`; + } + + if (id) { + Logger.info`ID Filter: ${id}`; + } + + Logger.debug`Full request URL: ${requestUrl}`; + } + + /** + * Make the indexing request with enhanced error handling + */ + private async makeIndexRequest(requestUrl: string): Promise { + try { + const response = await axios.post(requestUrl); + return response; + } catch (error) { + if (axios.isAxiosError(error)) { + const status = error.response?.status; + const statusText = error.response?.statusText; + const responseData = error.response?.data; - // Handle Axios errors with more detail - if (this.isAxiosError(error)) { - const axiosError = error as any; - const status = axiosError.response?.status; - const responseData = axiosError.response?.data as - | Record - | undefined; - - let errorMessage = `Repository indexing failed: ${axiosError.message}`; - let errorDetails: Record = { - status, - responseData, - }; - - // Handle common error cases + // Enhanced error handling based on response status if (status === 404) { - errorMessage = `Repository not found: The specified repository code may be invalid`; - } else if (status === 401 || status === 403) { - errorMessage = `Authentication error: Ensure you have proper permissions`; - } else if (status === 400) { - errorMessage = `Bad request: ${ - responseData?.message || "Invalid parameters" - }`; - } else if (status === 500) { - errorMessage = `Server error: The indexing service encountered an internal error`; - } else if (axiosError.code === "ECONNREFUSED") { - errorMessage = `Connection refused: The indexing service at ${ - options.indexUrl || "http://localhost:11235" - } is not available`; - } else if (axiosError.code === "ETIMEDOUT") { - errorMessage = `Request timeout: The indexing service did not respond in time`; + throw ErrorFactory.connection( + "Maestro indexing endpoint not found", + "Maestro", + requestUrl, + [ + "Check that Maestro service is running", + "Verify the index URL is correct", + "Ensure Maestro service version supports this endpoint", + "Check Maestro service logs for errors", + `Test manually: curl -X POST ${requestUrl}`, + ] + ); } - Logger.error(errorMessage); + if (status === 400) { + throw ErrorFactory.validation( + `Invalid indexing parameters: ${ + responseData?.message || statusText + }`, + { status, responseData }, + [ + "Check repository code format and validity", + "Verify organization parameter if provided", + "Ensure ID parameter format is correct", + "Review Maestro API documentation for parameter requirements", + "Contact administrator for valid parameter values", + ] + ); + } - // Provide some helpful tips based on error type - if (status === 404 || status === 400) { - Logger.tip( - `Verify that the repository code "${options.repositoryCode}" is correct` + if (status === 401 || status === 403) { + throw ErrorFactory.connection( + `Maestro access denied: ${statusText}`, + "Maestro", + requestUrl, + [ + "Check authentication credentials", + "Verify user permissions for indexing operations", + "Contact administrator for access rights", + "Ensure proper API keys or tokens are configured", + ] ); - } else if (axiosError.code === "ECONNREFUSED") { - Logger.tip( - `Ensure the indexing service is running on ${ - options.indexUrl || "http://localhost:11235" - }` + } + + if (status === 500) { + throw ErrorFactory.connection( + `Maestro server error: ${responseData?.message || statusText}`, + "Maestro", + requestUrl, + [ + "Maestro service encountered an internal error", + "Check Maestro service logs for details", + "Retry the operation after a brief delay", + "Contact system administrator if problem persists", + "Verify system resources and service health", + ] ); } - throw new ConductorError( - errorMessage, - ErrorCodes.CONNECTION_ERROR, - errorDetails + // Generic HTTP error + throw ErrorFactory.connection( + `Maestro request failed: ${status} ${statusText}`, + "Maestro", + requestUrl, + [ + `HTTP ${status}: ${statusText}`, + "Check Maestro service status and logs", + "Verify network connectivity", + "Review request parameters", + "Contact administrator if problem persists", + ] ); } - // Generic error handling - const errorMessage = - error instanceof Error ? error.message : String(error); - - throw new ConductorError( - `Repository indexing failed: ${errorMessage}`, - ErrorCodes.CONNECTION_ERROR, - error + // Network or other errors + throw ErrorFactory.connection( + `Failed to connect to Maestro: ${ + error instanceof Error ? error.message : String(error) + }`, + "Maestro", + requestUrl, + [ + "Check that Maestro service is running", + "Verify network connectivity", + "Check firewall and proxy settings", + "Ensure correct URL and port", + "Review network configuration", + ] ); } } /** - * Type guard to check if an error is an Axios error - * @param error Any error object - * @returns Whether the error is an Axios error + * Enhanced success logging with operation details */ - private isAxiosError(error: unknown): boolean { - return Boolean( - error && - typeof error === "object" && - "isAxiosError" in error && - (error as { isAxiosError: boolean }).isAxiosError === true - ); - } + private logSuccess( + responseData: any, + repositoryCode: string, + organization?: string, + id?: string + ): void { + Logger.success`Maestro indexing request completed successfully`; - /** - * Validates command line arguments - * @param cliOutput - The parsed command line arguments - * @returns Promise that resolves when validation is complete - * @throws ConductorError if validation fails - */ - protected async validate(cliOutput: CLIOutput): Promise { - const { options } = cliOutput; - const repositoryCode = - options.repositoryCode || process.env.REPOSITORY_CODE; + // Log response details if available + if (responseData) { + if (responseData.message) { + Logger.info`Response: ${responseData.message}`; + } - if (!repositoryCode) { - throw new ConductorError( - "No repository code provided. Use --repository-code option or set REPOSITORY_CODE environment variable.", - ErrorCodes.INVALID_ARGS - ); + if (responseData.indexedCount !== undefined) { + Logger.info`Records indexed: ${responseData.indexedCount}`; + } + + if (responseData.processingTime) { + Logger.info`Processing time: ${responseData.processingTime}`; + } } + + // Summary of what was indexed + Logger.section("Indexing Summary"); + Logger.info`Repository: ${repositoryCode}`; + + if (organization) { + Logger.info`Organization: ${organization}`; + } + + if (id) { + Logger.info`ID Filter: ${id}`; + } + + Logger.tipString("Check Maestro logs for detailed indexing results"); } } diff --git a/apps/conductor/src/commands/songCreateStudyCommand.ts b/apps/conductor/src/commands/songCreateStudyCommand.ts index 63c38d65..67fca889 100644 --- a/apps/conductor/src/commands/songCreateStudyCommand.ts +++ b/apps/conductor/src/commands/songCreateStudyCommand.ts @@ -1,171 +1,368 @@ -// src/commands/songCreateStudyCommand.ts -import { Command, CommandResult } from "./baseCommand"; +/** + * SONG Create Study Command + * + * Command for creating studies in the SONG service. + * Enhanced with ErrorFactory patterns for consistent error handling. + */ + +import { Command } from "./baseCommand"; import { CLIOutput } from "../types/cli"; import { Logger } from "../utils/logger"; -import chalk from "chalk"; -import { ConductorError, ErrorCodes } from "../utils/errors"; -import { SongService } from "../services/song-score"; -import { SongStudyCreateParams } from "../services/song-score/types"; +import { ErrorFactory } from "../utils/errors"; +import { SongService } from "../services/song-score/songService"; +import { ServiceConfig } from "../services/base/types"; /** * Command for creating studies in SONG service - * Refactored to use the new SongService + * Enhanced with comprehensive validation and error handling */ export class SongCreateStudyCommand extends Command { constructor() { - super("SONG Study Creation"); + super("SONG Create Study"); + } + + /** + * Enhanced validation with specific error messages for each parameter + */ + protected async validate(cliOutput: CLIOutput): Promise { + const { options } = cliOutput; + + Logger.debug`Validating SONG study creation parameters`; + + // Enhanced validation for each required parameter + this.validateSongUrl(options); + this.validateStudyId(options); + this.validateStudyName(options); + this.validateOrganization(options); + this.validateOptionalParameters(options); + + Logger.successString("SONG study creation parameters validated"); } /** - * Executes the SONG study creation process + * Enhanced execution with detailed logging and error handling */ - protected async execute(cliOutput: CLIOutput): Promise { + protected async execute(cliOutput: CLIOutput): Promise { const { options } = cliOutput; + // Extract validated configuration + const serviceConfig = this.extractServiceConfig(options); + const studyParams = this.extractStudyParams(options); + + Logger.info`Starting SONG study creation`; + Logger.info`Study ID: ${studyParams.studyId}`; + Logger.info`Study Name: ${studyParams.studyName}`; + Logger.info`Organization: ${studyParams.organization}`; + Logger.info`SONG URL: ${serviceConfig.url}`; + + // Create service instance with enhanced error handling + const songService = new SongService(serviceConfig); + + // Enhanced health check with specific feedback + Logger.info`Checking SONG service health...`; + const healthResult = await songService.checkHealth(); + if (!healthResult.healthy) { + throw ErrorFactory.connection( + "SONG service health check failed", + "SONG", + serviceConfig.url, + [ + "Check that SONG service is running", + `Verify service URL: ${serviceConfig.url}`, + "Check network connectivity and firewall settings", + "Review SONG service logs for errors", + `Test manually: curl ${serviceConfig.url}/health`, + healthResult.message + ? `Health check message: ${healthResult.message}` + : "", + ] + ); + } + + Logger.success`SONG service is healthy`; + + // Create study with enhanced error handling + Logger.info`Creating study in SONG...`; + const createResult = await songService.createStudy(studyParams); + + // Enhanced success logging + this.logCreateSuccess(createResult, studyParams); + + // Command completed successfully + } + + /** + * Enhanced SONG URL validation + */ + private validateSongUrl(options: any): void { + const songUrl = options.songUrl || process.env.SONG_URL; + + if (!songUrl) { + throw ErrorFactory.config("SONG service URL not configured", "songUrl", [ + "Set SONG URL: conductor songCreateStudy --song-url http://localhost:8080", + "Set SONG_URL environment variable", + "Verify SONG service is running and accessible", + "Check network connectivity to SONG service", + "Default SONG port is usually 8080", + ]); + } + try { - // Extract configuration - const studyParams = this.extractStudyParams(options); - const serviceConfig = this.extractServiceConfig(options); - - // Create service instance - const songService = new SongService(serviceConfig); - - // Check service health - const healthResult = await songService.checkHealth(); - if (!healthResult.healthy) { - throw new ConductorError( - `SONG service is not healthy: ${ - healthResult.message || "Unknown error" - }`, - ErrorCodes.CONNECTION_ERROR, - { healthResult } + const url = new URL(songUrl); + if (!["http:", "https:"].includes(url.protocol)) { + throw ErrorFactory.validation( + `Invalid protocol in SONG URL: ${url.protocol}`, + { songUrl, protocol: url.protocol }, + [ + "Protocol must be http or https", + "Use format: http://localhost:8080 or https://song.example.com", + "Check for typos in the URL", + "Verify the correct protocol with your administrator", + ] ); } + Logger.debug`Using SONG URL: ${songUrl}`; + } catch (error) { + if (error instanceof Error && error.name === "ConductorError") { + throw error; // Re-throw enhanced errors + } + + throw ErrorFactory.config( + `Invalid SONG URL format: ${songUrl}`, + "songUrl", + [ + "Use a valid URL format: http://localhost:8080", + "Include protocol (http:// or https://)", + "Check for typos in the URL", + "Verify port number is correct (usually 8080 for SONG)", + "Ensure proper URL encoding for special characters", + ] + ); + } + } + + /** + * Enhanced study ID validation + */ + private validateStudyId(options: any): void { + const studyId = options.studyId || process.env.STUDY_ID; - // Log creation info - this.logCreationInfo(studyParams, serviceConfig.url); + if (!studyId) { + throw ErrorFactory.args("Study ID not specified", "songCreateStudy", [ + "Provide study ID: conductor songCreateStudy --study-id my-study", + "Set STUDY_ID environment variable", + "Use a unique identifier for the study", + "Study IDs should be descriptive and meaningful", + "Example: 'cancer-genomics-2024' or 'clinical-trial-001'", + ]); + } - // Create study - const result = await songService.createStudy(studyParams); + if (typeof studyId !== "string" || studyId.trim() === "") { + throw ErrorFactory.validation("Invalid study ID format", { studyId }, [ + "Study ID must be a non-empty string", + "Use descriptive IDs like 'cancer-genomics-2024' or 'clinical-trial-001'", + "Avoid spaces and special characters", + "Use lowercase with hyphens or underscores", + ]); + } - // Log success - this.logSuccess(result); + // Validate study ID format + if (!/^[a-zA-Z0-9_-]+$/.test(studyId)) { + throw ErrorFactory.validation( + `Study ID contains invalid characters: ${studyId}`, + { studyId }, + [ + "Use only letters, numbers, hyphens, and underscores", + "Avoid spaces and special characters", + "Example: 'genomic-study-2024' or 'clinical_trial_phase1'", + "Keep IDs concise but descriptive", + ] + ); + } - return { - success: true, - details: result, - }; - } catch (error) { - return this.handleExecutionError(error); + // Check for reserved study IDs + const reservedIds = [ + "test", + "demo", + "admin", + "system", + "null", + "undefined", + ]; + if (reservedIds.includes(studyId.toLowerCase())) { + Logger.warn`Study ID '${studyId}' is a commonly used name - consider using a more specific identifier`; } + + Logger.debug`Study ID validated: ${studyId}`; } /** - * Validates command line arguments + * Enhanced study name validation */ - protected async validate(cliOutput: CLIOutput): Promise { - const { options } = cliOutput; + private validateStudyName(options: any): void { + const studyName = options.studyName || process.env.STUDY_NAME; - // Validate required parameters - const requiredParams = [ - { key: "songUrl", name: "SONG URL", envVar: "SONG_URL" }, - { key: "studyId", name: "Study ID", envVar: "STUDY_ID" }, - { key: "studyName", name: "Study name", envVar: "STUDY_NAME" }, - { key: "organization", name: "Organization", envVar: "ORGANIZATION" }, - ]; + if (!studyName) { + throw ErrorFactory.args("Study name not specified", "songCreateStudy", [ + "Provide study name: conductor songCreateStudy --study-name 'My Research Study'", + "Set STUDY_NAME environment variable", + "Study name should be descriptive and human-readable", + "Use quotes for names with spaces", + "Example: 'Cancer Genomics Research 2024'", + ]); + } - for (const param of requiredParams) { - const value = options[param.key] || process.env[param.envVar]; - if (!value) { - throw new ConductorError( - `${param.name} is required. Use --${param.key - .replace(/([A-Z])/g, "-$1") - .toLowerCase()} or set ${param.envVar} environment variable.`, - ErrorCodes.INVALID_ARGS - ); - } + if (typeof studyName !== "string" || studyName.trim() === "") { + throw ErrorFactory.validation( + "Invalid study name format", + { studyName }, + [ + "Study name must be a non-empty string", + "Use descriptive names that explain the study purpose", + "Avoid very long names (keep under 100 characters)", + "Include key details like disease, data type, or year", + ] + ); } + + // Length validation + if (studyName.length > 200) { + throw ErrorFactory.validation( + `Study name is too long: ${studyName.length} characters`, + { studyName, length: studyName.length }, + [ + "Keep study names under 200 characters", + "Use concise but descriptive names", + "Move detailed information to the description field", + "Focus on the key aspects of the study", + ] + ); + } + + Logger.debug`Study name validated: ${studyName}`; } /** - * Extract study parameters from options + * Enhanced organization validation */ - private extractStudyParams(options: any): SongStudyCreateParams { - return { - studyId: options.studyId || process.env.STUDY_ID || "demo", - name: options.studyName || process.env.STUDY_NAME || "string", - organization: - options.organization || process.env.ORGANIZATION || "string", - description: options.description || process.env.DESCRIPTION || "string", - force: options.force || false, - }; + private validateOrganization(options: any): void { + const organization = options.organization || process.env.ORGANIZATION; + + if (!organization) { + throw ErrorFactory.args("Organization not specified", "songCreateStudy", [ + "Provide organization: conductor songCreateStudy --organization 'My University'", + "Set ORGANIZATION environment variable", + "Organization identifies the institution conducting the study", + "Use your institution's official name", + "Example: 'University of Toronto' or 'OICR'", + ]); + } + + if (typeof organization !== "string" || organization.trim() === "") { + throw ErrorFactory.validation( + "Invalid organization format", + { organization }, + [ + "Organization must be a non-empty string", + "Use your institution's official name", + "Avoid abbreviations unless commonly recognized", + "Include department if relevant", + ] + ); + } + + Logger.debug`Organization validated: ${organization}`; + } + + /** + * Validate optional parameters + */ + private validateOptionalParameters(options: any): void { + // Validate auth token if provided + const authToken = options.authToken || process.env.AUTH_TOKEN; + if (authToken && typeof authToken === "string" && authToken.trim() === "") { + Logger.warn`Empty auth token provided - using default authentication`; + } + + // Validate description if provided + const description = options.description || process.env.DESCRIPTION; + if (description && description.length > 1000) { + Logger.warn`Study description is very long (${description.length} characters) - consider shortening`; + } + + Logger.debug`Optional parameters validated`; } /** * Extract service configuration from options */ - private extractServiceConfig(options: any) { + private extractServiceConfig(options: any): ServiceConfig { + const songUrl = options.songUrl || process.env.SONG_URL; + const authToken = options.authToken || process.env.AUTH_TOKEN || "123"; + return { - url: options.songUrl || process.env.SONG_URL || "http://localhost:8080", - timeout: 10000, + url: songUrl, + authToken, + timeout: 30000, // 30 second timeout retries: 3, - authToken: options.authToken || process.env.AUTH_TOKEN || "123", }; } /** - * Log creation information + * Extract study parameters from options */ - private logCreationInfo(params: SongStudyCreateParams, url: string): void { - Logger.info(`${chalk.bold.cyan("Creating Study in SONG:")}`); - Logger.info(`URL: ${url}/studies/${params.studyId}/`); - Logger.info(`Study ID: ${params.studyId}`); - Logger.info(`Study Name: ${params.name}`); - Logger.info(`Organization: ${params.organization}`); + private extractStudyParams(options: any): any { + return { + studyId: options.studyId || process.env.STUDY_ID, + studyName: options.studyName || process.env.STUDY_NAME, + organization: options.organization || process.env.ORGANIZATION, + description: + options.description || + process.env.DESCRIPTION || + `Study created via Conductor CLI at ${new Date().toISOString()}`, + }; } /** - * Log successful creation + * Enhanced success logging with study details */ - private logSuccess(result: any): void { - Logger.success("Study created successfully"); - Logger.generic(" "); - Logger.generic(chalk.gray(` - Study ID: ${result.studyId}`)); - Logger.generic(chalk.gray(` - Study Name: ${result.name}`)); - Logger.generic(chalk.gray(` - Organization: ${result.organization}`)); - Logger.generic(chalk.gray(` - Status: ${result.status}`)); - Logger.generic(" "); - } + private logCreateSuccess(createResult: any, studyParams: any): void { + Logger.success`Study created successfully in SONG`; - /** - * Handle execution errors with helpful user feedback - */ - private handleExecutionError(error: unknown): CommandResult { - if (error instanceof ConductorError) { - // Add context-specific help for common errors - if (error.code === ErrorCodes.CONNECTION_ERROR) { - Logger.info("\nConnection error. Check SONG service availability."); + // Log study details + Logger.section("Study Details"); + Logger.info`Study ID: ${studyParams.studyId}`; + Logger.info`Study Name: ${studyParams.studyName}`; + Logger.info`Organization: ${studyParams.organization}`; + + if (studyParams.description) { + Logger.info`Description: ${studyParams.description}`; + } + + // Log creation result details if available + if (createResult) { + if (createResult.createdAt) { + Logger.info`Created at: ${createResult.createdAt}`; } - if (error.details?.suggestion) { - Logger.tip(error.details.suggestion); + if (createResult.studyUrl) { + Logger.info`Study URL: ${createResult.studyUrl}`; } - return { - success: false, - errorMessage: error.message, - errorCode: error.code, - details: error.details, - }; + if (createResult.status) { + Logger.info`Status: ${createResult.status}`; + } } - // Handle unexpected errors - const errorMessage = error instanceof Error ? error.message : String(error); - return { - success: false, - errorMessage: `Study creation failed: ${errorMessage}`, - errorCode: ErrorCodes.CONNECTION_ERROR, - details: { originalError: error }, - }; + // Summary and next steps + Logger.section("Next Steps"); + Logger.tipString( + "Study is now ready for schema uploads and analysis submissions" + ); + Logger.tipString("Use 'songUploadSchema' command to add data schemas"); + Logger.tipString( + "Use 'songSubmitAnalysis' command to submit analysis data" + ); + Logger.tipString("Check SONG web interface to manage study settings"); } } diff --git a/apps/conductor/src/commands/songPublishAnalysisCommand.ts b/apps/conductor/src/commands/songPublishAnalysisCommand.ts index 9e9b816e..83ee445d 100644 --- a/apps/conductor/src/commands/songPublishAnalysisCommand.ts +++ b/apps/conductor/src/commands/songPublishAnalysisCommand.ts @@ -1,178 +1,268 @@ -// src/commands/songPublishAnalysisCommand.ts -import { Command, CommandResult } from "./baseCommand"; +/** + * SONG Publish Analysis Command + * + * Command for publishing analysis in the SONG service. + * Enhanced with ErrorFactory patterns for consistent error handling. + */ + +import { Command } from "./baseCommand"; import { CLIOutput } from "../types/cli"; import { Logger } from "../utils/logger"; -import chalk from "chalk"; -import { ConductorError, ErrorCodes } from "../utils/errors"; -import { SongService } from "../services/song-score"; -import { SongPublishParams } from "../services/song-score/types"; +import { ErrorFactory } from "../utils/errors"; +import { SongService } from "../services/song-score/songService"; +import { ServiceConfig } from "../services/base/types"; /** - * Command for publishing analyses in SONG service - * Refactored to use the new SongService + * Command for publishing analysis in SONG service + * Enhanced with comprehensive validation and error handling */ export class SongPublishAnalysisCommand extends Command { constructor() { - super("SONG Analysis Publication"); + super("SONG Publish Analysis"); } /** - * Executes the SONG analysis publication process + * Enhanced validation with specific error messages for each parameter */ - protected async execute(cliOutput: CLIOutput): Promise { + protected async validate(cliOutput: CLIOutput): Promise { const { options } = cliOutput; - try { - // Extract configuration - const publishParams = this.extractPublishParams(options); - const serviceConfig = this.extractServiceConfig(options); - - // Create service instance - const songService = new SongService(serviceConfig); - - // Check service health - const healthResult = await songService.checkHealth(); - if (!healthResult.healthy) { - throw new ConductorError( - `SONG service is not healthy: ${ - healthResult.message || "Unknown error" - }`, - ErrorCodes.CONNECTION_ERROR, - { healthResult } - ); - } + Logger.debug`Validating SONG analysis publication parameters`; - // Log publication info - this.logPublicationInfo(publishParams, serviceConfig.url); + // Enhanced validation for each required parameter + this.validateSongUrl(options); + this.validateStudyId(options); + this.validateOptionalParameters(options); - // Publish analysis - const result = await songService.publishAnalysis(publishParams); + Logger.successString("SONG analysis publication parameters validated"); + } - // Log success - this.logSuccess(result); + /** + * Enhanced execution with detailed logging and error handling + */ + protected async execute(cliOutput: CLIOutput): Promise { + const { options } = cliOutput; + + // Extract validated configuration + const serviceConfig = this.extractServiceConfig(options); + const studyId = options.studyId || process.env.STUDY_ID; + + Logger.info`Starting SONG analysis publication`; + Logger.info`Study ID: ${studyId}`; + Logger.info`SONG URL: ${serviceConfig.url}`; + + // Create service instance with enhanced error handling + const songService = new SongService(serviceConfig); + + // Enhanced health check with specific feedback + Logger.info`Checking SONG service health...`; + const healthResult = await songService.checkHealth(); + if (!healthResult.healthy) { + throw ErrorFactory.connection( + "SONG service health check failed", + "SONG", + serviceConfig.url, + [ + "Check that SONG service is running", + `Verify service URL: ${serviceConfig.url}`, + "Check network connectivity and firewall settings", + "Review SONG service logs for errors", + `Test manually: curl ${serviceConfig.url}/health`, + healthResult.message + ? `Health check message: ${healthResult.message}` + : "", + ] + ); + } - return { - success: true, - details: result, - }; + Logger.success`SONG service is healthy`; + + // Publish analysis with enhanced error handling + Logger.info`Publishing analysis in SONG...`; + const publishResult = await songService.publishAnalysis(studyId); + + // Enhanced success logging + this.logPublishSuccess(publishResult, studyId); + + // Command completed successfully + } + + /** + * Enhanced SONG URL validation + */ + private validateSongUrl(options: any): void { + const songUrl = options.songUrl || process.env.SONG_URL; + + if (!songUrl) { + throw ErrorFactory.config("SONG service URL not configured", "songUrl", [ + "Set SONG URL: conductor songPublishAnalysis --song-url http://localhost:8080", + "Set SONG_URL environment variable", + "Verify SONG service is running and accessible", + "Check network connectivity to SONG service", + "Default SONG port is usually 8080", + ]); + } + + try { + const url = new URL(songUrl); + if (!["http:", "https:"].includes(url.protocol)) { + throw ErrorFactory.validation( + `Invalid protocol in SONG URL: ${url.protocol}`, + { songUrl, protocol: url.protocol }, + [ + "Protocol must be http or https", + "Use format: http://localhost:8080 or https://song.example.com", + "Check for typos in the URL", + "Verify the correct protocol with your administrator", + ] + ); + } + Logger.debug`Using SONG URL: ${songUrl}`; } catch (error) { - return this.handleExecutionError(error); + if (error instanceof Error && error.name === "ConductorError") { + throw error; // Re-throw enhanced errors + } + + throw ErrorFactory.config( + `Invalid SONG URL format: ${songUrl}`, + "songUrl", + [ + "Use a valid URL format: http://localhost:8080", + "Include protocol (http:// or https://)", + "Check for typos in the URL", + "Verify port number is correct (usually 8080 for SONG)", + "Ensure proper URL encoding for special characters", + ] + ); } } /** - * Validates command line arguments + * Enhanced study ID validation */ - protected async validate(cliOutput: CLIOutput): Promise { - const { options } = cliOutput; + private validateStudyId(options: any): void { + const studyId = options.studyId || process.env.STUDY_ID; - // Validate analysis ID - const analysisId = this.getAnalysisId(options); - if (!analysisId) { - throw new ConductorError( - "Analysis ID not specified. Use --analysis-id or set ANALYSIS_ID environment variable.", - ErrorCodes.INVALID_ARGS + if (!studyId) { + throw ErrorFactory.args( + "Study ID not specified for analysis publication", + "songPublishAnalysis", + [ + "Provide study ID: conductor songPublishAnalysis --study-id my-study", + "Set STUDY_ID environment variable", + "Study ID should match the study containing the analysis", + "Ensure the study exists in SONG", + "Use the same study ID from when the analysis was submitted", + ] ); } - // Validate SONG URL - const songUrl = this.getSongUrl(options); - if (!songUrl) { - throw new ConductorError( - "SONG URL not specified. Use --song-url or set SONG_URL environment variable.", - ErrorCodes.INVALID_ARGS + if (typeof studyId !== "string" || studyId.trim() === "") { + throw ErrorFactory.validation("Invalid study ID format", { studyId }, [ + "Study ID must be a non-empty string", + "Use the exact study ID from SONG", + "Check for typos or extra whitespace", + "Verify the study exists before publishing analysis", + ]); + } + + // Basic format validation + if (!/^[a-zA-Z0-9_-]+$/.test(studyId)) { + throw ErrorFactory.validation( + `Study ID contains invalid characters: ${studyId}`, + { studyId }, + [ + "Study ID must contain only letters, numbers, hyphens, and underscores", + "Match the study ID used when creating the study", + "Check for typos or extra characters", + "Ensure the study exists in SONG", + ] ); } + + Logger.debug`Study ID validated: ${studyId}`; } /** - * Extract publish parameters from options + * Validate optional parameters */ - private extractPublishParams(options: any): SongPublishParams { - return { - analysisId: this.getAnalysisId(options)!, - studyId: options.studyId || process.env.STUDY_ID || "demo", - ignoreUndefinedMd5: options.ignoreUndefinedMd5 || false, - }; + private validateOptionalParameters(options: any): void { + // Validate auth token if provided + const authToken = options.authToken || process.env.AUTH_TOKEN; + if (authToken && typeof authToken === "string" && authToken.trim() === "") { + Logger.warn`Empty auth token provided - using default authentication`; + } + + // Validate analysis ID if provided + const analysisId = options.analysisId || process.env.ANALYSIS_ID; + if ( + analysisId && + (typeof analysisId !== "string" || analysisId.trim() === "") + ) { + throw ErrorFactory.validation( + "Invalid analysis ID format", + { analysisId }, + [ + "Analysis ID must be a non-empty string if provided", + "Use the exact analysis ID from SONG", + "Check for typos or extra whitespace", + "Leave empty to publish all unpublished analyses in the study", + ] + ); + } + + Logger.debug`Optional parameters validated`; } /** * Extract service configuration from options */ - private extractServiceConfig(options: any) { + private extractServiceConfig(options: any): ServiceConfig { + const songUrl = options.songUrl || process.env.SONG_URL; + const authToken = options.authToken || process.env.AUTH_TOKEN || "123"; + return { - url: this.getSongUrl(options)!, - timeout: 10000, + url: songUrl, + authToken, + timeout: 60000, // 60 second timeout for publish operations retries: 3, - authToken: options.authToken || process.env.AUTH_TOKEN || "123", }; } - private getAnalysisId(options: any): string | undefined { - return options.analysisId || process.env.ANALYSIS_ID; - } - - private getSongUrl(options: any): string | undefined { - return options.songUrl || process.env.SONG_URL; - } - /** - * Log publication information + * Enhanced success logging with publication details */ - private logPublicationInfo(params: SongPublishParams, url: string): void { - Logger.info(`${chalk.bold.cyan("Publishing Analysis in SONG:")}`); - Logger.info( - `URL: ${url}/studies/${params.studyId}/analysis/publish/${params.analysisId}` - ); - Logger.info(`Analysis ID: ${params.analysisId}`); - Logger.info(`Study ID: ${params.studyId}`); - } + private logPublishSuccess(publishResult: any, studyId: string): void { + Logger.success`Analysis published successfully in SONG`; - /** - * Log successful publication - */ - private logSuccess(result: any): void { - Logger.success("Analysis published successfully"); - Logger.generic(" "); - Logger.generic(chalk.gray(` - Analysis ID: ${result.analysisId}`)); - Logger.generic(chalk.gray(` - Study ID: ${result.studyId}`)); - Logger.generic(chalk.gray(` - Status: ${result.status}`)); - Logger.generic(" "); - } + // Log publication details if available + if (publishResult) { + if (publishResult.analysisId) { + Logger.info`Analysis ID: ${publishResult.analysisId}`; + } - /** - * Handle execution errors with helpful user feedback - */ - private handleExecutionError(error: unknown): CommandResult { - if (error instanceof ConductorError) { - // Add context-specific help for common errors - if (error.code === ErrorCodes.FILE_NOT_FOUND) { - Logger.tip( - "Make sure the analysis ID exists and belongs to the specified study" - ); - } else if (error.code === ErrorCodes.CONNECTION_ERROR) { - Logger.info("\nConnection error. Check SONG service availability."); + if (publishResult.publishedCount !== undefined) { + Logger.info`Analyses published: ${publishResult.publishedCount}`; } - if (error.details?.suggestion) { - Logger.tip(error.details.suggestion); + if (publishResult.status) { + Logger.info`Publication status: ${publishResult.status}`; } - return { - success: false, - errorMessage: error.message, - errorCode: error.code, - details: error.details, - }; + if (publishResult.publishedAt) { + Logger.info`Published at: ${publishResult.publishedAt}`; + } } - // Handle unexpected errors - const errorMessage = error instanceof Error ? error.message : String(error); - return { - success: false, - errorMessage: `Analysis publication failed: ${errorMessage}`, - errorCode: ErrorCodes.CONNECTION_ERROR, - details: { originalError: error }, - }; + // Summary information + Logger.section("Publication Summary"); + Logger.info`Study ID: ${studyId}`; + Logger.info`Publication timestamp: ${new Date().toISOString()}`; + + Logger.tipString("Analysis is now publicly accessible in SONG"); + Logger.tipString( + "Published analyses cannot be modified - create new analysis for updates" + ); + Logger.tipString("Check SONG web interface to verify publication status"); } } diff --git a/apps/conductor/src/commands/songSubmitAnalysisCommand.ts b/apps/conductor/src/commands/songSubmitAnalysisCommand.ts index db6d26d9..91714854 100644 --- a/apps/conductor/src/commands/songSubmitAnalysisCommand.ts +++ b/apps/conductor/src/commands/songSubmitAnalysisCommand.ts @@ -1,9 +1,9 @@ -// src/commands/songSubmitAnalysisCommand.ts - Combined with scoreManifestUpload -import { Command, CommandResult } from "./baseCommand"; +// src/commands/songSubmitAnalysisCommand.ts - Enhanced with ErrorFactory patterns +import { Command } from "./baseCommand"; import { CLIOutput } from "../types/cli"; import { Logger } from "../utils/logger"; import chalk from "chalk"; -import { ConductorError, ErrorCodes } from "../utils/errors"; +import { ErrorFactory } from "../utils/errors"; import { SongScoreService } from "../services/song-score"; import { SongScoreWorkflowParams } from "../services/song-score/types"; import * as fs from "fs"; @@ -11,7 +11,7 @@ import * as path from "path"; /** * Combined command for SONG analysis submission and Score file upload - * This replaces both songSubmitAnalysis and scoreManifestUpload commands + * Enhanced with ErrorFactory patterns and comprehensive validation */ export class SongSubmitAnalysisCommand extends Command { constructor() { @@ -19,104 +19,532 @@ export class SongSubmitAnalysisCommand extends Command { } /** - * Executes the combined SONG/Score workflow + * Validates command line arguments with enhanced error messages */ - protected async execute(cliOutput: CLIOutput): Promise { + protected async validate(cliOutput: CLIOutput): Promise { const { options } = cliOutput; - try { - // Extract configuration - const workflowParams = this.extractWorkflowParams(options); - const serviceConfig = this.extractServiceConfig(options); - const scoreConfig = this.extractScoreConfig(options); - - // Create combined service instance - const songScoreService = new SongScoreService(serviceConfig, scoreConfig); - - // Check Docker requirements for Score operations - await songScoreService.validateDockerRequirements(); - - // Check services health - const healthStatus = await songScoreService.checkServicesHealth(); - if (!healthStatus.overall) { - const issues = []; - if (!healthStatus.song) issues.push("SONG"); - if (!healthStatus.score) issues.push("Score"); - - throw new ConductorError( - `Service health check failed: ${issues.join( - ", " - )} service(s) not healthy`, - ErrorCodes.CONNECTION_ERROR, - { healthStatus } - ); - } - - // Log workflow info - this.logWorkflowInfo(workflowParams, serviceConfig.url, scoreConfig?.url); + Logger.debug`Validating SONG/Score workflow parameters`; - // Execute the complete workflow - const result = await songScoreService.executeWorkflow(workflowParams); + // Enhanced validation for all required parameters + this.validateAnalysisFile(options); + this.validateDataDirectory(options); + this.validateSongUrl(options); + this.validateStudyId(options); + this.validateScoreUrl(options); + this.validateManifestFile(options); + this.validateOptionalParameters(options); - // Log success/partial success - if (result.success) { - this.logSuccess(result); - } else { - this.logPartialSuccess(result); - } + // Validate file contents + await this.validateAnalysisFileContents(options); + await this.validateDataDirectoryContents(options); - return { - success: result.success, - details: result, - }; - } catch (error) { - return this.handleExecutionError(error); - } + Logger.successString("SONG/Score workflow parameters validated"); } /** - * Validates command line arguments + * Executes the combined SONG/Score workflow */ - protected async validate(cliOutput: CLIOutput): Promise { + protected async execute(cliOutput: CLIOutput): Promise { const { options } = cliOutput; - // Validate analysis file + // Extract configuration with enhanced validation + const workflowParams = this.extractWorkflowParams(options); + const serviceConfig = this.extractServiceConfig(options); + const scoreConfig = this.extractScoreConfig(options); + + Logger.info`Starting SONG/Score analysis workflow`; + Logger.info`Study ID: ${workflowParams.studyId}`; + Logger.info`Data Directory: ${workflowParams.dataDir}`; + Logger.info`Manifest File: ${workflowParams.manifestFile}`; + + // Create combined service instance with enhanced error handling + const songScoreService = new SongScoreService(serviceConfig, scoreConfig); + + // Enhanced Docker requirements validation + Logger.info`Validating Docker requirements for Score operations...`; + await songScoreService.validateDockerRequirements(); + + // Enhanced services health check + Logger.info`Checking SONG and Score services health...`; + const healthStatus = await songScoreService.checkServicesHealth(); + if (!healthStatus.overall) { + const issues = []; + if (!healthStatus.song) issues.push("SONG"); + if (!healthStatus.score) issues.push("Score"); + + throw ErrorFactory.connection( + `Service health check failed: ${issues.join( + ", " + )} service(s) not healthy`, + issues[0], + undefined, + [ + `Check that ${issues.join(" and ")} service(s) are running`, + "Verify service URLs and connectivity", + "Review service logs for errors", + "Check Docker containers if using containerized services", + "Ensure proper authentication and permissions", + ] + ); + } + + // Log workflow info with enhanced context + this.logWorkflowInfo(workflowParams, serviceConfig.url, scoreConfig?.url); + + // Execute the complete workflow with enhanced progress tracking + Logger.info`Executing SONG/Score workflow...`; + const result = await songScoreService.executeWorkflow(workflowParams); + + // Enhanced success/partial success logging + if (result.success) { + this.logSuccess(result); + } else { + this.logPartialSuccess(result); + } + + // Command completed successfully + } + + /** + * Enhanced analysis file validation + */ + private validateAnalysisFile(options: any): void { const analysisFile = this.getAnalysisFile(options); + if (!analysisFile) { - throw new ConductorError( - "Analysis file not specified. Use --analysis-file or set ANALYSIS_FILE environment variable.", - ErrorCodes.INVALID_ARGS + throw ErrorFactory.args( + "Analysis file not specified", + "songSubmitAnalysis", + [ + "Provide analysis file: conductor songSubmitAnalysis --analysis-file analysis.json", + "Set ANALYSIS_FILE environment variable", + "Analysis file should contain SONG analysis definition", + "Ensure file path is correct and accessible", + ] ); } if (!fs.existsSync(analysisFile)) { - throw new ConductorError( - `Analysis file not found: ${analysisFile}`, - ErrorCodes.FILE_NOT_FOUND + throw ErrorFactory.file( + `Analysis file not found: ${path.basename(analysisFile)}`, + analysisFile, + [ + "Check that the file path is correct", + "Ensure the file exists at the specified location", + "Verify file permissions allow read access", + `Current directory: ${process.cwd()}`, + "Use absolute path if relative path is not working", + ] ); } - // Validate data directory + const stats = fs.statSync(analysisFile); + if (stats.size === 0) { + throw ErrorFactory.file( + `Analysis file is empty: ${path.basename(analysisFile)}`, + analysisFile, + [ + "Ensure the file contains valid analysis definition", + "Check if the file was properly created", + "Verify the file is not corrupted", + ] + ); + } + + Logger.debug`Analysis file validated: ${analysisFile}`; + } + + /** + * Enhanced data directory validation + */ + private validateDataDirectory(options: any): void { const dataDir = this.getDataDir(options); + if (!fs.existsSync(dataDir)) { - throw new ConductorError( - `Data directory not found: ${dataDir}`, - ErrorCodes.FILE_NOT_FOUND - ); + throw ErrorFactory.file(`Data directory not found: ${dataDir}`, dataDir, [ + "Check that the directory path is correct", + "Ensure the directory exists", + "Verify permissions allow access", + `Current directory: ${process.cwd()}`, + "Create the directory if it doesn't exist", + ]); } - // Validate SONG URL + const stats = fs.statSync(dataDir); + if (!stats.isDirectory()) { + throw ErrorFactory.file(`Path is not a directory: ${dataDir}`, dataDir, [ + "Provide a directory path, not a file path", + "Check the path points to a directory", + "Ensure the path is correct", + ]); + } + + Logger.debug`Data directory validated: ${dataDir}`; + } + + /** + * Enhanced SONG URL validation with protocol checking + */ + private validateSongUrl(options: any): void { const songUrl = this.getSongUrl(options); + if (!songUrl) { - throw new ConductorError( - "SONG URL not specified. Use --song-url or set SONG_URL environment variable.", - ErrorCodes.INVALID_ARGS + throw ErrorFactory.config("SONG service URL not configured", "songUrl", [ + "Set SONG URL: conductor songSubmitAnalysis --song-url http://localhost:8080", + "Set SONG_URL environment variable", + "Verify SONG service is running and accessible", + "Check network connectivity to SONG service", + ]); + } + + try { + const url = new URL(songUrl); + if (!["http:", "https:"].includes(url.protocol)) { + throw ErrorFactory.validation( + `Invalid protocol in SONG URL: ${url.protocol}`, + { songUrl, protocol: url.protocol }, + [ + "Protocol must be http or https", + "Use format: http://localhost:8080 or https://song.example.com", + "Check for typos in the URL", + "Verify the correct protocol with your administrator", + ] + ); + } + Logger.debug`Using SONG URL: ${songUrl}`; + } catch (error) { + if (error instanceof Error && error.name === "ConductorError") { + throw error; // Re-throw enhanced errors + } + + throw ErrorFactory.config( + `Invalid SONG URL format: ${songUrl}`, + "songUrl", + [ + "Use a valid URL format: http://localhost:8080", + "Include protocol (http:// or https://)", + "Check for typos in the URL", + "Verify port number is correct (usually 8080 for SONG)", + ] + ); + } + } + + /** + * Enhanced study ID validation + */ + private validateStudyId(options: any): void { + const studyId = options.studyId || process.env.STUDY_ID; + + if (!studyId) { + throw ErrorFactory.args("Study ID not specified", "songSubmitAnalysis", [ + "Provide study ID: conductor songSubmitAnalysis --study-id my-study", + "Set STUDY_ID environment variable", + "Study must exist in SONG before submitting analysis", + "Create study first with songCreateStudy command", + ]); + } + + if (!/^[a-zA-Z0-9_-]+$/.test(studyId)) { + throw ErrorFactory.validation( + `Invalid study ID format: ${studyId}`, + { studyId }, + [ + "Study ID must contain only letters, numbers, hyphens, and underscores", + "Match the study ID used when creating the study", + "Check for typos or extra characters", + ] + ); + } + + Logger.debug`Study ID validated: ${studyId}`; + } + + /** + * Enhanced Score URL validation with protocol checking + */ + private validateScoreUrl(options: any): void { + const scoreUrl = this.getScoreUrl(options); + + try { + const url = new URL(scoreUrl); + if (!["http:", "https:"].includes(url.protocol)) { + throw ErrorFactory.validation( + `Invalid protocol in Score URL: ${url.protocol}`, + { scoreUrl, protocol: url.protocol }, + [ + "Protocol must be http or https", + "Use format: http://localhost:8087 or https://score.example.com", + "Check for typos in the URL", + "Verify the correct protocol with your administrator", + ] + ); + } + Logger.debug`Using Score URL: ${scoreUrl}`; + } catch (error) { + if (error instanceof Error && error.name === "ConductorError") { + throw error; // Re-throw enhanced errors + } + + throw ErrorFactory.config( + `Invalid Score URL format: ${scoreUrl}`, + "scoreUrl", + [ + "Use a valid URL format: http://localhost:8087", + "Include protocol (http:// or https://)", + "Check for typos in the URL", + "Verify port number is correct (usually 8087 for Score)", + ] ); } } /** - * Extract workflow parameters from options + * Enhanced manifest file validation + */ + private validateManifestFile(options: any): void { + const manifestFile = this.getManifestFile(options); + const manifestDir = path.dirname(manifestFile); + + // Create output directory if it doesn't exist + if (!fs.existsSync(manifestDir)) { + try { + fs.mkdirSync(manifestDir, { recursive: true }); + Logger.info`Created directory for manifest: ${manifestDir}`; + } catch (error) { + throw ErrorFactory.file( + `Cannot create manifest directory: ${manifestDir}`, + manifestDir, + [ + "Check directory permissions", + "Ensure parent directories exist", + "Verify disk space is available", + "Use a different output directory", + ] + ); + } + } + + Logger.debug`Manifest file path validated: ${manifestFile}`; + } + + /** + * Validate optional parameters + */ + private validateOptionalParameters(options: any): void { + // Validate auth token if provided + const authToken = options.authToken || process.env.AUTH_TOKEN; + if (authToken && typeof authToken === "string" && authToken.trim() === "") { + Logger.warn`Empty auth token provided - using empty token`; + } + + // Validate boolean flags + if ( + options.allowDuplicates !== undefined && + typeof options.allowDuplicates !== "boolean" + ) { + Logger.warn`Invalid allowDuplicates value, using false`; + } + + if ( + options.ignoreUndefinedMd5 !== undefined && + typeof options.ignoreUndefinedMd5 !== "boolean" + ) { + Logger.warn`Invalid ignoreUndefinedMd5 value, using false`; + } + + Logger.debug`Optional parameters validated`; + } + + /** + * Enhanced analysis file contents validation + */ + private async validateAnalysisFileContents(options: any): Promise { + const analysisFile = this.getAnalysisFile(options)!; + + try { + const fileContent = fs.readFileSync(analysisFile, "utf-8"); + + // Parse JSON and validate structure + let analysisData; + try { + analysisData = JSON.parse(fileContent); + } catch (error) { + throw ErrorFactory.file( + `Invalid JSON format in analysis file: ${path.basename( + analysisFile + )}`, + analysisFile, + [ + "Check JSON syntax for errors (missing commas, brackets, quotes)", + "Validate JSON structure using a JSON validator", + "Ensure file encoding is UTF-8", + "Try viewing the file in a JSON editor", + error instanceof Error ? `JSON error: ${error.message}` : "", + ].filter(Boolean) + ); + } + + // Validate required SONG analysis fields + if (!analysisData.analysisType || !analysisData.analysisType.name) { + throw ErrorFactory.validation( + `Missing required field 'analysisType.name' in analysis file`, + { analysisFile, analysisData: Object.keys(analysisData) }, + [ + "Analysis must have 'analysisType' object with 'name' field", + "Check SONG analysis schema requirements", + "Ensure analysis type is properly defined", + "Review SONG documentation for analysis structure", + ] + ); + } + + if ( + !analysisData.files || + !Array.isArray(analysisData.files) || + analysisData.files.length === 0 + ) { + throw ErrorFactory.validation( + `Missing or empty 'files' array in analysis file`, + { analysisFile, filesCount: analysisData.files?.length || 0 }, + [ + "Analysis must include 'files' array with at least one file", + "Each file should have objectId, fileName, and fileMd5sum", + "Ensure files are properly defined in the analysis", + "Check that file references match actual data files", + ] + ); + } + + // Validate files array structure + const invalidFiles = analysisData.files.filter( + (file: any, index: number) => { + const hasObjectId = + file.objectId && typeof file.objectId === "string"; + const hasFileName = + file.fileName && typeof file.fileName === "string"; + const hasFileMd5sum = + file.fileMd5sum && typeof file.fileMd5sum === "string"; + + return !hasObjectId || !hasFileName || !hasFileMd5sum; + } + ); + + if (invalidFiles.length > 0) { + throw ErrorFactory.validation( + `Invalid file entries in analysis (${invalidFiles.length} of ${analysisData.files.length})`, + { analysisFile, invalidFileCount: invalidFiles.length }, + [ + "Each file must have 'objectId', 'fileName', and 'fileMd5sum'", + "Check file entries are properly formatted", + "Ensure all required fields are strings", + "Review SONG file schema requirements", + ] + ); + } + + Logger.success`Analysis file structure validated: ${analysisData.analysisType.name} with ${analysisData.files.length} file(s)`; + } catch (error) { + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + throw ErrorFactory.file( + `Error validating analysis file: ${ + error instanceof Error ? error.message : String(error) + }`, + analysisFile, + [ + "Check file permissions and accessibility", + "Verify file is not corrupted", + "Ensure file encoding is UTF-8", + "Try opening the file manually to inspect content", + ] + ); + } + } + + /** + * Enhanced data directory contents validation + */ + private async validateDataDirectoryContents(options: any): Promise { + const dataDir = this.getDataDir(options); + + try { + const files = fs.readdirSync(dataDir); + + if (files.length === 0) { + throw ErrorFactory.file( + `Data directory is empty: ${path.basename(dataDir)}`, + dataDir, + [ + "Add data files to the directory", + "Ensure files match those referenced in analysis file", + "Check if files are in subdirectories", + "Verify file paths are correct", + ] + ); + } + + // Check for common data file types + const dataFiles = files.filter((file) => { + const ext = path.extname(file).toLowerCase(); + return [ + ".vcf", + ".bam", + ".fastq", + ".fq", + ".sam", + ".cram", + ".bed", + ".txt", + ".tsv", + ".csv", + ].includes(ext); + }); + + if (dataFiles.length === 0) { + Logger.warn`No common data file types found in directory`; + Logger.tipString( + "Ensure data files match those referenced in your analysis file" + ); + } else { + Logger.debug`Found ${dataFiles.length} data file(s) in directory`; + } + + // Check for large files that might cause issues + const largeFiles = files.filter((file) => { + try { + const filePath = path.join(dataDir, file); + const stats = fs.statSync(filePath); + return stats.size > 1024 * 1024 * 1024; // 1GB + } catch { + return false; + } + }); + + if (largeFiles.length > 0) { + Logger.warn`Large files detected (>1GB): ${largeFiles.join(", ")}`; + Logger.tipString("Large files may take longer to upload and process"); + } + } catch (error) { + throw ErrorFactory.file( + `Error reading data directory: ${ + error instanceof Error ? error.message : String(error) + }`, + dataDir, + [ + "Check directory permissions", + "Ensure directory is accessible", + "Verify directory is not corrupted", + ] + ); + } + } + + /** + * Extract workflow parameters with validation */ private extractWorkflowParams(options: any): SongScoreWorkflowParams { const analysisFile = this.getAnalysisFile(options)!; @@ -141,7 +569,7 @@ export class SongSubmitAnalysisCommand extends Command { private extractServiceConfig(options: any) { return { url: this.getSongUrl(options)!, - timeout: 20000, + timeout: 30000, // 30 seconds for analysis operations retries: 3, authToken: options.authToken || process.env.AUTH_TOKEN || "123", }; @@ -153,7 +581,7 @@ export class SongSubmitAnalysisCommand extends Command { private extractScoreConfig(options: any) { return { url: this.getScoreUrl(options), - timeout: 30000, + timeout: 300000, // 5 minutes for file uploads retries: 2, authToken: options.authToken || process.env.AUTH_TOKEN || "123", }; @@ -182,86 +610,79 @@ export class SongSubmitAnalysisCommand extends Command { } /** - * Log workflow information + * Enhanced workflow information logging */ private logWorkflowInfo( params: SongScoreWorkflowParams, songUrl: string, scoreUrl?: string ): void { - Logger.info(`${chalk.bold.cyan("SONG/Score Analysis Workflow:")}`); - Logger.info(`SONG URL: ${songUrl}`); - Logger.info(`Score URL: ${scoreUrl || "http://localhost:8087"}`); - Logger.info(`Study ID: ${params.studyId}`); - Logger.info(`Data Directory: ${params.dataDir}`); - Logger.info(`Manifest File: ${params.manifestFile}`); + Logger.info`${chalk.bold.cyan("SONG/Score Workflow Details:")}`; + Logger.generic(` SONG URL: ${songUrl}`); + Logger.generic(` Score URL: ${scoreUrl || "http://localhost:8087"}`); + Logger.generic(` Study ID: ${params.studyId}`); + Logger.generic(` Data Directory: ${params.dataDir}`); + Logger.generic(` Manifest File: ${params.manifestFile}`); + Logger.generic( + ` Allow Duplicates: ${params.allowDuplicates ? "Yes" : "No"}` + ); + Logger.generic( + ` Ignore Undefined MD5: ${params.ignoreUndefinedMd5 ? "Yes" : "No"}` + ); } /** - * Log successful workflow completion + * Enhanced successful workflow completion logging */ private logSuccess(result: any): void { - Logger.success("SONG/Score workflow completed successfully"); + Logger.success`SONG/Score workflow completed successfully`; Logger.generic(" "); - Logger.generic(chalk.gray(` - Analysis ID: ${result.analysisId}`)); - Logger.generic(chalk.gray(` - Study ID: ${result.studyId}`)); - Logger.generic(chalk.gray(` - Status: ${result.status}`)); - Logger.generic(chalk.gray(` - Manifest File: ${result.manifestFile}`)); + Logger.generic(chalk.gray(` ✓ Analysis ID: ${result.analysisId}`)); + Logger.generic(chalk.gray(` ✓ Study ID: ${result.studyId}`)); + Logger.generic(chalk.gray(` ✓ Status: ${result.status}`)); + Logger.generic(chalk.gray(` ✓ Manifest File: ${result.manifestFile}`)); + Logger.generic( + chalk.gray(` ✓ All Steps Completed: Submission → Upload → Publication`) + ); Logger.generic(" "); + Logger.tipString( + "Analysis is now available in SONG and files are uploaded to Score" + ); } /** - * Log partial success + * Enhanced partial success logging */ private logPartialSuccess(result: any): void { - Logger.warn("SONG/Score workflow completed with partial success"); + Logger.warn`SONG/Score workflow completed with partial success`; Logger.generic(" "); - Logger.generic(chalk.gray(` - Analysis ID: ${result.analysisId}`)); - Logger.generic(chalk.gray(` - Study ID: ${result.studyId}`)); - Logger.generic(chalk.gray(` - Status: ${result.status}`)); - Logger.generic(chalk.gray(` - Steps completed:`)); + Logger.generic(chalk.gray(` ⚠ Analysis ID: ${result.analysisId}`)); + Logger.generic(chalk.gray(` ⚠ Study ID: ${result.studyId}`)); + Logger.generic(chalk.gray(` ⚠ Status: ${result.status}`)); + Logger.generic(chalk.gray(` ⚠ Workflow Steps:`)); Logger.generic( - chalk.gray(` - Submitted: ${result.steps.submitted ? "✓" : "✗"}`) + chalk.gray( + ` - Analysis Submitted: ${result.steps.submitted ? "✓" : "✗"}` + ) ); Logger.generic( - chalk.gray(` - Uploaded: ${result.steps.uploaded ? "✓" : "✗"}`) + chalk.gray(` - Files Uploaded: ${result.steps.uploaded ? "✓" : "✗"}`) ); Logger.generic( - chalk.gray(` - Published: ${result.steps.published ? "✓" : "✗"}`) + chalk.gray( + ` - Analysis Published: ${result.steps.published ? "✓" : "✗"}` + ) ); Logger.generic(" "); - } - - /** - * Handle execution errors - */ - private handleExecutionError(error: unknown): CommandResult { - if (error instanceof ConductorError) { - // Add context-specific help - if (error.code === ErrorCodes.FILE_NOT_FOUND) { - Logger.info("\nFile or directory issue. Check paths and permissions."); - } else if (error.code === ErrorCodes.CONNECTION_ERROR) { - Logger.info("\nConnection error. Check service availability."); - } - if (error.details?.suggestion) { - Logger.tip(error.details.suggestion); - } - - return { - success: false, - errorMessage: error.message, - errorCode: error.code, - details: error.details, - }; + if (!result.steps.uploaded) { + Logger.tipString( + "Analysis was submitted but file upload failed - check Score service and file accessibility" + ); + } else if (!result.steps.published) { + Logger.tipString( + "Analysis and files are ready but publication failed - try running songPublishAnalysis command" + ); } - - const errorMessage = error instanceof Error ? error.message : String(error); - return { - success: false, - errorMessage: `SONG/Score workflow failed: ${errorMessage}`, - errorCode: ErrorCodes.CONNECTION_ERROR, - details: { originalError: error }, - }; } } diff --git a/apps/conductor/src/commands/songUploadSchemaCommand.ts b/apps/conductor/src/commands/songUploadSchemaCommand.ts index 3dfb325a..bbe43944 100644 --- a/apps/conductor/src/commands/songUploadSchemaCommand.ts +++ b/apps/conductor/src/commands/songUploadSchemaCommand.ts @@ -1,16 +1,20 @@ -// src/commands/songUploadSchemaCommand.ts -import { Command, CommandResult } from "./baseCommand"; +/** + * SONG Upload Schema Command + * + * Command for uploading schemas to the SONG service. + * Enhanced with ErrorFactory patterns for consistent error handling. + */ + +import { Command } from "./baseCommand"; import { CLIOutput } from "../types/cli"; import { Logger } from "../utils/logger"; -import chalk from "chalk"; -import { ConductorError, ErrorCodes } from "../utils/errors"; -import { SongService } from "../services/song-score"; -import { SongSchemaUploadParams } from "../services/song-score/types"; -import * as fs from "fs"; +import { ErrorFactory } from "../utils/errors"; +import { SongService } from "../services/song-score/songService"; +import { ServiceConfig } from "../services/base/types"; /** - * Command for uploading schemas to the SONG service - * Refactored to use the new SongService + * Command for uploading schemas to SONG service + * Enhanced with comprehensive validation and error handling */ export class SongUploadSchemaCommand extends Command { constructor() { @@ -18,193 +22,227 @@ export class SongUploadSchemaCommand extends Command { } /** - * Override validation since we don't use filePaths for this command + * Enhanced validation with specific error messages for each parameter */ protected async validate(cliOutput: CLIOutput): Promise { const { options } = cliOutput; - // Get schema file from various sources - const schemaFile = this.getSchemaFile(options); + Logger.debug`Validating SONG schema upload parameters`; - if (!schemaFile) { - throw new ConductorError( - "Schema file not specified. Use --schema-file or set SONG_SCHEMA environment variable.", - ErrorCodes.INVALID_ARGS - ); - } - - // Validate file exists and is readable - if (!fs.existsSync(schemaFile)) { - throw new ConductorError( - `Schema file not found: ${schemaFile}`, - ErrorCodes.FILE_NOT_FOUND - ); - } + // Enhanced validation for each required parameter + this.validateSongUrl(options); + this.validateSchemaFile(options); + this.validateOptionalParameters(options); - // Validate SONG URL - const songUrl = this.getSongUrl(options); - if (!songUrl) { - throw new ConductorError( - "SONG URL not specified. Use --song-url or set SONG_URL environment variable.", - ErrorCodes.INVALID_ARGS - ); - } + Logger.successString("SONG schema upload parameters validated"); } /** - * Executes the SONG schema upload process + * Enhanced execution with detailed logging and error handling */ - protected async execute(cliOutput: CLIOutput): Promise { + protected async execute(cliOutput: CLIOutput): Promise { const { options } = cliOutput; - try { - // Extract configuration - const schemaFile = this.getSchemaFile(options)!; - const serviceConfig = this.extractServiceConfig(options); - const uploadParams = this.extractUploadParams(schemaFile); - - // Create service instance - const songService = new SongService(serviceConfig); - - // Check service health - const healthResult = await songService.checkHealth(); - if (!healthResult.healthy) { - throw new ConductorError( - `SONG service is not healthy: ${ - healthResult.message || "Unknown error" - }`, - ErrorCodes.CONNECTION_ERROR, - { healthResult } - ); - } + // Extract validated configuration + const serviceConfig = this.extractServiceConfig(options); + const schemaFile = options.schemaFile || process.env.SONG_SCHEMA; + + Logger.info`Starting SONG schema upload`; + Logger.info`Schema file: ${schemaFile}`; + Logger.info`SONG URL: ${serviceConfig.url}`; + + // Create service instance with enhanced error handling + const songService = new SongService(serviceConfig); + + // Enhanced health check with specific feedback + Logger.info`Checking SONG service health...`; + const healthResult = await songService.checkHealth(); + if (!healthResult.healthy) { + throw ErrorFactory.connection( + "SONG service health check failed", + "SONG", + serviceConfig.url, + [ + "Check that SONG service is running", + `Verify service URL: ${serviceConfig.url}`, + "Check network connectivity and firewall settings", + "Review SONG service logs for errors", + `Test manually: curl ${serviceConfig.url}/health`, + healthResult.message + ? `Health check message: ${healthResult.message}` + : "", + ] + ); + } - // Log upload info - this.logUploadInfo(schemaFile, serviceConfig.url); + Logger.success`SONG service is healthy`; - // Upload schema - much simpler now! - const result = await songService.uploadSchema(uploadParams); + // Upload schema with enhanced error handling + Logger.info`Uploading schema to SONG...`; + const uploadResult = await songService.uploadSchema(schemaFile); - // Log success - this.logSuccess(result); + // Enhanced success logging + this.logUploadSuccess(uploadResult, schemaFile); - return { - success: true, - details: result, - }; - } catch (error) { - return this.handleExecutionError(error); - } + // Command completed successfully } /** - * Get schema file from various sources + * Enhanced SONG URL validation */ - private getSchemaFile(options: any): string | undefined { - return options.schemaFile || process.env.SONG_SCHEMA; - } + private validateSongUrl(options: any): void { + const songUrl = options.songUrl || process.env.SONG_URL; - /** - * Get SONG URL from various sources - */ - private getSongUrl(options: any): string | undefined { - return options.songUrl || process.env.SONG_URL; - } + if (!songUrl) { + throw ErrorFactory.config("SONG service URL not configured", "songUrl", [ + "Set SONG URL: conductor songUploadSchema --song-url http://localhost:8080", + "Set SONG_URL environment variable", + "Verify SONG service is running and accessible", + "Check network connectivity to SONG service", + "Default SONG port is usually 8080", + ]); + } - /** - * Extract service configuration from options - */ - private extractServiceConfig(options: any) { - return { - url: this.getSongUrl(options)!, - timeout: 10000, - retries: 3, - authToken: options.authToken || process.env.AUTH_TOKEN || "123", - }; + try { + const url = new URL(songUrl); + if (!["http:", "https:"].includes(url.protocol)) { + throw ErrorFactory.validation( + `Invalid protocol in SONG URL: ${url.protocol}`, + { songUrl, protocol: url.protocol }, + [ + "Protocol must be http or https", + "Use format: http://localhost:8080 or https://song.example.com", + "Check for typos in the URL", + "Verify the correct protocol with your administrator", + ] + ); + } + Logger.debug`Using SONG URL: ${songUrl}`; + } catch (error) { + if (error instanceof Error && error.name === "ConductorError") { + throw error; // Re-throw enhanced errors + } + + throw ErrorFactory.config( + `Invalid SONG URL format: ${songUrl}`, + "songUrl", + [ + "Use a valid URL format: http://localhost:8080", + "Include protocol (http:// or https://)", + "Check for typos in the URL", + "Verify port number is correct (usually 8080 for SONG)", + "Ensure proper URL encoding for special characters", + ] + ); + } } /** - * Extract upload parameters from schema file + * Enhanced schema file validation */ - private extractUploadParams(schemaFile: string): SongSchemaUploadParams { - try { - Logger.info(`Reading schema file: ${schemaFile}`); - const schemaContent = fs.readFileSync(schemaFile, "utf-8"); + private validateSchemaFile(options: any): void { + const schemaFile = options.schemaFile || process.env.SONG_SCHEMA; - return { - schemaContent, - }; - } catch (error) { - throw new ConductorError( - `Error reading schema file: ${ - error instanceof Error ? error.message : String(error) - }`, - ErrorCodes.FILE_ERROR, - error + if (!schemaFile) { + throw ErrorFactory.args( + "Schema file not specified for upload", + "songUploadSchema", + [ + "Provide schema file: conductor songUploadSchema --schema-file schema.json", + "Set SONG_SCHEMA environment variable", + "Schema file should be in JSON format", + "Ensure file contains valid SONG schema definition", + "Check schema documentation for format requirements", + ] ); } + + if (typeof schemaFile !== "string" || schemaFile.trim() === "") { + throw ErrorFactory.validation( + "Invalid schema file path", + { schemaFile }, + [ + "Schema file path must be a non-empty string", + "Use absolute or relative path to schema file", + "Check for typos in file path", + "Ensure file exists and is readable", + ] + ); + } + + // Basic file extension check + if (!schemaFile.toLowerCase().endsWith(".json")) { + Logger.warn`Schema file does not have .json extension: ${schemaFile}`; + Logger.tipString("SONG schemas are typically JSON files"); + } + + Logger.debug`Schema file validated: ${schemaFile}`; } /** - * Log upload information + * Validate optional parameters with helpful guidance */ - private logUploadInfo(schemaFile: string, serviceUrl: string): void { - Logger.info(`${chalk.bold.cyan("Uploading Schema to SONG:")}`); - Logger.info(`URL: ${serviceUrl}/schemas`); - Logger.info(`Schema File: ${schemaFile}`); + private validateOptionalParameters(options: any): void { + // Validate auth token if provided + const authToken = options.authToken || process.env.AUTH_TOKEN; + if (authToken && typeof authToken === "string" && authToken.trim() === "") { + Logger.warn`Empty auth token provided - using default authentication`; + } + + // Validate other optional parameters as needed + Logger.debug`Optional parameters validated`; } /** - * Log successful upload + * Extract service configuration from options */ - private logSuccess(result: any): void { - Logger.success("Schema uploaded successfully"); - Logger.generic(" "); - Logger.generic(chalk.gray(` - Schema ID: ${result.id || "N/A"}`)); - Logger.generic( - chalk.gray(` - Schema Name: ${result.name || "Unnamed"}`) - ); - Logger.generic( - chalk.gray(` - Schema Version: ${result.version || "N/A"}`) - ); - Logger.generic(" "); + private extractServiceConfig(options: any): ServiceConfig { + const songUrl = options.songUrl || process.env.SONG_URL; + const authToken = options.authToken || process.env.AUTH_TOKEN || "123"; + + return { + url: songUrl, + authToken, + timeout: 30000, // 30 second timeout + retries: 3, + }; } /** - * Handle execution errors with helpful user feedback + * Enhanced success logging with upload details */ - private handleExecutionError(error: unknown): CommandResult { - if (error instanceof ConductorError) { - // Add context-specific help for common SONG errors - if (error.code === ErrorCodes.VALIDATION_FAILED) { - Logger.info("\nSchema validation failed. Check your schema structure."); - Logger.tip( - 'Ensure your schema has required fields: "name" and "schema"' - ); - } else if (error.code === ErrorCodes.FILE_NOT_FOUND) { - Logger.info("\nSchema file not found. Check the file path."); - } else if (error.code === ErrorCodes.CONNECTION_ERROR) { - Logger.info("\nConnection error. Check SONG service availability."); + private logUploadSuccess(uploadResult: any, schemaFile: string): void { + Logger.success`Schema uploaded successfully to SONG`; + + // Log upload details if available + if (uploadResult) { + if (uploadResult.schemaId) { + Logger.info`Schema ID: ${uploadResult.schemaId}`; } - if (error.details?.suggestion) { - Logger.tip(error.details.suggestion); + if (uploadResult.version) { + Logger.info`Schema version: ${uploadResult.version}`; } - return { - success: false, - errorMessage: error.message, - errorCode: error.code, - details: error.details, - }; + if (uploadResult.name) { + Logger.info`Schema name: ${uploadResult.name}`; + } + + if (uploadResult.description) { + Logger.info`Description: ${uploadResult.description}`; + } } - // Handle unexpected errors - const errorMessage = error instanceof Error ? error.message : String(error); - return { - success: false, - errorMessage: `Schema upload failed: ${errorMessage}`, - errorCode: ErrorCodes.CONNECTION_ERROR, - details: { originalError: error }, - }; + // Summary information + Logger.section("Upload Summary"); + Logger.info`Source file: ${schemaFile}`; + Logger.info`Upload timestamp: ${new Date().toISOString()}`; + + Logger.tipString( + "Schema is now available for use in SONG studies and analyses" + ); + Logger.tipString( + "Use 'songCreateStudy' command to create studies with this schema" + ); } } diff --git a/apps/conductor/src/commands/uploadCsvCommand.ts b/apps/conductor/src/commands/uploadCsvCommand.ts index c1a0a409..8f746bf4 100644 --- a/apps/conductor/src/commands/uploadCsvCommand.ts +++ b/apps/conductor/src/commands/uploadCsvCommand.ts @@ -2,14 +2,15 @@ * Upload Command * * Command implementation for uploading CSV data to Elasticsearch. + * Enhanced with ErrorFactory and improved user feedback. */ import { validateBatchSize } from "../validations/elasticsearchValidator"; import { validateDelimiter } from "../validations/utils"; -import { Command, CommandResult } from "./baseCommand"; +import { Command } from "./baseCommand"; import { CLIOutput } from "../types/cli"; import { Logger } from "../utils/logger"; -import { ConductorError, ErrorCodes } from "../utils/errors"; +import { ErrorFactory } from "../utils/errors"; import { createClientFromConfig, validateConnection, @@ -37,12 +38,11 @@ export class UploadCommand extends Command { /** * Executes the upload process for all specified files * @param cliOutput The CLI configuration and inputs - * @returns Promise with success/failure information */ - protected async execute(cliOutput: CLIOutput): Promise { + protected async execute(cliOutput: CLIOutput): Promise { const { config, filePaths } = cliOutput; - Logger.info(`Input files specified: ${filePaths.length}`, filePaths); + Logger.info`Starting CSV upload process for ${filePaths.length} file(s)`; // Process each file let successCount = 0; @@ -50,168 +50,349 @@ export class UploadCommand extends Command { const failureDetails: Record = {}; for (const filePath of filePaths) { - Logger.debug(`Processing File: ${filePath}`); + Logger.debug`Processing file: ${filePath}`; try { await this.processFile(filePath, config); - Logger.debug(`Successfully processed ${filePath}`); + Logger.success`Successfully processed: ${path.basename(filePath)}`; successCount++; } catch (error) { failureCount++; - // Log the error but continue to the next file - if (error instanceof ConductorError) { - Logger.debug( - `Skipping file '${filePath}': [${error.code}] ${error.message}` - ); - if (error.details) { - Logger.debug(`Error details: ${JSON.stringify(error.details)}`); - } - failureDetails[filePath] = { - code: error.code, - message: error.message, - details: error.details, - }; - } else if (error instanceof Error) { - Logger.debug(`Skipping file '${filePath}': ${error.message}`); + const fileName = path.basename(filePath); + + // Enhanced error logging with file context + if (error instanceof Error) { + Logger.error`Failed to process ${fileName}: ${error.message}`; failureDetails[filePath] = { + fileName, message: error.message, + suggestion: "Check file format and Elasticsearch connectivity", }; } else { - Logger.debug(`Skipping file '${filePath}' due to an error`); + Logger.error`Failed to process ${fileName} due to unknown error`; failureDetails[filePath] = { - message: "Unknown error", + fileName, + message: "Unknown error occurred", }; } } } - // Return the CommandResult + // Enhanced result reporting with error throwing if (failureCount === 0) { - return { - success: true, - details: { - filesProcessed: successCount, - }, - }; + Logger.success`All ${successCount} file(s) processed successfully`; + // Success - method completes normally } else if (successCount === 0) { - return { - success: false, - errorMessage: `Failed to process all ${failureCount} files`, - errorCode: ErrorCodes.VALIDATION_FAILED, - details: failureDetails, - }; - } else { - // Partial success - return { - success: true, - details: { - filesProcessed: successCount, - filesFailed: failureCount, + // Throw error with suggestions instead of returning failure result + throw ErrorFactory.validation( + `Failed to process ${failureCount} file(s)`, + { + totalFiles: filePaths.length, failureDetails, - }, - }; + suggestions: [ + "Check that files exist and are readable", + "Verify CSV format and headers", + "Ensure Elasticsearch is accessible", + "Use --debug for detailed error information", + ], + } + ); + } else { + // Partial success - log warning but don't fail + Logger.warn`Processed ${successCount} of ${filePaths.length} files successfully`; + Logger.infoString(`${failureCount} files failed - see details above`); + + // For partial success, we could either succeed or fail depending on requirements + // Here we'll succeed but warn about partial failures + Logger.tipString( + "Some files failed to process - check error details above" + ); } } /** * Validates command line arguments and configuration * @param cliOutput The CLI configuration and inputs - * @throws ConductorError if validation fails + * @throws Enhanced errors with specific guidance */ protected async validate(cliOutput: CLIOutput): Promise { const { config, filePaths } = cliOutput; - // Validate files first + // Enhanced file validation + if (!filePaths || filePaths.length === 0) { + throw ErrorFactory.args("No CSV files specified for upload", "upload", [ + "Provide one or more CSV files: conductor upload -f data.csv", + "Use wildcards for multiple files: conductor upload -f *.csv", + "Specify multiple files: conductor upload -f file1.csv file2.csv", + ]); + } + + Logger.debug`Validating ${filePaths.length} input file(s)`; + + // Validate files with enhanced error messages const fileValidationResult = await validateFiles(filePaths); if (!fileValidationResult.valid) { - throw new ConductorError("Invalid input files", ErrorCodes.INVALID_FILE, { - errors: fileValidationResult.errors, - }); + const invalidFiles = filePaths.filter((fp) => !fs.existsSync(fp)); + const nonCsvFiles = filePaths.filter( + (fp) => + fs.existsSync(fp) && + !fp.toLowerCase().endsWith(".csv") && + !fp.toLowerCase().endsWith(".tsv") + ); + + const suggestions = [ + "Check that file paths are correct", + "Ensure files exist and are readable", + ]; + + if (invalidFiles.length > 0) { + suggestions.push( + `Missing files: ${invalidFiles + .map((f) => path.basename(f)) + .join(", ")}` + ); + } + + if (nonCsvFiles.length > 0) { + suggestions.push("Only CSV and TSV files are supported"); + suggestions.push( + `Invalid extensions found: ${nonCsvFiles + .map((f) => path.extname(f)) + .join(", ")}` + ); + } + + suggestions.push(`Current directory: ${process.cwd()}`); + + throw ErrorFactory.file( + "Invalid or missing input files", + filePaths[0], + suggestions + ); } - // Validate delimiter + // Enhanced delimiter validation try { validateDelimiter(config.delimiter); + Logger.debug`Using delimiter: '${config.delimiter}'`; } catch (error) { - throw new ConductorError( - "Invalid delimiter", - ErrorCodes.VALIDATION_FAILED, - error + throw ErrorFactory.config( + "Invalid CSV delimiter specified", + "delimiter", + [ + "Use a single character delimiter (comma, tab, semicolon, etc.)", + "Common delimiters: ',' (comma), '\\t' (tab), ';' (semicolon)", + "Example: conductor upload -f data.csv --delimiter ';'", + ] ); } - // Validate batch size + // Enhanced batch size validation try { validateBatchSize(config.batchSize); + Logger.debug`Using batch size: ${config.batchSize}`; } catch (error) { - throw new ConductorError( - "Invalid batch size", - ErrorCodes.VALIDATION_FAILED, - error - ); + throw ErrorFactory.config("Invalid batch size specified", "batchSize", [ + "Use a positive number between 1 and 10000", + "Recommended values: 500-2000 for most files", + "Smaller batches for large documents, larger for simple data", + "Example: conductor upload -f data.csv --batch-size 1000", + ]); } - // Validate each file's CSV headers + // Enhanced CSV header validation for each file for (const filePath of filePaths) { - await this.validateFileHeaders(filePath, config.delimiter); + try { + await this.validateFileHeaders(filePath, config.delimiter); + Logger.debug`Validated headers for: ${path.basename(filePath)}`; + } catch (error) { + if (error instanceof Error) { + throw ErrorFactory.csv( + `Invalid CSV structure in file: ${path.basename(filePath)}`, + filePath, + undefined, + [ + "Check that the first row contains valid column headers", + "Ensure headers use only letters, numbers, and underscores", + "Remove special characters from column names", + `Verify delimiter '${config.delimiter}' is correct for this file`, + "Check file encoding (should be UTF-8)", + ] + ); + } + throw error; + } } + + Logger.debugString("Input validation completed"); } /** - * Validates headers for a single file + * Validates headers for a single file with enhanced error context */ private async validateFileHeaders( filePath: string, delimiter: string ): Promise { try { + if (!fs.existsSync(filePath)) { + throw ErrorFactory.file( + `CSV file not found: ${path.basename(filePath)}`, + filePath + ); + } + const fileContent = fs.readFileSync(filePath, "utf-8"); - const [headerLine] = fileContent.split("\n"); + const lines = fileContent.split("\n"); - if (!headerLine) { - throw new ConductorError( - `CSV file is empty or has no headers: ${filePath}`, - ErrorCodes.INVALID_FILE + if (lines.length === 0 || !lines[0].trim()) { + throw ErrorFactory.csv( + `CSV file is empty or has no headers: ${path.basename(filePath)}`, + filePath, + 1, + [ + "Ensure the file contains data", + "Check that the first row has column headers", + "Verify file is not corrupted", + ] ); } + const headerLine = lines[0]; const parseResult = parseCSVLine(headerLine, delimiter, true); - if (!parseResult || !parseResult[0]) { - throw new ConductorError( - `Failed to parse CSV headers: ${filePath}`, - ErrorCodes.PARSING_ERROR + + if (!parseResult || !parseResult[0] || parseResult[0].length === 0) { + throw ErrorFactory.csv( + `Failed to parse CSV headers in: ${path.basename(filePath)}`, + filePath, + 1, + [ + `Check that delimiter '${delimiter}' is correct for this file`, + "Ensure headers are properly formatted", + "Verify file encoding (should be UTF-8)", + "Try a different delimiter if needed: --delimiter ';' or --delimiter '\\t'", + ] ); } const headers = parseResult[0]; + Logger.debug`Found ${headers.length} headers in ${path.basename( + filePath + )}`; // Validate CSV structure using our validation function await validateCSVStructure(headers); } catch (error) { - if (error instanceof ConductorError) { - // Rethrow ConductorErrors + if (error instanceof Error && error.name === "ConductorError") { + // Re-throw our enhanced errors as-is throw error; } - throw new ConductorError( - `Error validating CSV headers: ${filePath}`, - ErrorCodes.VALIDATION_FAILED, - error + + // Wrap other errors with enhanced context + throw ErrorFactory.csv( + `Error validating CSV headers: ${ + error instanceof Error ? error.message : String(error) + }`, + filePath, + 1, + [ + "Check file format and structure", + "Ensure proper CSV formatting", + "Verify file is not corrupted", + "Try opening the file in a text editor to inspect manually", + ] ); } } /** - * Processes a single file + * Processes a single file with enhanced error handling */ private async processFile(filePath: string, config: any): Promise { - // Set up Elasticsearch client - const client = createClientFromConfig(config); + const fileName = path.basename(filePath); + + try { + Logger.info`Processing: ${fileName}`; + + // Set up Elasticsearch client with enhanced error handling + let client; + try { + client = createClientFromConfig(config); + Logger.debug`Created Elasticsearch client for ${config.elasticsearch.url}`; + } catch (error) { + throw ErrorFactory.connection( + "Failed to create Elasticsearch client", + "Elasticsearch", + config.elasticsearch.url, + [ + "Check Elasticsearch URL format", + "Verify authentication credentials", + "Ensure Elasticsearch is running", + `Test connection: curl ${config.elasticsearch.url}`, + ] + ); + } - // Validate Elasticsearch connection and index - await validateConnection(client); - await validateIndex(client, config.elasticsearch.index); + // Validate connection with enhanced error handling + try { + await validateConnection(client); + Logger.debug`Validated connection to Elasticsearch`; + } catch (error) { + throw ErrorFactory.connection( + "Cannot connect to Elasticsearch", + "Elasticsearch", + config.elasticsearch.url, + [ + "Check that Elasticsearch is running and accessible", + "Verify network connectivity", + "Confirm authentication credentials are correct", + "Check firewall and security group settings", + `Test manually: curl ${config.elasticsearch.url}/_cluster/health`, + ] + ); + } - // Process the file - await processCSVFile(filePath, config, client); + // Validate index with enhanced error handling + try { + await validateIndex(client, config.elasticsearch.index); + Logger.debug`Validated index: ${config.elasticsearch.index}`; + } catch (error) { + throw ErrorFactory.index( + `Target index '${config.elasticsearch.index}' is not accessible`, + config.elasticsearch.index, + [ + `Create the index first: PUT /${config.elasticsearch.index}`, + "Check index permissions and mappings", + "Verify index name is correct", + `List available indices: GET /_cat/indices`, + "Use a different index name with --index parameter", + ] + ); + } + + // Process the file with enhanced progress tracking + Logger.info`Uploading data from ${fileName} to index '${config.elasticsearch.index}'`; + await processCSVFile(filePath, config, client); + } catch (error) { + // Add file context to any errors that bubble up + if (error instanceof Error && error.name === "ConductorError") { + // Re-throw our enhanced errors + throw error; + } + + // Wrap unexpected errors with file context + throw ErrorFactory.file( + `Failed to process CSV file: ${ + error instanceof Error ? error.message : String(error) + }`, + filePath, + [ + "Check file format and content", + "Verify Elasticsearch connectivity", + "Ensure sufficient permissions", + "Use --debug flag for detailed error information", + ] + ); + } } } diff --git a/apps/conductor/src/config/serviceConfigManager.ts b/apps/conductor/src/config/serviceConfigManager.ts deleted file mode 100644 index 5eef87e8..00000000 --- a/apps/conductor/src/config/serviceConfigManager.ts +++ /dev/null @@ -1,212 +0,0 @@ -// src/config/ServiceConfigManager.ts -/** - * Unified service configuration management - * Replaces scattered config objects throughout commands and services - */ - -import { Environment } from "./environment"; -import { ServiceConfig } from "../services/base/types"; - -interface StandardServiceConfig extends ServiceConfig { - name: string; - retries: number; - retryDelay: number; -} - -interface ElasticsearchConfig extends StandardServiceConfig { - user: string; - password: string; - index: string; - batchSize: number; - delimiter: string; -} - -interface FileServiceConfig extends StandardServiceConfig { - dataDir: string; - outputDir: string; - manifestFile?: string; -} - -interface LyricConfig extends StandardServiceConfig { - categoryId: string; - organization: string; - maxRetries: number; - retryDelay: number; -} - -export class ServiceConfigManager { - /** - * Create Elasticsearch configuration - */ - static createElasticsearchConfig( - overrides: Partial = {} - ): ElasticsearchConfig { - const env = Environment.services.elasticsearch; - const defaults = Environment.defaults.elasticsearch; - - return { - name: "Elasticsearch", - url: env.url, - authToken: undefined, // ES uses user/password - timeout: Environment.defaults.timeouts.default, - retries: 3, - retryDelay: 1000, - user: env.user, - password: env.password, - index: defaults.index, - batchSize: defaults.batchSize, - delimiter: defaults.delimiter, - ...overrides, - }; - } - - /** - * Create Lectern service configuration - */ - static createLecternConfig( - overrides: Partial = {} - ): StandardServiceConfig { - const env = Environment.services.lectern; - - return { - name: "Lectern", - url: env.url, - authToken: env.authToken, - timeout: Environment.defaults.timeouts.default, - retries: 3, - retryDelay: 1000, - ...overrides, - }; - } - - /** - * Create Lyric service configuration - */ - static createLyricConfig(overrides: Partial = {}): LyricConfig { - const env = Environment.services.lyric; - const defaults = Environment.defaults.lyric; - - return { - name: "Lyric", - url: env.url, - authToken: undefined, - timeout: Environment.defaults.timeouts.upload, // Longer timeout for uploads - retries: 3, - retryDelay: defaults.retryDelay, // Use the environment default - categoryId: env.categoryId, - organization: env.organization, - maxRetries: defaults.maxRetries, - ...overrides, - }; - } - - /** - * Create SONG service configuration - */ - static createSongConfig( - overrides: Partial = {} - ): StandardServiceConfig { - const env = Environment.services.song; - - return { - name: "SONG", - url: env.url, - authToken: env.authToken, - timeout: Environment.defaults.timeouts.upload, - retries: 3, - retryDelay: 1000, - ...overrides, - }; - } - - /** - * Create Score service configuration - */ - static createScoreConfig( - overrides: Partial = {} - ): StandardServiceConfig { - const env = Environment.services.score; - - return { - name: "Score", - url: env.url, - authToken: env.authToken, - timeout: Environment.defaults.timeouts.upload, - retries: 2, // Lower retries for file uploads - retryDelay: 2000, - ...overrides, - }; - } - - /** - * Create Maestro service configuration - */ - static createMaestroConfig( - overrides: Partial = {} - ): StandardServiceConfig { - const env = Environment.services.maestro; - - return { - name: "Maestro", - url: env.url, - authToken: undefined, - timeout: Environment.defaults.timeouts.default, - retries: 3, - retryDelay: 1000, - ...overrides, - }; - } - - /** - * Create file service configuration (for commands that handle files) - */ - static createFileServiceConfig( - baseConfig: StandardServiceConfig, - fileOptions: Partial = {} - ): FileServiceConfig { - return { - ...baseConfig, - dataDir: fileOptions.dataDir || "./data", - outputDir: fileOptions.outputDir || "./output", - manifestFile: fileOptions.manifestFile, - ...fileOptions, - }; - } - - /** - * Validate service configuration - */ - static validateConfig(config: StandardServiceConfig): void { - if (!config.url) { - throw new Error(`Missing URL for ${config.name} service`); - } - - if (config.timeout && config.timeout < 1000) { - throw new Error( - `Timeout too low for ${config.name} service (minimum 1000ms)` - ); - } - - if (config.retries && config.retries < 0) { - throw new Error(`Invalid retries value for ${config.name} service`); - } - } - - /** - * Get all configured services status - */ - static getServicesOverview() { - const env = Environment.services; - return { - elasticsearch: { - url: env.elasticsearch.url, - configured: !!env.elasticsearch.url, - }, - lectern: { url: env.lectern.url, configured: !!env.lectern.url }, - lyric: { url: env.lyric.url, configured: !!env.lyric.url }, - song: { url: env.song.url, configured: !!env.song.url }, - score: { url: env.score.url, configured: !!env.score.url }, - maestro: { url: env.maestro.url, configured: !!env.maestro.url }, - }; - } -} diff --git a/apps/conductor/src/main.ts b/apps/conductor/src/main.ts index dbbb5937..a63563c6 100644 --- a/apps/conductor/src/main.ts +++ b/apps/conductor/src/main.ts @@ -1,88 +1,29 @@ #!/usr/bin/env node -// src/main.ts - Simplified main entry point import { setupCLI } from "./cli"; import { CommandRegistry } from "./commands/commandRegistry"; -import { Environment } from "./config/environment"; -import { ConductorError, ErrorCodes, handleError } from "./utils/errors"; +import { handleError } from "./utils/errors"; import { Logger } from "./utils/logger"; -import chalk from "chalk"; - -// Add global unhandled rejection handler -process.on("unhandledRejection", (reason, promise) => { - console.error("Unhandled Rejection at:", promise, "reason:", reason); -}); async function main() { try { - // Initialize environment and logging - if (Environment.isDebug) { + // Enable debug mode from environment BEFORE any logging + if (process.argv.includes("--debug")) { Logger.enableDebug(); } - Logger.header(`Conductor: Data Processing Pipeline`); - Logger.info(chalk.grey.italic` Version: 1.0.0`); - Logger.generic(" "); - - // Setup CLI and get parsed arguments const cliOutput = await setupCLI(); + Logger.debug`Version: 1.0.0`; + Logger.debug`Profile: ${cliOutput.profile}`; - Logger.info(chalk.grey.italic` Profile: ${cliOutput.profile}`); - Logger.generic(" "); + // Initialize other logger settings (not debug mode again) Logger.initialize(); Logger.debug`Starting CLI setup`; - Logger.debug`Creating command instance`; - - // Use the simplified command registry - const command = CommandRegistry.createCommand(cliOutput.profile); - - Logger.debug`Running command`; - - // Execute the command - const result = await command.run(cliOutput); - - // Check command result and handle errors - if (!result.success) { - throw new ConductorError( - result.errorMessage || "Command execution failed", - result.errorCode || ErrorCodes.UNKNOWN_ERROR, - result.details - ); - } - - Logger.success(`Command '${cliOutput.profile}' completed successfully`); + Logger.debug`Executing command via registry`; + await CommandRegistry.execute(cliOutput.profile, cliOutput); } catch (error) { - // Enhanced error handling with helpful context - if (Environment.isDebug) { - console.error("FATAL ERROR:", error); - } - - // Special handling for unknown commands - if (error instanceof Error && error.message.includes("Unknown command")) { - Logger.error(error.message); - Logger.generic(""); - CommandRegistry.displayHelp(); - process.exit(1); - } - - // Let the handleError function handle other errors - handleError(error); + handleError(error, () => {}); } } - -// Enhanced error handling for uncaught errors -main().catch((error) => { - if (Environment.isDebug) { - console.error("UNCAUGHT ERROR IN MAIN:", error); - } - - // Try to provide helpful information even for uncaught errors - if (error instanceof Error && error.message.includes("command")) { - Logger.error("Command execution failed"); - Logger.tip("Use --debug flag for detailed error information"); - CommandRegistry.displayHelp(); - } else { - handleError(error); - } -}); +main().catch(handleError); diff --git a/apps/conductor/src/services/base/HttpService.ts b/apps/conductor/src/services/base/HttpService.ts index 0e6ff4ab..94e61510 100644 --- a/apps/conductor/src/services/base/HttpService.ts +++ b/apps/conductor/src/services/base/HttpService.ts @@ -1,7 +1,7 @@ -// src/services/base/HttpService.ts +// src/services/base/HttpService.ts - Fixed Logger calls import axios from "axios"; import { Logger } from "../../utils/logger"; -import { ConductorError, ErrorCodes } from "../../utils/errors"; +import { ErrorFactory, ErrorCodes } from "../../utils/errors"; import { ServiceConfig, RequestOptions, ServiceResponse } from "./types"; export class HttpService { @@ -10,6 +10,10 @@ export class HttpService { constructor(config: ServiceConfig) { this.config = config; + + // Enhanced configuration validation + this.validateConfig(config); + this.client = axios.create({ baseURL: config.url, timeout: config.timeout || 10000, @@ -60,6 +64,58 @@ export class HttpService { return this.makeRequest("DELETE", endpoint, undefined, options); } + /** + * Enhanced configuration validation + */ + private validateConfig(config: ServiceConfig): void { + if (!config.url) { + throw ErrorFactory.config( + "Service URL is required for HTTP client configuration", + "url", + [ + "Provide a valid service URL", + "Check service configuration", + "Verify environment variables are set", + ] + ); + } + + try { + const url = new URL(config.url); + if (!["http:", "https:"].includes(url.protocol)) { + throw ErrorFactory.config( + `Invalid service URL protocol: ${url.protocol}`, + "url", + [ + "Use HTTP or HTTPS protocol", + "Example: http://localhost:8080", + "Example: https://api.service.com", + ] + ); + } + } catch (error) { + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + throw ErrorFactory.config( + `Invalid service URL format: ${config.url}`, + "url", + [ + "Use a valid URL format with protocol", + "Example: http://localhost:8080", + "Check for typos in the URL", + ] + ); + } + + if (config.timeout && (config.timeout < 1000 || config.timeout > 300000)) { + Logger.warn`Timeout value ${config.timeout}ms is outside recommended range (1000-300000ms)`; + } + } + + /** + * Enhanced request method with better error handling and retry logic + */ private async makeRequest( method: string, endpoint: string, @@ -80,7 +136,7 @@ export class HttpService { for (let attempt = 1; attempt <= maxRetries; attempt++) { try { - Logger.debug( + Logger.debugString( `${method} ${endpoint} (attempt ${attempt}/${maxRetries})` ); @@ -98,16 +154,24 @@ export class HttpService { throw error; } - Logger.warn( - `Request failed, retrying in ${retryDelay}ms... (${attempt}/${maxRetries})` + const backoffDelay = retryDelay * attempt; // Exponential backoff + Logger.warnString( + `Request failed, retrying in ${backoffDelay}ms... (${attempt}/${maxRetries})` ); - await this.delay(retryDelay * attempt); // Exponential backoff + await this.delay(backoffDelay); } } - throw new ConductorError( + throw ErrorFactory.connection( "Request failed after all retries", - ErrorCodes.CONNECTION_ERROR + "HTTP Service", + this.config.url, + [ + "Check service connectivity and availability", + "Verify network connectivity", + "Check service health and status", + "Review request parameters and authentication", + ] ); } @@ -115,63 +179,217 @@ export class HttpService { return token.startsWith("Bearer ") ? token : `Bearer ${token}`; } + /** + * Enhanced error handling with ErrorFactory patterns + */ private handleAxiosError(error: any): never { if (error.response) { // Server responded with error status const status = error.response.status; const data = error.response.data; + const url = error.config?.url || "unknown"; let errorMessage = `HTTP ${status}`; if (data?.message) { errorMessage += `: ${data.message}`; } else if (data?.error) { errorMessage += `: ${data.error}`; + } else if (typeof data === "string") { + errorMessage += `: ${data}`; + } + + // Enhanced error handling based on status codes + if (status === 401) { + throw ErrorFactory.connection( + `Authentication failed: ${errorMessage}`, + "HTTP Service", + this.config.url, + [ + "Check authentication credentials", + "Verify API token is valid and not expired", + "Ensure proper authentication headers", + "Contact service administrator for access", + ] + ); + } else if (status === 403) { + throw ErrorFactory.connection( + `Access forbidden: ${errorMessage}`, + "HTTP Service", + this.config.url, + [ + "You may not have permission for this operation", + "Check user roles and privileges", + "Verify API access permissions", + "Contact administrator for required permissions", + ] + ); + } else if (status === 404) { + throw ErrorFactory.connection( + `Resource not found: ${errorMessage}`, + "HTTP Service", + this.config.url, + [ + "Check the endpoint URL is correct", + "Verify the resource exists", + `Check service is running at: ${this.config.url}`, + "Review API documentation for correct endpoints", + ] + ); + } else if (status === 400) { + throw ErrorFactory.validation( + `Bad request: ${errorMessage}`, + { + status, + responseData: data, + url, + }, + [ + "Check request parameters and format", + "Verify all required fields are provided", + "Review request payload structure", + "Check data types and validation rules", + ] + ); + } else if (status === 422) { + throw ErrorFactory.validation( + `Validation failed: ${errorMessage}`, + { + status, + responseData: data, + url, + }, + [ + "Check input data validation", + "Verify all required fields are present and valid", + "Review data format and constraints", + "Check for conflicting or duplicate data", + ] + ); + } else if (status === 429) { + throw ErrorFactory.connection( + `Rate limit exceeded: ${errorMessage}`, + "HTTP Service", + this.config.url, + [ + "Too many requests sent to the service", + "Wait before retrying the request", + "Consider implementing request throttling", + "Check rate limit policies and quotas", + ] + ); + } else if (status >= 500) { + throw ErrorFactory.connection( + `Server error: ${errorMessage}`, + "HTTP Service", + this.config.url, + [ + "Service is experiencing internal errors", + "Check service health and status", + "Try again later if the service is temporarily down", + "Contact service administrator if problem persists", + ] + ); } - const errorCode = this.getErrorCodeFromStatus(status); - throw new ConductorError(errorMessage, errorCode, { - status, - responseData: data, - url: error.config?.url, - }); + // Generic HTTP error + throw ErrorFactory.connection( + errorMessage, + "HTTP Service", + this.config.url, + [ + "Check request parameters and format", + "Verify service connectivity", + "Review API documentation", + "Check service status and health", + ] + ); } else if (error.request) { // Request made but no response - throw new ConductorError( - "No response received from server", - ErrorCodes.CONNECTION_ERROR, - { url: error.config?.url } + const errorCode = error.code || "UNKNOWN"; + + if (errorCode === "ECONNREFUSED") { + throw ErrorFactory.connection( + "Connection refused - service not accessible", + "HTTP Service", + this.config.url, + [ + "Check that the service is running", + `Verify service URL: ${this.config.url}`, + "Check network connectivity", + "Verify firewall and security settings", + ] + ); + } else if (errorCode === "ETIMEDOUT" || errorCode === "ECONNABORTED") { + throw ErrorFactory.connection( + "Request timed out", + "HTTP Service", + this.config.url, + [ + "Service may be overloaded or slow", + "Check network connectivity and latency", + "Consider increasing timeout settings", + "Try again later if service is busy", + ] + ); + } else if (errorCode === "ENOTFOUND") { + throw ErrorFactory.connection( + "Service hostname not found", + "HTTP Service", + this.config.url, + [ + "Check service URL spelling and format", + "Verify DNS resolution works", + "Check network connectivity", + "Try using IP address instead of hostname", + ] + ); + } + + throw ErrorFactory.connection( + `No response received from service (${errorCode})`, + "HTTP Service", + this.config.url, + [ + "Check service connectivity and availability", + "Verify network configuration", + "Check firewall and proxy settings", + "Try again later if service is temporarily unavailable", + ] ); } else { // Request setup error - throw new ConductorError( - `Request error: ${error.message}`, - ErrorCodes.CONNECTION_ERROR + throw ErrorFactory.validation( + `Request configuration error: ${error.message}`, + { error: error.message }, + [ + "Check request parameters and configuration", + "Verify data format and structure", + "Review authentication setup", + "Check client configuration", + ] ); } } - private getErrorCodeFromStatus(status: number): string { - switch (status) { - case 401: - case 403: - return ErrorCodes.AUTH_ERROR; - case 404: - return ErrorCodes.FILE_NOT_FOUND; - case 400: - return ErrorCodes.VALIDATION_FAILED; - default: - return ErrorCodes.CONNECTION_ERROR; - } - } - + /** + * Enhanced retry logic with better error classification + */ private isRetryableError(error: any): boolean { if (!error.response) { - return true; // Network errors are retryable + // Network errors are generally retryable + const retryableCodes = ["ECONNRESET", "ECONNABORTED", "ETIMEDOUT"]; + return retryableCodes.includes(error.code); } const status = error.response.status; - // Retry on server errors, but not client errors - return status >= 500 || status === 429; // 429 = Too Many Requests + + // Retry on server errors and rate limiting, but not client errors + if (status >= 500 || status === 429) { + return true; + } + + // Don't retry client errors (4xx) + return false; } private delay(ms: number): Promise { diff --git a/apps/conductor/src/services/base/baseService.ts b/apps/conductor/src/services/base/baseService.ts index ccb3cc59..c3c0f6e9 100644 --- a/apps/conductor/src/services/base/baseService.ts +++ b/apps/conductor/src/services/base/baseService.ts @@ -1,7 +1,7 @@ -// src/services/base/BaseService.ts +// src/services/base/BaseService.ts - Enhanced with ErrorFactory patterns import { HttpService } from "./HttpService"; import { Logger } from "../../utils/logger"; -import { ConductorError, ErrorCodes } from "../../utils/errors"; +import { ErrorFactory } from "../../utils/errors"; import { ServiceConfig, HealthCheckResult } from "./types"; export abstract class BaseService { @@ -21,7 +21,7 @@ export abstract class BaseService { const startTime = Date.now(); try { - Logger.info(`Checking ${this.serviceName} health...`); + Logger.debug`Checking ${this.serviceName} health at ${this.config.url}${this.healthEndpoint}`; const response = await this.http.get(this.healthEndpoint, { timeout: 5000, @@ -32,11 +32,9 @@ export abstract class BaseService { const isHealthy = this.isHealthyResponse(response.data, response.status); if (isHealthy) { - Logger.info(`✓ ${this.serviceName} is healthy (${responseTime}ms)`); + Logger.debug`${this.serviceName} is healthy (${responseTime}ms)`; } else { - Logger.warn( - `⚠ ${this.serviceName} health check returned unhealthy status` - ); + Logger.warn`${this.serviceName} health check returned unhealthy status`; } return { @@ -46,13 +44,14 @@ export abstract class BaseService { }; } catch (error) { const responseTime = Date.now() - startTime; - Logger.error( - `✗ ${this.serviceName} health check failed (${responseTime}ms)` - ); + Logger.error`${this.serviceName} health check failed (${responseTime}ms)`; + + // Enhanced error context for health check failures + const healthError = this.createHealthCheckError(error, responseTime); return { healthy: false, - message: error instanceof Error ? error.message : String(error), + message: healthError.message, responseTime, }; } @@ -84,15 +83,23 @@ export abstract class BaseService { } protected handleServiceError(error: unknown, operation: string): never { - if (error instanceof ConductorError) { + if (error instanceof Error && error.name === "ConductorError") { throw error; } - const errorMessage = error instanceof Error ? error.message : String(error); - throw new ConductorError( - `${this.serviceName} ${operation} failed: ${errorMessage}`, - ErrorCodes.CONNECTION_ERROR, - { service: this.serviceName, operation, originalError: error } + // Enhanced error handling with service context + throw ErrorFactory.connection( + `${this.serviceName} ${operation} failed`, + this.serviceName, + this.config.url, + [ + `Check that ${this.serviceName} is running and accessible`, + `Verify service URL: ${this.config.url}`, + "Check network connectivity and firewall settings", + "Confirm authentication credentials if required", + `Test manually: curl ${this.config.url}${this.healthEndpoint}`, + "Check service logs for additional details", + ] ); } @@ -100,10 +107,13 @@ export abstract class BaseService { return url.endsWith("/") ? url.slice(0, -1) : url; } - // Updated validation method with better type support + /** + * Enhanced validation method with better error messages + */ protected validateRequiredFields>( data: T, - fields: (keyof T)[] + fields: (keyof T)[], + context?: string ): void { const missingFields = fields.filter( (field) => @@ -111,18 +121,34 @@ export abstract class BaseService { ); if (missingFields.length > 0) { - throw new ConductorError( - `Missing required fields: ${missingFields.join(", ")}`, - ErrorCodes.VALIDATION_FAILED, - { missingFields, provided: Object.keys(data) } + const contextMsg = context ? ` for ${context}` : ""; + + throw ErrorFactory.validation( + `Missing required fields${contextMsg}`, + { + missingFields: missingFields.map(String), + provided: Object.keys(data), + context, + }, + [ + `Provide values for: ${missingFields.map(String).join(", ")}`, + "Check the request payload structure", + "Verify all required parameters are included", + context + ? `Review ${context} documentation for required fields` + : "Review API documentation", + ] ); } } - // Alternative validation method for simple objects + /** + * Alternative validation method for simple objects + */ protected validateRequired( data: Record, - fields: string[] + fields: string[], + context?: string ): void { const missingFields = fields.filter( (field) => @@ -130,11 +156,198 @@ export abstract class BaseService { ); if (missingFields.length > 0) { - throw new ConductorError( - `Missing required fields: ${missingFields.join(", ")}`, - ErrorCodes.VALIDATION_FAILED, - { missingFields, provided: Object.keys(data) } + const contextMsg = context ? ` for ${context}` : ""; + + throw ErrorFactory.validation( + `Missing required fields${contextMsg}`, + { + missingFields, + provided: Object.keys(data), + context, + }, + [ + `Provide values for: ${missingFields.join(", ")}`, + "Check the request payload structure", + "Verify all required parameters are included", + context + ? `Review ${context} documentation for required fields` + : "Review API documentation", + ] + ); + } + } + + /** + * Enhanced file validation with specific error context + */ + protected validateFileExists(filePath: string, fileType?: string): void { + const fs = require("fs"); + + if (!filePath) { + throw ErrorFactory.args( + `${fileType || "File"} path not provided`, + undefined, + [ + `Specify a ${fileType || "file"} path`, + "Check command line arguments", + "Verify the parameter is not empty", + ] + ); + } + + if (!fs.existsSync(filePath)) { + throw ErrorFactory.file( + `${fileType || "File"} not found: ${filePath}`, + filePath, + [ + "Check that the file path is correct", + "Ensure the file exists at the specified location", + "Verify file permissions allow read access", + `Current directory: ${process.cwd()}`, + ] + ); + } + + // Check if file is readable + try { + fs.accessSync(filePath, fs.constants.R_OK); + } catch (error) { + throw ErrorFactory.file( + `${fileType || "File"} is not readable: ${filePath}`, + filePath, + [ + "Check file permissions", + "Ensure the file is not locked by another process", + "Verify you have read access to the file", + "Try copying the file to a different location", + ] + ); + } + + // Check if file has content + const stats = fs.statSync(filePath); + if (stats.size === 0) { + throw ErrorFactory.file( + `${fileType || "File"} is empty: ${filePath}`, + filePath, + [ + "Ensure the file contains data", + "Check if the file was properly created", + "Verify the file is not corrupted", + ] + ); + } + } + + /** + * Enhanced JSON parsing with specific error context + */ + protected parseJsonFile(filePath: string, fileType?: string): any { + this.validateFileExists(filePath, fileType); + + const fs = require("fs"); + const path = require("path"); + + try { + const fileContent = fs.readFileSync(filePath, "utf-8"); + return JSON.parse(fileContent); + } catch (error) { + if (error instanceof SyntaxError) { + throw ErrorFactory.file( + `Invalid JSON format in ${fileType || "file"}: ${path.basename( + filePath + )}`, + filePath, + [ + "Check JSON syntax for errors (missing commas, brackets, quotes)", + "Validate JSON structure using a JSON validator", + "Ensure file encoding is UTF-8", + "Try viewing the file in a JSON editor", + `JSON error: ${error.message}`, + ] + ); + } + + throw ErrorFactory.file( + `Error reading ${fileType || "file"}: ${ + error instanceof Error ? error.message : String(error) + }`, + filePath, + [ + "Check file permissions and accessibility", + "Verify file is not corrupted", + "Ensure file is properly formatted", + "Try opening the file manually to inspect content", + ] ); } } + + /** + * Create enhanced health check error with service-specific guidance + */ + private createHealthCheckError(error: unknown, responseTime: number): Error { + const baseUrl = this.normalizeUrl(this.config.url); + + if (error instanceof Error) { + // Connection refused + if (error.message.includes("ECONNREFUSED")) { + return ErrorFactory.connection( + `Cannot connect to ${this.serviceName} - connection refused`, + this.serviceName, + baseUrl, + [ + `Check that ${this.serviceName} is running`, + `Verify service URL: ${baseUrl}`, + "Check if the service port is correct", + "Confirm no firewall is blocking the connection", + `Test connection: curl ${baseUrl}${this.healthEndpoint}`, + ] + ); + } + + // Timeout + if ( + error.message.includes("timeout") || + error.message.includes("ETIMEDOUT") + ) { + return ErrorFactory.connection( + `${this.serviceName} health check timed out (${responseTime}ms)`, + this.serviceName, + baseUrl, + [ + "Service may be overloaded or starting up", + "Check service performance and resource usage", + "Verify network latency is acceptable", + "Consider increasing timeout if service is slow", + "Check service logs for performance issues", + ] + ); + } + + // Authentication errors + if (error.message.includes("401") || error.message.includes("403")) { + return ErrorFactory.connection( + `${this.serviceName} authentication failed`, + this.serviceName, + baseUrl, + [ + "Check authentication credentials", + "Verify API tokens are valid and not expired", + "Confirm proper authentication headers", + "Check service authentication configuration", + ] + ); + } + } + + // Generic connection error + return ErrorFactory.connection( + `${this.serviceName} health check failed: ${ + error instanceof Error ? error.message : String(error) + }`, + this.serviceName, + baseUrl + ); + } } diff --git a/apps/conductor/src/services/csvProcessor/csvParser.ts b/apps/conductor/src/services/csvProcessor/csvParser.ts index 2743b800..4665ab2d 100644 --- a/apps/conductor/src/services/csvProcessor/csvParser.ts +++ b/apps/conductor/src/services/csvProcessor/csvParser.ts @@ -1,7 +1,9 @@ +// src/services/csvProcessor/csvParser.ts - Enhanced with ErrorFactory patterns import * as fs from "fs"; // File system operations import * as readline from "readline"; // Reading files line by line import { parse as csvParse } from "csv-parse/sync"; // CSV parsing functionality import { Logger } from "../../utils/logger"; +import { ErrorFactory } from "../../utils/errors"; /** * CSV Processing utility @@ -12,74 +14,380 @@ import { Logger } from "../../utils/logger"; * * Used by the Conductor to prepare data for Elasticsearch ingestion. * Handles type conversion, null values, and submitter metadata. + * Enhanced with ErrorFactory patterns for consistent error handling. */ /** * Counts the total number of lines in a file, excluding the header + * Enhanced with comprehensive error handling + * * @param filePath - Path to the CSV file * @returns Promise resolving to number of data lines (excluding header) */ - export async function countFileLines(filePath: string): Promise { // Notify user that counting is in progress + Logger.debug`csvParser: Beginning data transfer`; + Logger.debug`csvParser: Calculating records to upload`; - Logger.debug(`csvParser: Beginning data transfer`); - Logger.debug(`csvParser: Calculating records to upload`); + // Enhanced file validation + if (!filePath || typeof filePath !== "string") { + throw ErrorFactory.args( + "File path is required for line counting", + "countFileLines", + [ + "Provide a valid file path", + "Ensure path is a non-empty string", + "Check file path parameter", + ] + ); + } + + if (!fs.existsSync(filePath)) { + throw ErrorFactory.file( + `CSV file not found for line counting: ${filePath}`, + filePath, + [ + "Check that the file path is correct", + "Ensure the file exists at the specified location", + "Verify file permissions allow read access", + `Current directory: ${process.cwd()}`, + ] + ); + } + + // Check file readability + try { + fs.accessSync(filePath, fs.constants.R_OK); + } catch (error) { + throw ErrorFactory.file(`CSV file is not readable: ${filePath}`, filePath, [ + "Check file permissions", + "Ensure the file is not locked by another process", + "Verify you have read access to the file", + "Try copying the file to a different location", + ]); + } - // Create a readline interface to read file line by line - const rl = readline.createInterface({ - input: fs.createReadStream(filePath), - crlfDelay: Infinity, // Handle different line endings - }); + // Check file size + let fileStats: fs.Stats; + try { + fileStats = fs.statSync(filePath); + } catch (error) { + throw ErrorFactory.file( + `Cannot read file statistics: ${filePath}`, + filePath, + [ + "Check file exists and is accessible", + "Verify file permissions", + "Ensure file is not corrupted", + "Try using absolute path if relative path fails", + ] + ); + } + + if (fileStats.size === 0) { + throw ErrorFactory.file(`CSV file is empty: ${filePath}`, filePath, [ + "Ensure the file contains data", + "Check if the file was properly created", + "Verify the file is not corrupted", + "CSV files must have at least a header row", + ]); + } + + let rl: readline.Interface; + + try { + // Create a readline interface to read file line by line + rl = readline.createInterface({ + input: fs.createReadStream(filePath), + crlfDelay: Infinity, // Handle different line endings + }); + } catch (error) { + throw ErrorFactory.file( + `Failed to open CSV file for reading: ${filePath}`, + filePath, + [ + "Check file permissions allow read access", + "Ensure file is not locked by another process", + "Verify file encoding is supported", + "Try copying the file to a different location", + ] + ); + } let lines = 0; - // Count each line in file - for await (const _ of rl) { - lines++; + + try { + // Count each line in file + for await (const _ of rl) { + lines++; + } + } catch (error) { + // Ensure readline interface is closed + try { + rl.close(); + } catch (closeError) { + Logger.debug`Error closing readline interface: ${closeError}`; + } + + throw ErrorFactory.file( + `Error reading CSV file during line counting: ${ + error instanceof Error ? error.message : String(error) + }`, + filePath, + [ + "Check file is not corrupted", + "Verify file encoding (should be UTF-8)", + "Ensure file is complete and not truncated", + "Try opening the file in a text editor to verify content", + ] + ); + } + + // Ensure readline interface is properly closed + try { + rl.close(); + } catch (closeError) { + Logger.debug`Error closing readline interface: ${closeError}`; + } + + if (lines === 0) { + throw ErrorFactory.csv( + `CSV file contains no lines: ${filePath}`, + filePath, + undefined, + [ + "Ensure the file contains data", + "Check if the file was properly created", + "Verify the file is not empty", + "CSV files must have at least a header row", + ] + ); } const recordCount = lines - 1; // Subtract header line from total count + + if (recordCount < 0) { + throw ErrorFactory.csv( + `CSV file has no data rows: ${filePath}`, + filePath, + undefined, + [ + "Ensure the file contains data beyond the header row", + "Check if data was properly written to the file", + "Verify the CSV format is correct", + "CSV files need both headers and data rows", + ] + ); + } + Logger.debug`Found ${recordCount} data records in ${filePath}`; return recordCount; } /** * Parses a single line of CSV data into an array of values + * Enhanced with comprehensive error handling and validation + * * @param line - Raw CSV line string * @param delimiter - CSV delimiter character + * @param isHeaderRow - Whether this is a header row (for enhanced logging) * @returns Array of parsed values from the CSV line */ - export function parseCSVLine( line: string, delimiter: string, - isHeaderRow: boolean = true + isHeaderRow: boolean = false ): any[] { + // Enhanced parameter validation + if (typeof line !== "string") { + throw ErrorFactory.args( + "CSV line must be a string for parsing", + "parseCSVLine", + [ + "Ensure line parameter is a string", + "Check data source for correct format", + "Verify file reading process", + ] + ); + } + + if (!delimiter || typeof delimiter !== "string") { + throw ErrorFactory.args( + "CSV delimiter is required for parsing", + "parseCSVLine", + [ + "Provide a valid delimiter character", + "Common delimiters: ',' (comma), '\\t' (tab), ';' (semicolon)", + "Check configuration settings", + ] + ); + } + + if (delimiter.length !== 1) { + throw ErrorFactory.args( + `Invalid delimiter length: '${delimiter}' (must be exactly 1 character)`, + "parseCSVLine", + [ + "Delimiter must be exactly one character", + "Common delimiters: ',' (comma), ';' (semicolon), '\\t' (tab)", + "Check delimiter configuration", + ] + ); + } + + // Handle empty lines + if (line.trim() === "") { + if (isHeaderRow) { + throw ErrorFactory.csv("Header row is empty", undefined, 1, [ + "Ensure the first row contains column headers", + "Check CSV file format", + "Verify file is not corrupted", + ]); + } + Logger.debug`Skipping empty line during CSV parsing`; + return []; + } + try { const parseOptions = { delimiter: delimiter, trim: true, skipEmptyLines: true, relax_column_count: true, + relaxQuotes: true, // Handle improperly quoted fields }; - // If it's a header row, only parse the first line + // Enhanced logging based on row type if (isHeaderRow) { - Logger.debug`Parsing header row with delimiter '${delimiter}'`; - const result = csvParse(line, parseOptions); - return result[0] ? [result[0]] : []; + Logger.debug`Parsing header row with delimiter '${delimiter.replace( + "\t", + "\\t" + )}'`; + } else { + Logger.debug`Parsing data row with delimiter '${delimiter.replace( + "\t", + "\\t" + )}'`; + } + + // Parse the line + const result = csvParse(line, parseOptions); + + if (!result || !Array.isArray(result)) { + throw ErrorFactory.csv( + "CSV parsing returned invalid result", + undefined, + isHeaderRow ? 1 : undefined, + [ + "Check CSV line format and structure", + "Verify delimiter is correct for this file", + "Ensure proper CSV escaping for special characters", + "Check for malformed CSV syntax", + `Current delimiter: '${delimiter.replace("\t", "\\t")}'`, + ] + ); + } + + if (result.length === 0) { + if (isHeaderRow) { + throw ErrorFactory.csv("No data found in header row", undefined, 1, [ + "Ensure the header row contains column names", + "Check CSV format and delimiter", + "Verify file is not corrupted", + ]); + } + Logger.debug`CSV line produced no data after parsing`; + return []; } - // For data rows, parse normally - Logger.debug`Parsing data row with delimiter '${delimiter}'`; - return csvParse(line, parseOptions); + const parsedData = result[0]; + + if (!Array.isArray(parsedData)) { + throw ErrorFactory.csv( + "Parsed CSV data is not in expected array format", + undefined, + isHeaderRow ? 1 : undefined, + [ + "Check CSV parsing library compatibility", + "Verify CSV line structure is valid", + "Ensure delimiter matches file format", + "Check for unusual CSV formatting", + `Current delimiter: '${delimiter.replace("\t", "\\t")}'`, + ] + ); + } + + // Enhanced validation for header rows + if (isHeaderRow) { + const emptyHeaders = parsedData.filter( + (header, index) => !header || header.trim() === "" + ); + + if (emptyHeaders.length > 0) { + throw ErrorFactory.csv( + `Empty headers detected in CSV (${emptyHeaders.length} of ${parsedData.length})`, + undefined, + 1, + [ + "Ensure all columns have header names", + "Remove empty columns from the CSV", + "Check for extra delimiters in the header row", + "Verify CSV format is correct", + ] + ); + } + + Logger.debug`Successfully parsed ${parsedData.length} headers`; + } + + return [parsedData]; } catch (error) { - Logger.error`Error parsing CSV line: ${ + // Enhanced error handling with context + const rowType = isHeaderRow ? "header" : "data"; + const linePreview = + line.length > 100 ? `${line.substring(0, 100)}...` : line; + + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + // Handle CSV parsing specific errors + if (error instanceof Error) { + if (error.message.includes("Invalid")) { + throw ErrorFactory.csv( + `Invalid CSV format in ${rowType} row: ${error.message}`, + undefined, + isHeaderRow ? 1 : undefined, + [ + `Check ${rowType} row format and structure`, + `Verify delimiter '${delimiter.replace("\t", "\\t")}' is correct`, + "Ensure proper CSV escaping for special characters", + "Check for unmatched quotes or malformed fields", + `Problem line: ${linePreview}`, + ] + ); + } + } + + Logger.error`Error parsing CSV ${rowType} row: ${ error instanceof Error ? error.message : String(error) }`; - Logger.debug`Failed line content: ${line.substring(0, 100)}${ - line.length > 100 ? "..." : "" - }`; - return []; + Logger.debug`Failed line content: ${linePreview}`; + + throw ErrorFactory.csv( + `Failed to parse CSV ${rowType} row`, + undefined, + isHeaderRow ? 1 : undefined, + [ + `Check ${rowType} row format and delimiter`, + `Verify delimiter '${delimiter.replace( + "\t", + "\\t" + )}' is correct for this file`, + "Ensure proper CSV format and escaping", + "Check file encoding (should be UTF-8)", + `Problem line: ${linePreview}`, + ] + ); } } diff --git a/apps/conductor/src/services/csvProcessor/index.ts b/apps/conductor/src/services/csvProcessor/index.ts index ede6eee9..b19848a6 100644 --- a/apps/conductor/src/services/csvProcessor/index.ts +++ b/apps/conductor/src/services/csvProcessor/index.ts @@ -1,3 +1,4 @@ +// src/services/csvProcessor/index.ts - Enhanced with ErrorFactory patterns import * as fs from "fs"; import * as readline from "readline"; import { Client } from "@elastic/elasticsearch"; @@ -6,9 +7,10 @@ import { countFileLines, parseCSVLine } from "./csvParser"; import { Logger } from "../../utils/logger"; import { validateCSVStructure, + validateCSVHeaders, validateHeadersMatchMappings, } from "../../validations"; -import { ConductorError, ErrorCodes } from "../../utils/errors"; +import { ErrorFactory } from "../../utils/errors"; import { CSVProcessingErrorHandler } from "./logHandler"; import { sendBulkWriteRequest } from "../elasticsearch"; import { formatDuration, calculateETA, createProgressBar } from "./progressBar"; @@ -16,6 +18,7 @@ import { createRecordMetadata } from "./metadata"; /** * Processes a CSV file and indexes the data into Elasticsearch. + * Enhanced with ErrorFactory patterns for better error handling. * * @param filePath - Path to the CSV file to process * @param config - Configuration object @@ -34,161 +37,339 @@ export async function processCSVFile( const batchedRecords: object[] = []; const processingStartTime = new Date().toISOString(); + // Enhanced file validation + if (!fs.existsSync(filePath)) { + throw ErrorFactory.file(`CSV file not found: ${filePath}`, filePath, [ + "Check that the file path is correct", + "Ensure the file exists at the specified location", + "Verify file permissions allow read access", + `Current directory: ${process.cwd()}`, + ]); + } + // Get total lines upfront to avoid repeated calls - const totalLines = await countFileLines(filePath); + let totalLines: number; + try { + totalLines = await countFileLines(filePath); + } catch (error) { + throw ErrorFactory.file( + `Failed to count lines in CSV file: ${filePath}`, + filePath, + [ + "Check file is not corrupted", + "Verify file permissions allow read access", + "Ensure file is not locked by another process", + "Try copying the file to a different location", + ] + ); + } + + if (totalLines === 0) { + throw ErrorFactory.csv( + `CSV file is empty: ${filePath}`, + filePath, + undefined, + [ + "Ensure the file contains data", + "Check if the file was properly created", + "Verify the file is not corrupted", + "CSV files must have at least a header row", + ] + ); + } Logger.info`Processing file: ${filePath}`; - const fileStream = fs.createReadStream(filePath); - const rl = readline.createInterface({ - input: fileStream, - crlfDelay: Infinity, - }); + let fileStream: fs.ReadStream; + let rl: readline.Interface; + + try { + fileStream = fs.createReadStream(filePath); + rl = readline.createInterface({ + input: fileStream, + crlfDelay: Infinity, + }); + } catch (error) { + throw ErrorFactory.file( + `Failed to open CSV file for reading: ${filePath}`, + filePath, + [ + "Check file permissions allow read access", + "Ensure file is not locked by another process", + "Verify file is not corrupted", + "Try copying the file to a different location", + ] + ); + } try { for await (const line of rl) { try { if (isFirstLine) { - headers = parseCSVLine(line, config.delimiter, true)[0] || []; - Logger.info`Validating headers against the ${config.elasticsearch.index} mapping`; - await validateCSVStructure(headers); - Logger.info("Headers validated against index mapping"); - await validateHeadersMatchMappings( - client, - headers, - config.elasticsearch.index - ); - isFirstLine = false; + // Enhanced header processing + try { + const headerResult = parseCSVLine(line, config.delimiter, true); + headers = headerResult[0] || []; - Logger.generic(`\n Processing data into elasticsearch...\n`); - continue; - } + if (headers.length === 0) { + throw ErrorFactory.csv( + `No headers found in CSV file: ${filePath}`, + filePath, + 1, + [ + "Ensure the first row contains column headers", + "Check that the delimiter is correct", + "Verify the file format is valid CSV", + `Current delimiter: '${config.delimiter}'`, + ] + ); + } - const rowValues = parseCSVLine(line, config.delimiter)[0] || []; - const metadata = createRecordMetadata( - filePath, - processingStartTime, - processedRecords + 1 - ); - const record = { - submission_metadata: metadata, - data: Object.fromEntries(headers.map((h, i) => [h, rowValues[i]])), - }; - - batchedRecords.push(record); - processedRecords++; - - // Update progress more frequently - if (processedRecords % 10 === 0) { - updateProgressDisplay( - processedRecords, - totalLines - 1, // Subtract 1 to account for header - startTime - ); + Logger.debug`Validating CSV headers and structure`; + + // Validate CSV structure using the available validation function + await validateCSVStructure(headers); + + Logger.info`Validating headers against the ${config.elasticsearch.index} mapping`; + + // Validate headers match Elasticsearch index mapping + await validateHeadersMatchMappings( + client, + headers, + config.elasticsearch.index + ); + + Logger.success`Headers validated successfully`; + isFirstLine = false; + + Logger.generic(`\n Processing data into elasticsearch...\n`); + continue; + } catch (error) { + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + throw ErrorFactory.csv( + `Failed to process CSV headers: ${ + error instanceof Error ? error.message : String(error) + }`, + filePath, + 1, + [ + "Check CSV header format and structure", + "Ensure headers follow naming conventions", + "Verify delimiter is correct", + "Check for special characters in headers", + "Ensure headers match Elasticsearch index mapping", + ] + ); + } } - if (batchedRecords.length >= config.batchSize) { - await sendBatchToElasticsearch( - client, - batchedRecords, - config.elasticsearch.index, - (count) => { - failedRecords += count; + // Enhanced data row processing + if (!isFirstLine) { + try { + const parsedRow = parseCSVLine(line, config.delimiter, false); + const rowData = parsedRow[0]; + + if (!rowData || rowData.length === 0) { + Logger.debug`Skipping empty row at line ${processedRecords + 2}`; + continue; } - ); - batchedRecords.length = 0; + + // Enhanced row validation + if (rowData.length !== headers.length) { + Logger.warn`Row ${processedRecords + 2} has ${ + rowData.length + } columns, expected ${headers.length} (header count)`; + } + + // Create record with metadata and data + const metadata = createRecordMetadata( + filePath, + processingStartTime, + processedRecords + 1 + ); + + // Create the final record structure + const record = { + submission_metadata: metadata, + ...Object.fromEntries( + headers.map((header, index) => [header, rowData[index] || null]) + ), + }; + + batchedRecords.push(record); + processedRecords++; + + // Enhanced batch processing with progress tracking + if (batchedRecords.length >= config.batchSize) { + await processBatch( + client, + batchedRecords, + config, + processedRecords, + totalLines + ); + batchedRecords.length = 0; // Clear the array + } + + // Enhanced progress reporting + if (processedRecords % 1000 === 0) { + const progress = ((processedRecords / totalLines) * 100).toFixed( + 1 + ); + Logger.info`Processed ${processedRecords.toLocaleString()} records (${progress}%)`; + } + } catch (rowError) { + failedRecords++; + CSVProcessingErrorHandler.handleProcessingError( + rowError, + processedRecords, + false, + config.delimiter + ); + } } } catch (lineError) { - // Handle individual line processing errors - Logger.warn`Error processing line: ${line.substring(0, 50)}`; + // Handle line-level errors + if (isFirstLine) { + throw lineError; // Re-throw header errors + } + failedRecords++; + Logger.error`Error processing line ${processedRecords + 2}: ${ + lineError instanceof Error ? lineError.message : String(lineError) + }`; + + // Continue processing other lines for data errors + if (failedRecords > processedRecords * 0.1) { + // Stop if more than 10% of records fail + throw ErrorFactory.csv( + `Too many failed records (${failedRecords} failures in ${processedRecords} processed)`, + filePath, + processedRecords + 2, + [ + "Check CSV data format and consistency", + "Verify data types match expected format", + "Review failed records for common patterns", + "Consider fixing source data before reprocessing", + ] + ); + } } } - // Final batch and progress update + // Process any remaining records in the final batch if (batchedRecords.length > 0) { - await sendBatchToElasticsearch( + await processBatch( client, batchedRecords, - config.elasticsearch.index, - (count) => { - failedRecords += count; - } + config, + processedRecords, + totalLines ); } - // Ensure final progress is displayed - updateProgressDisplay(processedRecords, totalLines, startTime); + // Enhanced completion logging + const duration = Date.now() - startTime; + const recordsPerSecond = Math.round(processedRecords / (duration / 1000)); - // Display final summary - CSVProcessingErrorHandler.displaySummary( - processedRecords, - failedRecords, - startTime - ); + Logger.success`CSV processing completed successfully`; + Logger.info`Processed ${processedRecords.toLocaleString()} records in ${formatDuration( + duration + )}`; + Logger.info`Average rate: ${recordsPerSecond.toLocaleString()} records/second`; + + if (failedRecords > 0) { + Logger.warn`${failedRecords} records failed to process`; + Logger.tipString( + "Review error messages above for details on failed records" + ); + } } catch (error) { - rl.close(); - - // Use the error handler to process and throw the error - CSVProcessingErrorHandler.handleProcessingError( - error, - processedRecords, - isFirstLine, - config.delimiter + // Enhanced error handling for the entire processing operation + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + throw ErrorFactory.csv( + `CSV processing failed: ${ + error instanceof Error ? error.message : String(error) + }`, + filePath, + undefined, + [ + "Check CSV file format and structure", + "Verify all required fields are present", + "Ensure data types are consistent", + "Check file permissions and accessibility", + "Review Elasticsearch connectivity and settings", + ] ); + } finally { + // Ensure resources are properly cleaned up + try { + if (rl) rl.close(); + if (fileStream) fileStream.destroy(); + } catch (cleanupError) { + Logger.debug`Error during cleanup: ${cleanupError}`; + } } } /** - * Updates the progress display in the console - * - * @param processed - Number of processed records - * @param total - Total number of records - * @param startTime - When processing started - */ -function updateProgressDisplay( - processed: number, - total: number, - startTime: number -): void { - const elapsedMs = Math.max(1, Date.now() - startTime); - const progress = Math.min(100, (processed / total) * 100); - const progressBar = createProgressBar(progress); - const eta = calculateETA(processed, total, elapsedMs / 1000); - const recordsPerSecond = Math.round(processed / (elapsedMs / 1000)); - - // Use \r to overwrite previous line - process.stdout.write("\r"); - process.stdout.write( - ` ${progressBar} | ` + // Added space before progress bar - `${processed}/${total} | ` + - `⏱ ${formatDuration(elapsedMs)} | ` + - `🏁 ${eta} | ` + - `⚡${recordsPerSecond} rows/sec` // Added space after rows/sec - ); -} - -/** - * Sends a batch of records to Elasticsearch - * - * @param client - Elasticsearch client - * @param records - Records to send - * @param indexName - Target index - * @param onFailure - Callback to track failed records + * Enhanced batch processing with comprehensive error handling */ -async function sendBatchToElasticsearch( +async function processBatch( client: Client, records: object[], - indexName: string, - onFailure: (count: number) => void + config: Config, + processedRecords: number, + totalLines: number ): Promise { try { - await sendBulkWriteRequest(client, records, indexName, onFailure); - } catch (error) { - throw new ConductorError( - "Failed to send batch to Elasticsearch", - ErrorCodes.CONNECTION_ERROR, - error + Logger.debug`Processing batch of ${records.length} records`; + + // Enhanced progress calculation + const progress = Math.min((processedRecords / totalLines) * 100, 100); + const eta = calculateETA(Date.now(), processedRecords, totalLines); + + Logger.info`${createProgressBar(progress, 30)} ${progress.toFixed( + 1 + )}% ${eta}`; + + // Enhanced bulk write with proper function signature + await sendBulkWriteRequest( + client, + records, + config.elasticsearch.index, + (failureCount: number) => { + if (failureCount > 0) { + Logger.warn`${failureCount} records failed to index in this batch`; + } + }, + { + maxRetries: 3, + refresh: true, + } + ); + + Logger.debug`Successfully processed batch of ${records.length} records`; + } catch (batchError) { + throw ErrorFactory.connection( + `Batch processing failed: ${ + batchError instanceof Error ? batchError.message : String(batchError) + }`, + "Elasticsearch", + config.elasticsearch.url, + [ + "Check Elasticsearch connectivity and health", + "Verify index exists and has proper permissions", + "Review batch size - try reducing if too large", + "Check cluster resources (disk space, memory)", + "Ensure proper authentication credentials", + ] ); } } diff --git a/apps/conductor/src/services/csvProcessor/logHandler.ts b/apps/conductor/src/services/csvProcessor/logHandler.ts index 028268f2..9a2b4e79 100644 --- a/apps/conductor/src/services/csvProcessor/logHandler.ts +++ b/apps/conductor/src/services/csvProcessor/logHandler.ts @@ -1,20 +1,22 @@ -import { ConductorError, ErrorCodes } from "../../utils/errors"; +// src/services/csvProcessor/logHandler.ts - Enhanced with ErrorFactory patterns +import { ErrorFactory, ErrorCodes } from "../../utils/errors"; import { Logger } from "../../utils/logger"; import { formatDuration } from "./progressBar"; /** * Error handler for CSV processing operations. * Manages CSV-specific errors and generates appropriate error logs. + * Enhanced with ErrorFactory patterns for consistent user guidance. */ export class CSVProcessingErrorHandler { /** - * Handles errors during CSV processing + * Handles errors during CSV processing with enhanced error analysis * * @param error - The error that occurred * @param processedRecords - Number of records processed before error * @param isFirstLine - Whether the error occurred on the first line (headers) * @param delimiter - CSV delimiter character - * @throws ConductorError with appropriate error code and message + * @throws Enhanced ConductorError with appropriate error code and guidance */ public static handleProcessingError( error: unknown, @@ -25,35 +27,72 @@ export class CSVProcessingErrorHandler { // Convert to string for guaranteed safe output const errorMessage = error instanceof Error ? error.message : String(error); + // If it's already a ConductorError, preserve it with additional context + if (error instanceof Error && error.name === "ConductorError") { + // Add CSV processing context to existing errors + const existingError = error as any; + const enhancedDetails = { + ...existingError.details, + processedRecords, + isFirstLine, + delimiter, + context: "CSV processing", + }; + + throw ErrorFactory.csv( + existingError.message, + existingError.details?.filePath, + isFirstLine ? 1 : undefined, + existingError.details?.suggestions || [ + "Check CSV file format and structure", + "Verify delimiter and encoding settings", + "Review error details for specific guidance", + ] + ); + } + if (isFirstLine) { - // First line errors are usually header parsing issues - Logger.error(`CSV header parsing failed: ${errorMessage}`); - Logger.tip(`Make sure your CSV file uses '${delimiter}' as a delimiter`); + // Enhanced first line (header) error handling + Logger.error`CSV header parsing failed: ${errorMessage}`; + Logger.tip`Make sure your CSV file uses '${delimiter.replace( + "\t", + "\\t" + )}' as a delimiter`; - throw new ConductorError( + // Analyze header-specific issues + const suggestions = this.generateHeaderErrorSuggestions( + errorMessage, + delimiter + ); + + throw ErrorFactory.csv( "Failed to parse CSV headers", - ErrorCodes.VALIDATION_FAILED, - { originalError: error } + undefined, + 1, + suggestions ); } else { - // General processing errors - Logger.error( - `CSV processing failed after ${processedRecords} records: ${errorMessage}` + // Enhanced data processing error handling + Logger.error`CSV processing failed after ${processedRecords} records: ${errorMessage}`; + + // Analyze data processing issues + const suggestions = this.generateDataProcessingErrorSuggestions( + errorMessage, + processedRecords, + delimiter ); - throw new ConductorError( - "CSV processing failed", - ErrorCodes.CSV_ERROR, // Using CSV_ERROR instead of PROCESSING_FAILED - { - recordsProcessed: processedRecords, - originalError: error, - } + throw ErrorFactory.csv( + `CSV processing failed after processing ${processedRecords} records`, + undefined, + undefined, + suggestions ); } } /** - * Displays a summary of the CSV processing operation + * Displays a comprehensive summary of the CSV processing operation * * @param processed - Total number of processed records * @param failed - Number of failed records @@ -74,26 +113,375 @@ export class CSVProcessingErrorHandler { // Clear the current line process.stdout.write("\n"); - if (failed > 0) { - Logger.warn(`Transfer to elasticsearch completed with partial errors`); + // Enhanced summary display based on results + if (failed > 0 && successfulRecords > 0) { + Logger.warn`Transfer to elasticsearch completed with partial errors`; + Logger.generic(" "); + Logger.generic(`📊 Processing Summary:`); + } else if (failed > 0 && successfulRecords === 0) { + Logger.error`Transfer to elasticsearch failed completely`; + Logger.generic(" "); + Logger.generic(`❌ Processing Summary:`); } else if (processed === 0) { - Logger.warn(`No records were processed`); + Logger.warn`No records were processed`; + Logger.generic(" "); + Logger.generic(`⚠️ Processing Summary:`); } else { - Logger.success(`Transfer to elasticsearch completed successfully`); + Logger.success`Transfer to elasticsearch completed successfully`; + Logger.generic(" "); + Logger.generic(`✅ Processing Summary:`); } - // Print summary - Logger.generic(` ▸ Total Records processed: ${processed}`); - Logger.generic(` ▸ Records Successfully transferred: ${successfulRecords}`); + // Enhanced metrics display + Logger.generic(` ▸ Total Records processed: ${processed.toLocaleString()}`); + Logger.generic( + ` ▸ Records Successfully transferred: ${successfulRecords.toLocaleString()}` + ); if (failed > 0) { - Logger.warn(` ▸ Records Unsuccessfully transferred: ${failed}`); + const failureRate = ((failed / processed) * 100).toFixed(1); + Logger.warn` ▸ Records Unsuccessfully transferred: ${failed.toLocaleString()} (${failureRate}%)`; Logger.generic(` ▸ Error logs outputted to: /logs/`); + + // Enhanced failure analysis + if (failed > processed * 0.5) { + Logger.generic(" "); + Logger.warn`High failure rate detected (>${failureRate}%)`; + Logger.tipString("Consider reviewing data format and index mappings"); + } } + // Enhanced performance metrics + const processingRate = Math.round(recordsPerSecond); Logger.generic( - ` ▸ Processing speed: ${Math.round(recordsPerSecond)} rows/sec` + ` ▸ Processing speed: ${processingRate.toLocaleString()} rows/sec` ); Logger.generic(` ⏱ Total processing time: ${formatDuration(elapsedMs)}`); + + // Enhanced performance insights + if (processingRate < 100) { + Logger.generic(" "); + Logger.tipString("Consider increasing batch size for better performance"); + } else if (processingRate > 5000) { + Logger.generic(" "); + Logger.tipString("Excellent processing performance!"); + } + + // Enhanced recommendations based on results + if (failed === 0 && processed > 0) { + Logger.generic(" "); + Logger.tipString( + "All records processed successfully - data is ready for analysis" + ); + } else if (failed > 0) { + Logger.generic(" "); + Logger.tipString( + "Review failed records and consider reprocessing with corrected data" + ); + } + } + + /** + * Generate specific suggestions for header parsing errors + */ + private static generateHeaderErrorSuggestions( + errorMessage: string, + delimiter: string + ): string[] { + const suggestions: string[] = []; + + // Analyze error message for specific issues + if (errorMessage.toLowerCase().includes("delimiter")) { + suggestions.push( + `Verify delimiter '${delimiter.replace( + "\t", + "\\t" + )}' is correct for this CSV` + ); + suggestions.push( + "Try common delimiters: ',' (comma), ';' (semicolon), '\\t' (tab)" + ); + suggestions.push( + "Check if the file uses a different delimiter than expected" + ); + } + + if (errorMessage.toLowerCase().includes("encoding")) { + suggestions.push("Check file encoding - should be UTF-8"); + suggestions.push( + "Try opening the file in a text editor to verify encoding" + ); + suggestions.push("Convert file to UTF-8 if using a different encoding"); + } + + if (errorMessage.toLowerCase().includes("quote")) { + suggestions.push("Check for unmatched quotes in header row"); + suggestions.push( + "Ensure proper CSV escaping for header names with special characters" + ); + suggestions.push("Remove or properly escape quotes in column names"); + } + + if (errorMessage.toLowerCase().includes("empty")) { + suggestions.push("Ensure the first row contains column headers"); + suggestions.push("Check that the file is not empty or corrupted"); + suggestions.push( + "Verify the CSV has proper structure with headers and data" + ); + } + + // Add general header suggestions if no specific ones were added + if (suggestions.length === 0) { + suggestions.push("Check CSV file format and header structure"); + suggestions.push( + `Verify delimiter '${delimiter.replace("\t", "\\t")}' is correct` + ); + suggestions.push( + "Ensure headers follow naming conventions (letters, numbers, underscores)" + ); + suggestions.push("Check file encoding (should be UTF-8)"); + } + + // Always add file inspection suggestion + suggestions.push( + "Try opening the file in a text editor to inspect the first row manually" + ); + + return suggestions; + } + + /** + * Generate specific suggestions for data processing errors + */ + private static generateDataProcessingErrorSuggestions( + errorMessage: string, + processedRecords: number, + delimiter: string + ): string[] { + const suggestions: string[] = []; + + // Analyze error message for specific data processing issues + if (errorMessage.toLowerCase().includes("elasticsearch")) { + suggestions.push("Check Elasticsearch service connectivity and health"); + suggestions.push("Verify index exists and has proper permissions"); + suggestions.push("Ensure cluster has sufficient resources"); + suggestions.push("Review Elasticsearch logs for additional details"); + } + + if ( + errorMessage.toLowerCase().includes("batch") || + errorMessage.toLowerCase().includes("bulk") + ) { + suggestions.push("Try reducing batch size to handle large documents"); + suggestions.push("Check for document size limits in Elasticsearch"); + suggestions.push("Consider splitting large files into smaller chunks"); + } + + if ( + errorMessage.toLowerCase().includes("memory") || + errorMessage.toLowerCase().includes("heap") + ) { + suggestions.push("Reduce batch size to lower memory usage"); + suggestions.push("Process files in smaller chunks"); + suggestions.push("Check system memory availability"); + suggestions.push("Consider increasing Node.js heap size"); + } + + if (errorMessage.toLowerCase().includes("timeout")) { + suggestions.push("Increase timeout settings for large operations"); + suggestions.push("Check network connectivity to Elasticsearch"); + suggestions.push("Verify Elasticsearch cluster performance"); + suggestions.push("Consider processing in smaller batches"); + } + + if ( + errorMessage.toLowerCase().includes("mapping") || + errorMessage.toLowerCase().includes("field") + ) { + suggestions.push("Check data types match Elasticsearch index mapping"); + suggestions.push("Verify field names are consistent with mapping"); + suggestions.push("Update index mapping or modify data format"); + suggestions.push("Check for special characters in field values"); + } + + if ( + errorMessage.toLowerCase().includes("parse") || + errorMessage.toLowerCase().includes("format") + ) { + suggestions.push("Check CSV data format consistency"); + suggestions.push( + `Verify delimiter '${delimiter.replace( + "\t", + "\\t" + )}' is used consistently` + ); + suggestions.push("Look for malformed rows or inconsistent column counts"); + suggestions.push("Check for special characters that need escaping"); + } + + // Add progress-based suggestions + if (processedRecords === 0) { + suggestions.push( + "Error occurred immediately - check file format and headers" + ); + suggestions.push("Verify the CSV file structure and delimiter"); + suggestions.push("Ensure Elasticsearch connection is working"); + } else if (processedRecords < 100) { + suggestions.push( + `Error occurred early (record ${processedRecords}) - check data format` + ); + suggestions.push("Review the first few data rows for format issues"); + suggestions.push("Check for inconsistent data types in early records"); + } else { + suggestions.push( + `Error occurred after processing ${processedRecords} records` + ); + suggestions.push( + "Check for data format changes or corruption in later records" + ); + suggestions.push( + "Consider processing in smaller batches to isolate issues" + ); + } + + // Add general suggestions if no specific ones were added + if (suggestions.length === 0) { + suggestions.push("Check CSV data format and structure"); + suggestions.push("Verify Elasticsearch connectivity and configuration"); + suggestions.push("Review system resources (memory, disk space)"); + suggestions.push("Check for data corruption or format inconsistencies"); + } + + // Always add debug suggestion + suggestions.push("Use --debug flag for detailed error information"); + + return suggestions; + } + + /** + * Enhanced error categorization for better user guidance + */ + public static categorizeError(error: unknown): { + category: string; + severity: "low" | "medium" | "high" | "critical"; + recoverable: boolean; + } { + const errorMessage = error instanceof Error ? error.message : String(error); + const lowerMessage = errorMessage.toLowerCase(); + + // Critical errors (cannot continue) + if ( + lowerMessage.includes("file not found") || + lowerMessage.includes("permission denied") + ) { + return { + category: "File Access", + severity: "critical", + recoverable: false, + }; + } + + if ( + lowerMessage.includes("elasticsearch") && + lowerMessage.includes("connection") + ) { + return { + category: "Connection", + severity: "critical", + recoverable: false, + }; + } + + // High severity errors (major issues) + if (lowerMessage.includes("memory") || lowerMessage.includes("heap")) { + return { category: "Resource", severity: "high", recoverable: true }; + } + + if (lowerMessage.includes("header") || lowerMessage.includes("delimiter")) { + return { category: "CSV Format", severity: "high", recoverable: true }; + } + + // Medium severity errors (data issues) + if ( + lowerMessage.includes("mapping") || + lowerMessage.includes("validation") + ) { + return { + category: "Data Validation", + severity: "medium", + recoverable: true, + }; + } + + if (lowerMessage.includes("batch") || lowerMessage.includes("bulk")) { + return { category: "Processing", severity: "medium", recoverable: true }; + } + + // Low severity errors (minor issues) + if (lowerMessage.includes("timeout")) { + return { category: "Performance", severity: "low", recoverable: true }; + } + + // Default categorization + return { category: "General", severity: "medium", recoverable: true }; + } + + /** + * Generate recovery suggestions based on error categorization + */ + public static generateRecoverySuggestions(error: unknown): string[] { + const { category, severity, recoverable } = this.categorizeError(error); + + if (!recoverable) { + return [ + "This error requires immediate attention before processing can continue", + "Fix the underlying issue and restart the operation", + "Contact support if the problem persists", + ]; + } + + const suggestions: string[] = []; + + switch (category) { + case "CSV Format": + suggestions.push("Fix CSV format issues and retry"); + suggestions.push("Validate file structure before reprocessing"); + break; + + case "Resource": + suggestions.push("Reduce batch size and retry"); + suggestions.push("Close other applications to free memory"); + suggestions.push("Process file in smaller chunks"); + break; + + case "Data Validation": + suggestions.push("Review and correct data format"); + suggestions.push("Update index mapping if needed"); + suggestions.push("Clean invalid data entries"); + break; + + case "Processing": + suggestions.push("Adjust processing parameters"); + suggestions.push("Retry with smaller batch sizes"); + break; + + case "Performance": + suggestions.push("Retry the operation"); + suggestions.push("Check system performance"); + break; + + default: + suggestions.push("Review error details and try again"); + suggestions.push("Contact support if issues persist"); + } + + // Add severity-based suggestions + if (severity === "high" || severity === "critical") { + suggestions.push("Address this issue before continuing"); + } else { + suggestions.push("This issue may be temporary - consider retrying"); + } + + return suggestions; } } diff --git a/apps/conductor/src/services/elasticsearch/bulk.ts b/apps/conductor/src/services/elasticsearch/bulk.ts index 0aab1ec2..33532c4e 100644 --- a/apps/conductor/src/services/elasticsearch/bulk.ts +++ b/apps/conductor/src/services/elasticsearch/bulk.ts @@ -2,10 +2,11 @@ * Elasticsearch Bulk Operations Module * * Provides functions for bulk indexing operations in Elasticsearch. + * Enhanced with ErrorFactory patterns for consistent error handling. */ import { Client } from "@elastic/elasticsearch"; -import { ConductorError, ErrorCodes } from "../../utils/errors"; +import { ErrorFactory, ErrorCodes } from "../../utils/errors"; import { Logger } from "../../utils/logger"; /** @@ -21,13 +22,14 @@ interface BulkOptions { /** * Sends a bulk write request to Elasticsearch. + * Enhanced with ErrorFactory patterns for better error handling. * * @param client - The Elasticsearch client instance * @param records - An array of records to be indexed * @param indexName - The name of the Elasticsearch index * @param onFailure - Callback function to handle failed records * @param options - Optional configuration for bulk operations - * @throws Error after all retries are exhausted + * @throws Enhanced ConductorError with specific guidance if bulk operation fails */ export async function sendBulkWriteRequest( client: Client, @@ -39,69 +41,185 @@ export async function sendBulkWriteRequest( const maxRetries = options.maxRetries || 3; const refresh = options.refresh !== undefined ? options.refresh : true; + // Enhanced parameter validation + if (!client) { + throw ErrorFactory.args( + "Elasticsearch client is required for bulk operations", + "bulk", + [ + "Ensure Elasticsearch client is properly initialized", + "Check client connection and configuration", + "Verify Elasticsearch service is running", + ] + ); + } + + if (!records || records.length === 0) { + throw ErrorFactory.args("No records provided for bulk indexing", "bulk", [ + "Ensure records array is not empty", + "Check data processing pipeline", + "Verify CSV file contains data", + ]); + } + + if (!indexName || typeof indexName !== "string") { + throw ErrorFactory.args( + "Valid index name is required for bulk operations", + "bulk", + [ + "Provide a valid Elasticsearch index name", + "Check index name configuration", + "Use --index parameter to specify target index", + ] + ); + } + let attempt = 0; let success = false; + let lastError: Error | null = null; + + Logger.debugString( + `Attempting bulk write of ${records.length} records to index '${indexName}'` + ); while (attempt < maxRetries && !success) { try { + attempt++; + Logger.debugString(`Bulk write attempt ${attempt}/${maxRetries}`); + + // Prepare bulk request body const body = records.flatMap((doc) => [ { index: { _index: indexName } }, doc, ]); + // Execute bulk request const { body: result } = await client.bulk({ body, refresh, }); + // Enhanced error handling for bulk response if (result.errors) { let failureCount = 0; + const errorDetails: string[] = []; + result.items.forEach((item: any, index: number) => { if (item.index?.error) { failureCount++; - Logger.error( - `Bulk indexing error for record ${index}: status=${ - item.index.status - }, error=${JSON.stringify(item.index.error)}, document=${ - item.index._id - }` + const error = item.index.error; + const errorType = error.type || "unknown"; + const errorReason = error.reason || "unknown reason"; + + Logger.errorString( + `Bulk indexing error for record ${index}: status=${item.index.status}, type=${errorType}, reason=${errorReason}` ); + + // Collect unique error types for better feedback + const errorSummary = `${errorType}: ${errorReason}`; + if (!errorDetails.includes(errorSummary)) { + errorDetails.push(errorSummary); + } } }); onFailure(failureCount); + // Enhanced error analysis and suggestions + if (failureCount === records.length) { + // All records failed + throw ErrorFactory.index( + `All ${records.length} records failed bulk indexing`, + indexName, + [ + "Check index mapping compatibility with data", + "Verify index exists and is writable", + "Review data format and field types", + ...errorDetails.slice(0, 3).map((detail) => `Error: ${detail}`), + "Use smaller batch sizes if documents are too large", + "Check Elasticsearch cluster health and resources", + ] + ); + } else if (failureCount > records.length * 0.5) { + // More than half failed + Logger.warnString( + `High failure rate: ${failureCount}/${records.length} records failed` + ); + Logger.tipString("Consider reviewing data format and index mappings"); + } + // If some records succeeded, consider it a partial success if (failureCount < records.length) { success = true; - } else { - attempt++; + Logger.infoString( + `Partial success: ${records.length - failureCount}/${ + records.length + } records indexed successfully` + ); } } else { + // All records succeeded success = true; + Logger.debugString( + `All ${records.length} records indexed successfully` + ); } } catch (error) { - Logger.error( - `Error sending to Elasticsearch (Attempt ${attempt + 1}): ${ - error instanceof Error ? error.message : String(error) - }` + lastError = error instanceof Error ? error : new Error(String(error)); + + Logger.errorString( + `Bulk indexing attempt ${attempt} failed: ${lastError.message}` ); onFailure(records.length); - attempt++; if (attempt < maxRetries) { - Logger.info(`Retrying... (${attempt}/${maxRetries})`); - // Add backoff delay between retries - await new Promise((resolve) => setTimeout(resolve, 1000 * attempt)); + const backoffDelay = 1000 * attempt; // Exponential backoff + Logger.infoString( + `Retrying in ${backoffDelay}ms... (${attempt}/${maxRetries})` + ); + await new Promise((resolve) => setTimeout(resolve, backoffDelay)); } } } if (!success) { - Logger.error(`Failed to send bulk request after ${maxRetries} attempts.`); - throw new ConductorError( - "Failed to send bulk request after retries", - ErrorCodes.ES_ERROR + // Enhanced error message based on the type of failure + const errorMessage = lastError?.message || "Unknown bulk operation error"; + + let suggestions = [ + "Check Elasticsearch service connectivity and health", + "Verify index exists and has proper permissions", + "Review data format and compatibility with index mapping", + "Consider reducing batch size for large documents", + "Check cluster resources (disk space, memory)", + ]; + + // Add specific suggestions based on error patterns + if (errorMessage.includes("timeout")) { + suggestions.unshift("Increase timeout settings for large batches"); + suggestions.unshift("Try smaller batch sizes to reduce processing time"); + } else if ( + errorMessage.includes("memory") || + errorMessage.includes("heap") + ) { + suggestions.unshift("Reduce batch size to lower memory usage"); + suggestions.unshift("Check Elasticsearch heap size configuration"); + } else if (errorMessage.includes("mapping")) { + suggestions.unshift("Check field mappings match your data types"); + suggestions.unshift("Update index mapping or modify data format"); + } else if ( + errorMessage.includes("permission") || + errorMessage.includes("403") + ) { + suggestions.unshift("Check index write permissions"); + suggestions.unshift("Verify authentication credentials"); + } + + throw ErrorFactory.connection( + `Bulk indexing failed after ${maxRetries} attempts: ${errorMessage}`, + "Elasticsearch", + undefined, + suggestions ); } } diff --git a/apps/conductor/src/services/elasticsearch/client.ts b/apps/conductor/src/services/elasticsearch/client.ts index a8274695..59433229 100644 --- a/apps/conductor/src/services/elasticsearch/client.ts +++ b/apps/conductor/src/services/elasticsearch/client.ts @@ -1,81 +1,257 @@ /** * Elasticsearch Client Module * + * Enhanced with ErrorFactory patterns for better error handling and user feedback. * Provides functions for creating and managing Elasticsearch client connections. */ import { Client, ClientOptions } from "@elastic/elasticsearch"; import { Config } from "../../types/cli"; -import { ConductorError, ErrorCodes } from "../../utils/errors"; +import { ErrorFactory } from "../../utils/errors"; import { Logger } from "../../utils/logger"; /** - * Interface for Elasticsearch client options + * Interface for Elasticsearch client options with enhanced validation */ interface ESClientOptions { url: string; username?: string; password?: string; requestTimeout?: number; + retries?: number; } /** - * Creates an Elasticsearch client from application config. + * Enhanced client creation from application config with comprehensive validation * * @param config - Application configuration * @returns A configured Elasticsearch client instance + * @throws Enhanced ConductorError if client creation fails */ export function createClientFromConfig(config: Config): Client { - // Use a default localhost URL if no URL is provided - const url = config.elasticsearch.url || "http://localhost:9200"; + // Enhanced URL validation and defaults + const url = validateAndNormalizeUrl(config.elasticsearch?.url); - Logger.info(`Connecting to Elasticsearch at: ${url}`); + Logger.info`Connecting to Elasticsearch at: ${url}`; - return createClient({ + // Validate authentication configuration + const authConfig = validateAuthConfiguration(config.elasticsearch); + + // Create client options with enhanced validation + const esClientOptions: ESClientOptions = { url, - username: config.elasticsearch.user, - password: config.elasticsearch.password, - }); + username: authConfig.user, + password: authConfig.password, + requestTimeout: 30000, // Increased default timeout + retries: 3, + }; + + return createClient(esClientOptions); } /** - * Validates connection to Elasticsearch + * Enhanced connection validation with detailed health information * * @param client - Elasticsearch client instance * @returns Promise resolving to true if connection is valid - * @throws ConductorError if connection fails + * @throws Enhanced ConductorError with specific guidance if connection fails */ export async function validateConnection(client: Client): Promise { try { - const result = await client.info(); - Logger.debug( - `Connected to Elasticsearch cluster: ${result.body.cluster_name}` - ); + Logger.debug`Validating Elasticsearch connection...`; + + // Enhanced connection test with timeout + const startTime = Date.now(); + const [infoResult, healthResult] = await Promise.all([ + Promise.race([ + client.info(), + new Promise((_, reject) => + setTimeout(() => reject(new Error("Info request timeout")), 10000) + ), + ]), + Promise.race([ + client.cluster.health(), + new Promise((_, reject) => + setTimeout(() => reject(new Error("Health check timeout")), 10000) + ), + ]).catch(() => null), // Health check is optional + ]); + + const responseTime = Date.now() - startTime; + + // Extract connection information + const info = (infoResult as any).body; + const health = healthResult ? (healthResult as any).body : null; + + // Log detailed connection information + Logger.debug`Connected to Elasticsearch cluster successfully (${responseTime}ms)`; + Logger.debug`Cluster: ${info.cluster_name}`; + Logger.debug`Version: ${info.version.number}`; + + if (health) { + Logger.debug`Cluster Status: ${health.status}`; + Logger.debug`Active Nodes: ${health.number_of_nodes}`; + + // Provide health warnings + if (health.status === "red") { + Logger.warn`Cluster health is RED - some data may be unavailable`; + Logger.tipString("Check cluster configuration and node status"); + } else if (health.status === "yellow") { + Logger.warn`Cluster health is YELLOW - replicas may be missing`; + Logger.tipString( + "This is often normal for single-node development clusters" + ); + } + } + + // Version compatibility check + validateElasticsearchVersion(info.version.number); + return true; + } catch (error: any) { + // Enhanced error analysis + const connectionError = analyzeConnectionError(error); + throw connectionError; + } +} + +/** + * Enhanced URL validation and normalization + */ +function validateAndNormalizeUrl(url?: string): string { + if (!url) { + Logger.info`No Elasticsearch URL specified, using default: http://localhost:9200`; + return "http://localhost:9200"; + } + + // Validate URL format + try { + const parsedUrl = new URL(url); + + // Validate protocol + if (!["http:", "https:"].includes(parsedUrl.protocol)) { + throw ErrorFactory.config( + `Invalid Elasticsearch URL protocol: ${parsedUrl.protocol}`, + "url", + [ + "Use HTTP or HTTPS protocol", + "Example: http://localhost:9200", + "Example: https://elasticsearch.company.com:9200", + "Check if SSL/TLS is required", + ] + ); + } + + // Validate port + if (parsedUrl.port && isNaN(parseInt(parsedUrl.port))) { + throw ErrorFactory.config( + `Invalid port in Elasticsearch URL: ${parsedUrl.port}`, + "url", + [ + "Use a valid port number", + "Default Elasticsearch port is 9200", + "Check your Elasticsearch configuration", + "Example: http://localhost:9200", + ] + ); + } + + // Log URL details for debugging + Logger.debug`Elasticsearch URL validated: ${parsedUrl.protocol}//${parsedUrl.host}`; + + return url; } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - throw new ConductorError( - `Failed to connect to Elasticsearch: ${errorMessage}`, - ErrorCodes.CONNECTION_ERROR, - error + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + throw ErrorFactory.config( + `Invalid Elasticsearch URL format: ${url}`, + "url", + [ + "Use a valid URL format with protocol", + "Example: http://localhost:9200", + "Example: https://elasticsearch.company.com:9200", + "Check for typos in the URL", + "Ensure proper protocol (http:// or https://)", + ] ); } } /** - * Creates an Elasticsearch client using the provided configuration. - * Private helper function for createClientFromConfig. - * - * @param options - Configuration options for the Elasticsearch client - * @returns A configured Elasticsearch client instance - * @throws ConductorError if client creation fails + * Enhanced authentication configuration validation */ -function createClient(options: ESClientOptions): Client { +function validateAuthConfiguration(esConfig: any): { + user?: string; + password?: string; +} { + const user = esConfig?.user; + const password = esConfig?.password; + + // If one auth field is provided, both should be provided + if ((user && !password) || (!user && password)) { + throw ErrorFactory.config( + "Incomplete Elasticsearch authentication configuration", + "authentication", + [ + "Provide both username and password, or neither", + "Use --user and --password parameters together", + "Set both ELASTICSEARCH_USER and ELASTICSEARCH_PASSWORD environment variables", + "Check if authentication is required for your Elasticsearch instance", + ] + ); + } + + if (user && password) { + Logger.debug`Using authentication for user: ${user}`; + + // Validate username format + if (typeof user !== "string" || user.trim() === "") { + throw ErrorFactory.config( + "Invalid Elasticsearch username format", + "user", + [ + "Username must be a non-empty string", + "Check username spelling and format", + "Verify username exists in Elasticsearch", + ] + ); + } + + // Validate password format + if (typeof password !== "string" || password.trim() === "") { + throw ErrorFactory.config( + "Invalid Elasticsearch password format", + "password", + [ + "Password must be a non-empty string", + "Check password is correct", + "Verify password hasn't expired", + ] + ); + } + } else { + Logger.debug`Using Elasticsearch without authentication`; + } + + return { user, password }; +} + +/** + * Enhanced client options creation with validation + */ +function createClientOptions(options: ESClientOptions): ClientOptions { const clientOptions: ClientOptions = { node: options.url, - requestTimeout: options.requestTimeout || 10000, // 10 seconds timeout + requestTimeout: options.requestTimeout || 30000, + maxRetries: options.retries || 3, + resurrectStrategy: "ping", + sniffOnStart: false, // Disable sniffing for simpler setup + sniffOnConnectionFault: false, }; + // Add authentication if provided if (options.username && options.password) { clientOptions.auth = { username: options.username, @@ -83,14 +259,238 @@ function createClient(options: ESClientOptions): Client { }; } + return clientOptions; +} + +/** + * Enhanced Elasticsearch client creation with error handling + */ +function createClient(options: ESClientOptions): Client { + try { + const clientOptions = createClientOptions(options); + const client = new Client(clientOptions); + + Logger.debug`Elasticsearch client created successfully`; + + return client; + } catch (error) { + throw ErrorFactory.connection( + "Failed to create Elasticsearch client", + "Elasticsearch", + options.url, + [ + "Check Elasticsearch configuration parameters", + "Verify URL format and accessibility", + "Ensure authentication credentials are correct", + "Check client library compatibility", + "Review connection settings", + ] + ); + } +} + +/** + * Validate Elasticsearch version compatibility + */ +function validateElasticsearchVersion(version: string): void { try { - return new Client(clientOptions); + const versionParts = version.split(".").map((part) => parseInt(part)); + const majorVersion = versionParts[0]; + const minorVersion = versionParts[1]; + + // Check for supported versions (7.x and 8.x) + if (majorVersion < 7) { + Logger.warn`Elasticsearch version ${version} is quite old and may have compatibility issues`; + Logger.tipString( + "Consider upgrading to Elasticsearch 7.x or 8.x for better features and support" + ); + } else if (majorVersion > 8) { + Logger.warn`Elasticsearch version ${version} is newer than tested versions`; + Logger.tipString( + "This client library may not support all features of this Elasticsearch version" + ); + } else { + Logger.debug`Elasticsearch version ${version} is supported`; + } + + // Specific version warnings + if (majorVersion === 7 && minorVersion < 10) { + Logger.warn`Elasticsearch 7.${minorVersion} has known issues with some operations`; + Logger.tipString( + "Consider upgrading to Elasticsearch 7.10+ for better stability" + ); + } } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - throw new ConductorError( - `Failed to create Elasticsearch client: ${errorMessage}`, - ErrorCodes.CONNECTION_ERROR, - error + Logger.debug`Could not parse Elasticsearch version: ${version}`; + // Don't throw - version parsing is informational + } +} + +/** + * Enhanced connection error analysis + */ +function analyzeConnectionError(error: any): Error { + const errorMessage = error.message || String(error); + + // Connection refused + if (errorMessage.includes("ECONNREFUSED")) { + return ErrorFactory.connection( + "Cannot connect to Elasticsearch - connection refused", + "Elasticsearch", + undefined, + [ + "Check that Elasticsearch is running", + "Verify the URL and port are correct", + "Ensure no firewall is blocking the connection", + "Check if Elasticsearch is binding to the correct interface", + "Test with: curl http://localhost:9200", + ] ); } + + // Timeout errors + if (errorMessage.includes("timeout") || errorMessage.includes("ETIMEDOUT")) { + return ErrorFactory.connection( + "Elasticsearch connection timed out", + "Elasticsearch", + undefined, + [ + "Elasticsearch may be starting up or overloaded", + "Check Elasticsearch service health and performance", + "Verify network connectivity and latency", + "Consider increasing timeout settings", + "Check system resources (CPU, memory, disk space)", + "Review Elasticsearch logs for performance issues", + ] + ); + } + + // Authentication errors + if (errorMessage.includes("401") || errorMessage.includes("Unauthorized")) { + return ErrorFactory.connection( + "Elasticsearch authentication failed", + "Elasticsearch", + undefined, + [ + "Check username and password are correct", + "Verify authentication credentials haven't expired", + "Ensure user has proper cluster permissions", + "Check if authentication is enabled on this Elasticsearch instance", + "Review Elasticsearch security configuration", + ] + ); + } + + // Permission errors + if (errorMessage.includes("403") || errorMessage.includes("Forbidden")) { + return ErrorFactory.connection( + "Elasticsearch access forbidden - insufficient permissions", + "Elasticsearch", + undefined, + [ + "User lacks necessary cluster or index permissions", + "Check user roles and privileges in Elasticsearch", + "Verify cluster-level permissions", + "Contact Elasticsearch administrator for access", + "Review security policy and user roles", + ] + ); + } + + // SSL/TLS errors + if ( + errorMessage.includes("SSL") || + errorMessage.includes("certificate") || + errorMessage.includes("CERT") + ) { + return ErrorFactory.connection( + "Elasticsearch SSL/TLS connection error", + "Elasticsearch", + undefined, + [ + "Check SSL certificate validity and trust", + "Verify TLS configuration matches server settings", + "Ensure proper SSL/TLS version compatibility", + "Check if HTTPS is required for this instance", + "Try HTTP if HTTPS is causing issues (development only)", + "Verify certificate authority and trust chain", + ] + ); + } + + // DNS resolution errors + if ( + errorMessage.includes("ENOTFOUND") || + errorMessage.includes("getaddrinfo") + ) { + return ErrorFactory.connection( + "Cannot resolve Elasticsearch hostname", + "Elasticsearch", + undefined, + [ + "Check hostname spelling in the URL", + "Verify DNS resolution is working", + "Try using IP address instead of hostname", + "Check network connectivity and DNS servers", + "Test with: nslookup ", + "Verify hosts file doesn't have conflicting entries", + ] + ); + } + + // Version compatibility errors + if ( + errorMessage.includes("version") || + errorMessage.includes("compatibility") + ) { + return ErrorFactory.connection( + "Elasticsearch version compatibility issue", + "Elasticsearch", + undefined, + [ + "Check Elasticsearch version compatibility with client", + "Verify client library version supports your Elasticsearch version", + "Update client library if needed", + "Check Elasticsearch version with: GET /", + "Review compatibility matrix in documentation", + "Consider upgrading Elasticsearch or downgrading client", + ] + ); + } + + // Network errors + if ( + errorMessage.includes("ENOTCONN") || + errorMessage.includes("ECONNRESET") + ) { + return ErrorFactory.connection( + "Elasticsearch network connection error", + "Elasticsearch", + undefined, + [ + "Network connection was interrupted", + "Check network stability and connectivity", + "Verify Elasticsearch service stability", + "Check for network proxies or load balancers", + "Review firewall and security group settings", + "Consider connection pooling or retry strategies", + ] + ); + } + + // Generic connection error with enhanced context + return ErrorFactory.connection( + `Elasticsearch connection failed: ${errorMessage}`, + "Elasticsearch", + undefined, + [ + "Check Elasticsearch service is running and accessible", + "Verify connection parameters (URL, auth, etc.)", + "Review network connectivity and firewall settings", + "Check Elasticsearch service logs for errors", + "Test basic connectivity with curl or similar tool", + "Ensure Elasticsearch is properly configured", + "Use --debug flag for detailed connection information", + ] + ); } diff --git a/apps/conductor/src/services/lectern/index.ts b/apps/conductor/src/services/lectern/index.ts index 2e143888..e16f4d08 100644 --- a/apps/conductor/src/services/lectern/index.ts +++ b/apps/conductor/src/services/lectern/index.ts @@ -1,3 +1,3 @@ // src/services/lectern/index.ts -export { LecternService } from "./LecternService"; +export { LecternService } from "./lecternService"; export * from "./types"; diff --git a/apps/conductor/src/services/lectern/lecternService.ts b/apps/conductor/src/services/lectern/lecternService.ts index 11725e09..0c650f06 100644 --- a/apps/conductor/src/services/lectern/lecternService.ts +++ b/apps/conductor/src/services/lectern/lecternService.ts @@ -1,8 +1,8 @@ -// src/services/lectern/LecternService.ts +// src/services/lectern/LecternService.ts - Enhanced with ErrorFactory patterns import { BaseService } from "../base/baseService"; import { ServiceConfig } from "../base/types"; import { Logger } from "../../utils/logger"; -import { ConductorError, ErrorCodes } from "../../utils/errors"; +import { ErrorFactory } from "../../utils/errors"; import { LecternSchemaUploadParams, LecternUploadResponse, @@ -24,60 +24,59 @@ export class LecternService extends BaseService { } /** - * Upload a schema to Lectern + * Upload a schema to Lectern with enhanced error handling */ async uploadSchema( params: LecternSchemaUploadParams ): Promise { try { - this.validateRequired(params, ["schemaContent"]); + this.validateRequired(params, ["schemaContent"], "schema upload"); - // Parse and validate JSON + // Enhanced JSON parsing and validation let schemaData: any; try { schemaData = JSON.parse(params.schemaContent); } catch (error) { - throw new ConductorError( - `Invalid schema format: ${ - error instanceof Error ? error.message : String(error) - }`, - ErrorCodes.INVALID_FILE, - error + throw ErrorFactory.validation( + "Invalid JSON format in Lectern schema", + { error: error instanceof Error ? error.message : String(error) }, + [ + "Check JSON syntax for errors (missing commas, brackets, quotes)", + "Validate JSON structure using a JSON validator", + "Ensure file encoding is UTF-8", + "Try viewing the file in a JSON editor", + error instanceof Error ? `JSON error: ${error.message}` : "", + ].filter(Boolean) ); } - // Basic schema validation - if (!schemaData.name) { - throw new ConductorError( - 'Schema must have a "name" field', - ErrorCodes.VALIDATION_FAILED - ); - } - - if (!schemaData.schemas || typeof schemaData.schemas !== "object") { - throw new ConductorError( - 'Schema must have a "schema" field containing the JSON schema definition', - ErrorCodes.VALIDATION_FAILED - ); - } + // Enhanced schema structure validation + this.validateLecternSchemaStructure(schemaData); - Logger.info(`Uploading schema: ${schemaData.name}`); + Logger.debug`Uploading Lectern schema: ${schemaData.name}`; - // Upload to Lectern + // Upload to Lectern with enhanced error handling const response = await this.http.post( "/dictionaries", schemaData ); - // Check for errors in response + // Enhanced response validation if (response.data?.error) { - throw new ConductorError( + throw ErrorFactory.connection( `Lectern API error: ${response.data.error}`, - ErrorCodes.CONNECTION_ERROR + "Lectern", + this.config.url, + [ + "Check schema format and structure", + "Verify Lectern service is properly configured", + "Review schema for required fields and valid values", + "Check Lectern service logs for additional details", + ] ); } - Logger.success(`Schema "${schemaData.name}" uploaded successfully`); + Logger.success`Lectern schema uploaded successfully: ${schemaData.name}`; return response.data; } catch (error) { @@ -86,56 +85,118 @@ export class LecternService extends BaseService { } /** - * Get all dictionaries from Lectern + * Get all dictionaries from Lectern with enhanced error handling */ async getDictionaries(): Promise { try { + Logger.debug`Fetching all dictionaries from Lectern`; + const response = await this.http.get( "/dictionaries" ); - return Array.isArray(response.data) ? response.data : []; + + const dictionaries = Array.isArray(response.data) ? response.data : []; + + Logger.debug`Retrieved ${dictionaries.length} dictionaries from Lectern`; + + return dictionaries; } catch (error) { this.handleServiceError(error, "get dictionaries"); } } /** - * Get a specific dictionary by ID + * Get a specific dictionary by ID with enhanced error handling */ async getDictionary(dictionaryId: string): Promise { try { + if (!dictionaryId || typeof dictionaryId !== "string") { + throw ErrorFactory.args( + "Dictionary ID required to fetch dictionary", + undefined, + [ + "Provide a valid dictionary ID", + "Dictionary ID should be a non-empty string", + "Get available dictionary IDs with getDictionaries()", + ] + ); + } + + Logger.debug`Fetching dictionary from Lectern: ${dictionaryId}`; + const response = await this.http.get( - `/dictionaries/${dictionaryId}` + `/dictionaries/${encodeURIComponent(dictionaryId)}` ); + + Logger.debug`Successfully retrieved dictionary: ${ + response.data.name || dictionaryId + }`; + return response.data; } catch (error) { + // Enhanced error handling for 404 cases + if (error instanceof Error && error.message.includes("404")) { + throw ErrorFactory.validation( + `Dictionary not found in Lectern: ${dictionaryId}`, + { dictionaryId }, + [ + "Check that the dictionary ID is correct", + "Verify the dictionary exists in Lectern", + "Use getDictionaries() to see available dictionaries", + "Ensure the dictionary was successfully uploaded", + ] + ); + } + this.handleServiceError(error, "get dictionary"); } } /** - * Find a dictionary by name and version + * Find a dictionary by name and version with enhanced error handling */ async findDictionary( name: string, version: string ): Promise { try { + if (!name || !version) { + throw ErrorFactory.args( + "Dictionary name and version required for search", + undefined, + [ + "Provide both dictionary name and version", + "Name and version must be non-empty strings", + "Example: findDictionary('clinical-data', '1.0')", + ] + ); + } + + Logger.debug`Searching for dictionary: ${name} v${version}`; + const dictionaries = await this.getDictionaries(); const dictionary = dictionaries.find( (dict) => dict.name === name && dict.version === version ); + if (dictionary) { + Logger.debug`Found dictionary: ${name} v${version} (ID: ${dictionary._id})`; + } else { + Logger.debug`Dictionary not found: ${name} v${version}`; + } + return dictionary || null; } catch (error) { - Logger.warn(`Could not find dictionary ${name} v${version}: ${error}`); + Logger.warn`Could not search for dictionary ${name} v${version}: ${ + error instanceof Error ? error.message : String(error) + }`; return null; } } /** - * Validate that a centric entity exists in a dictionary + * Validate that a centric entity exists in a dictionary with enhanced feedback */ async validateCentricEntity( dictionaryName: string, @@ -143,9 +204,19 @@ export class LecternService extends BaseService { centricEntity: string ): Promise { try { - Logger.info( - `Validating entity '${centricEntity}' in dictionary '${dictionaryName}' v${dictionaryVersion}` - ); + if (!dictionaryName || !dictionaryVersion || !centricEntity) { + throw ErrorFactory.args( + "Dictionary name, version, and centric entity required for validation", + undefined, + [ + "Provide all required parameters: name, version, entity", + "All parameters must be non-empty strings", + "Example: validateCentricEntity('clinical-data', '1.0', 'donor')", + ] + ); + } + + Logger.info`Validating centric entity '${centricEntity}' in dictionary '${dictionaryName}' v${dictionaryVersion}`; // Find the dictionary const dictionary = await this.findDictionary( @@ -154,6 +225,7 @@ export class LecternService extends BaseService { ); if (!dictionary) { + Logger.warn`Dictionary not found: ${dictionaryName} v${dictionaryVersion}`; return { exists: false, entities: [], @@ -164,15 +236,16 @@ export class LecternService extends BaseService { // Get detailed dictionary info with schemas const detailedDict = await this.getDictionary(dictionary._id); - // Extract entity names from schemas - const entities = detailedDict.schemas?.map((schema) => schema.name) || []; + // Extract entity names from schemas with enhanced validation + const entities = this.extractEntitiesFromSchemas(detailedDict.schemas); const entityExists = entities.includes(centricEntity); if (entityExists) { - Logger.info(`✓ Entity '${centricEntity}' found in dictionary`); + Logger.success`Centric entity validated: ${centricEntity}`; } else { - Logger.warn(`⚠ Entity '${centricEntity}' not found in dictionary`); + Logger.warn`Centric entity not found in dictionary: ${centricEntity}`; + Logger.info`Available entities: ${entities.join(", ")}`; } return { @@ -186,38 +259,226 @@ export class LecternService extends BaseService { } /** - * Get all available entities across all dictionaries + * Get all available entities across all dictionaries with enhanced error handling */ async getAllEntities(): Promise { try { + Logger.debug`Fetching all entities from all Lectern dictionaries`; + const dictionaries = await this.getDictionaries(); const allEntities = new Set(); for (const dict of dictionaries) { - const detailedDict = await this.getDictionary(dict._id); - detailedDict.schemas?.forEach((schema) => { - if (schema.name) { - allEntities.add(schema.name); - } - }); + try { + const detailedDict = await this.getDictionary(dict._id); + const entities = this.extractEntitiesFromSchemas( + detailedDict.schemas + ); + entities.forEach((entity) => allEntities.add(entity)); + } catch (error) { + Logger.warn`Could not process dictionary ${dict.name || dict._id}: ${ + error instanceof Error ? error.message : String(error) + }`; + continue; + } } - return Array.from(allEntities); + const entitiesArray = Array.from(allEntities); + Logger.debug`Found ${entitiesArray.length} unique entities across all dictionaries`; + + return entitiesArray; } catch (error) { this.handleServiceError(error, "get all entities"); } } /** - * Check if Lectern has any dictionaries + * Check if Lectern has any dictionaries with enhanced feedback */ async hasDictionaries(): Promise { try { const dictionaries = await this.getDictionaries(); - return dictionaries.length > 0; + const hasDicts = dictionaries.length > 0; + + Logger.debug`Lectern has ${dictionaries.length} dictionaries`; + + return hasDicts; } catch (error) { - Logger.warn(`Could not check for dictionaries: ${error}`); + Logger.warn`Could not check for dictionaries: ${ + error instanceof Error ? error.message : String(error) + }`; return false; } } + + /** + * Enhanced Lectern schema structure validation + */ + private validateLecternSchemaStructure(schema: any): void { + if (!schema || typeof schema !== "object") { + throw ErrorFactory.validation( + "Invalid Lectern schema structure", + { schema: typeof schema }, + [ + "Schema must be a valid JSON object", + "Check that the file contains proper schema definition", + "Ensure the schema follows Lectern format requirements", + "Review Lectern documentation for schema structure", + ] + ); + } + + // Check for required Lectern schema fields + const requiredFields = ["name", "schemas"]; + const missingFields = requiredFields.filter((field) => !schema[field]); + + if (missingFields.length > 0) { + throw ErrorFactory.validation( + "Missing required fields in Lectern schema", + { + missingFields, + providedFields: Object.keys(schema), + schema: schema, + }, + [ + `Add missing fields: ${missingFields.join(", ")}`, + "Lectern schemas require 'name' and 'schemas' fields", + "The 'name' field should be a descriptive string", + "The 'schemas' field should be an array of schema definitions", + "Check Lectern documentation for required schema format", + ] + ); + } + + // Enhanced name validation + if (typeof schema.name !== "string" || schema.name.trim() === "") { + throw ErrorFactory.validation( + "Invalid schema name in Lectern schema", + { name: schema.name, type: typeof schema.name }, + [ + "Schema 'name' must be a non-empty string", + "Use a descriptive name for the schema", + "Example: 'clinical-data-dictionary' or 'genomic-metadata'", + "Avoid special characters in schema names", + ] + ); + } + + // Enhanced schemas array validation + if (!Array.isArray(schema.schemas)) { + throw ErrorFactory.validation( + "Invalid 'schemas' field in Lectern schema", + { schemas: typeof schema.schemas, provided: schema.schemas }, + [ + "'schemas' field must be an array", + "Include at least one schema definition", + "Each schema should define entity structure", + "Check array syntax and structure", + ] + ); + } + + if (schema.schemas.length === 0) { + throw ErrorFactory.validation( + "Empty schemas array in Lectern schema", + { schemaName: schema.name }, + [ + "Include at least one schema definition", + "Add schema objects to the 'schemas' array", + "Each schema should define an entity type", + "Check if schemas were properly defined", + ] + ); + } + + // Validate individual schema entries + schema.schemas.forEach((schemaEntry: any, index: number) => { + if (!schemaEntry.name) { + throw ErrorFactory.validation( + `Schema entry ${index + 1} missing 'name' field`, + { index, schema: schemaEntry }, + [ + "Each schema in the array must have a 'name' field", + "Names identify entity types (e.g., 'donor', 'specimen')", + "Ensure all schema entries are properly formatted", + "Check schema entry structure and required fields", + ] + ); + } + }); + + Logger.debug`Lectern schema structure validated: ${schema.name} with ${schema.schemas.length} schema(s)`; + } + + /** + * Extract entities from schemas with error handling + */ + private extractEntitiesFromSchemas(schemas?: any[]): string[] { + if (!schemas || !Array.isArray(schemas)) { + Logger.debug`No schemas provided or invalid schemas array`; + return []; + } + + const entities: string[] = []; + + schemas.forEach((schema, index) => { + if (schema?.name && typeof schema.name === "string") { + entities.push(schema.name); + } else { + Logger.debug`Schema ${index} missing or invalid name field`; + } + }); + + return entities; + } + + /** + * Enhanced service error handling with Lectern-specific context + */ + protected handleServiceError(error: unknown, operation: string): never { + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + // Enhanced error handling with Lectern-specific guidance + const errorMessage = error instanceof Error ? error.message : String(error); + + let suggestions = [ + `Check that Lectern service is running and accessible`, + `Verify service URL: ${this.config.url}`, + "Check network connectivity and firewall settings", + "Confirm Lectern service configuration", + "Review Lectern service logs for additional details", + ]; + + // Add operation-specific suggestions + if (operation === "schema upload") { + suggestions = [ + "Verify schema format follows Lectern requirements", + "Ensure schema has required 'name' and 'schemas' fields", + "Check for valid JSON structure and syntax", + ...suggestions, + ]; + } else if (operation === "get dictionaries") { + suggestions = [ + "Lectern service may not have any dictionaries uploaded", + "Verify Lectern API endpoint is accessible", + ...suggestions, + ]; + } else if (operation === "centric entity validation") { + suggestions = [ + "Check that dictionary exists in Lectern", + "Verify entity name spelling and case", + "Ensure dictionary has properly defined schemas", + ...suggestions, + ]; + } + + throw ErrorFactory.connection( + `Lectern ${operation} failed: ${errorMessage}`, + "Lectern", + this.config.url, + suggestions + ); + } } diff --git a/apps/conductor/src/services/lyric/LyricRegistrationService.ts b/apps/conductor/src/services/lyric/LyricRegistrationService.ts index 8fea2a60..0c347813 100644 --- a/apps/conductor/src/services/lyric/LyricRegistrationService.ts +++ b/apps/conductor/src/services/lyric/LyricRegistrationService.ts @@ -1,8 +1,8 @@ -// src/services/lyric/LyricRegistrationService.ts +// src/services/lyric/LyricRegistrationService.ts - Enhanced with ErrorFactory patterns import { BaseService } from "../base/baseService"; import { ServiceConfig } from "../base/types"; import { Logger } from "../../utils/logger"; -import { ConductorError, ErrorCodes } from "../../utils/errors"; +import { ErrorFactory } from "../../utils/errors"; import { DictionaryRegistrationParams, LyricRegistrationResponse, @@ -22,31 +22,22 @@ export class LyricRegistrationService extends BaseService { } /** - * Register a dictionary with the Lyric service + * Register a dictionary with the Lyric service with enhanced error handling */ async registerDictionary( params: DictionaryRegistrationParams ): Promise { try { - // Validate required parameters - this.validateRequired(params, [ - "categoryName", - "dictionaryName", - "dictionaryVersion", - "defaultCentricEntity", - ]); - - Logger.info( - `Registering dictionary: ${params.dictionaryName} v${params.dictionaryVersion}` - ); + // Enhanced parameter validation + this.validateRegistrationParams(params); + + Logger.debug`Registering Lyric dictionary: ${params.dictionaryName} v${params.dictionaryVersion}`; + Logger.debug`Registration details - Category: ${params.categoryName}, Entity: ${params.defaultCentricEntity}`; - // Prepare form data - const formData = new URLSearchParams(); - formData.append("categoryName", params.categoryName); - formData.append("dictionaryName", params.dictionaryName); - formData.append("dictionaryVersion", params.dictionaryVersion); - formData.append("defaultCentricEntity", params.defaultCentricEntity); + // Enhanced form data preparation + const formData = this.prepareRegistrationFormData(params); + // Make registration request with enhanced error handling const response = await this.http.post( "/dictionary/register", formData.toString(), @@ -57,15 +48,10 @@ export class LyricRegistrationService extends BaseService { } ); - // Check for API-level errors in response - if (response.data?.error) { - throw new ConductorError( - `Lyric API error: ${response.data.error}`, - ErrorCodes.CONNECTION_ERROR - ); - } + // Enhanced response validation + this.validateRegistrationResponse(response.data, params); - Logger.success("Dictionary registered successfully"); + Logger.debug`Dictionary registered service successful`; return { success: true, @@ -78,7 +64,7 @@ export class LyricRegistrationService extends BaseService { } /** - * Check if a dictionary is already registered + * Check if a dictionary is already registered with enhanced validation */ async checkDictionaryExists(params: { categoryName: string; @@ -86,39 +72,355 @@ export class LyricRegistrationService extends BaseService { dictionaryVersion: string; }): Promise { try { - // This would need to be implemented based on Lyric's API - // For now, returning false as a placeholder - Logger.debug( - `Checking if dictionary exists: ${params.dictionaryName} v${params.dictionaryVersion}` - ); + if ( + !params.categoryName || + !params.dictionaryName || + !params.dictionaryVersion + ) { + throw ErrorFactory.args( + "Category name, dictionary name, and version required to check existence", + undefined, + [ + "Provide all required parameters for dictionary lookup", + "All parameters must be non-empty strings", + "Example: checkDictionaryExists({categoryName: 'clinical', dictionaryName: 'data-dict', dictionaryVersion: '1.0'})", + ] + ); + } + + Logger.debug`Checking if dictionary exists: ${params.dictionaryName} v${params.dictionaryVersion} in category ${params.categoryName}`; + + // Implementation would depend on Lyric's API + // For now, returning false as a placeholder with enhanced logging + Logger.debug`Dictionary existence check not yet implemented - assuming dictionary does not exist`; + return false; } catch (error) { - Logger.warn(`Could not check dictionary existence: ${error}`); + Logger.warn`Could not check dictionary existence: ${ + error instanceof Error ? error.message : String(error) + }`; return false; } } /** - * Get list of registered dictionaries + * Get list of registered dictionaries with enhanced error handling */ async getDictionaries(): Promise { try { + Logger.debug`Fetching registered dictionaries from Lyric`; + const response = await this.http.get("/dictionaries"); - return Array.isArray(response.data) ? response.data : []; + + const dictionaries = Array.isArray(response.data) ? response.data : []; + + Logger.debug`Retrieved ${dictionaries.length} registered dictionaries from Lyric`; + + return dictionaries; } catch (error) { this.handleServiceError(error, "get dictionaries"); } } /** - * Get categories available in Lyric + * Get categories available in Lyric with enhanced error handling */ async getCategories(): Promise { try { + Logger.debug`Fetching available categories from Lyric`; + const response = await this.http.get("/categories"); - return Array.isArray(response.data) ? response.data : []; + + const categories = Array.isArray(response.data) ? response.data : []; + + Logger.debug`Retrieved ${categories.length} available categories from Lyric`; + + return categories; } catch (error) { this.handleServiceError(error, "get categories"); } } + + /** + * Enhanced validation of registration parameters + */ + private validateRegistrationParams( + params: DictionaryRegistrationParams + ): void { + // Validate required fields with enhanced error messages + const requiredFields = [ + "categoryName", + "dictionaryName", + "dictionaryVersion", + "defaultCentricEntity", + ]; + + this.validateRequired(params, requiredFields, "dictionary registration"); + + // Enhanced individual field validation + this.validateCategoryName(params.categoryName); + this.validateDictionaryName(params.dictionaryName); + this.validateDictionaryVersion(params.dictionaryVersion); + this.validateCentricEntity(params.defaultCentricEntity); + + Logger.debug`Registration parameters validated successfully`; + } + + /** + * Enhanced category name validation + */ + private validateCategoryName(categoryName: string): void { + if (typeof categoryName !== "string" || categoryName.trim() === "") { + throw ErrorFactory.validation( + "Invalid category name for Lyric registration", + { categoryName, type: typeof categoryName }, + [ + "Category name must be a non-empty string", + "Use descriptive names that group related dictionaries", + "Examples: 'clinical', 'genomics', 'metadata'", + "Avoid special characters and spaces", + ] + ); + } + + // Check for valid category name format + if (!/^[a-zA-Z0-9_-]+$/.test(categoryName)) { + throw ErrorFactory.validation( + `Category name contains invalid characters: ${categoryName}`, + { categoryName }, + [ + "Use only letters, numbers, hyphens, and underscores", + "Avoid spaces and special characters", + "Example: 'clinical-data' or 'genomic_metadata'", + "Keep names simple and descriptive", + ] + ); + } + + Logger.debug`Category name validated: ${categoryName}`; + } + + /** + * Enhanced dictionary name validation + */ + private validateDictionaryName(dictionaryName: string): void { + if (typeof dictionaryName !== "string" || dictionaryName.trim() === "") { + throw ErrorFactory.validation( + "Invalid dictionary name for Lyric registration", + { dictionaryName, type: typeof dictionaryName }, + [ + "Dictionary name must be a non-empty string", + "Use descriptive names like 'clinical-data' or 'genomic-metadata'", + "Names should match your Lectern schema name", + "Avoid special characters and spaces", + ] + ); + } + + // Validate name format + if (!/^[a-zA-Z0-9_-]+$/.test(dictionaryName)) { + throw ErrorFactory.validation( + `Dictionary name contains invalid characters: ${dictionaryName}`, + { dictionaryName }, + [ + "Use only letters, numbers, hyphens, and underscores", + "Avoid spaces and special characters", + "Example: 'clinical-data-v1' or 'genomic_metadata'", + "Keep names concise but descriptive", + ] + ); + } + + Logger.debug`Dictionary name validated: ${dictionaryName}`; + } + + /** + * Enhanced dictionary version validation + */ + private validateDictionaryVersion(version: string): void { + if (typeof version !== "string" || version.trim() === "") { + throw ErrorFactory.validation( + "Invalid dictionary version for Lyric registration", + { version, type: typeof version }, + [ + "Version must be a non-empty string", + "Use semantic versioning format: major.minor or major.minor.patch", + "Examples: '1.0', '2.1.3', '1.0.0-beta'", + "Increment versions when schema changes", + ] + ); + } + + // Basic version format validation (warn but don't fail) + if (!/^\d+(\.\d+)*(-[a-zA-Z0-9]+)?$/.test(version)) { + Logger.warn`Version format '${version}' doesn't follow semantic versioning`; + Logger.tipString("Consider using semantic versioning: major.minor.patch"); + } + + Logger.debug`Dictionary version validated: ${version}`; + } + + /** + * Enhanced centric entity validation + */ + private validateCentricEntity(centricEntity: string): void { + if (typeof centricEntity !== "string" || centricEntity.trim() === "") { + throw ErrorFactory.validation( + "Invalid centric entity for Lyric registration", + { centricEntity, type: typeof centricEntity }, + [ + "Centric entity must be a non-empty string", + "Use entity names from your dictionary schema", + "Examples: 'donor', 'specimen', 'sample', 'file'", + "Entity must be defined in your Lectern schema", + ] + ); + } + + // Basic entity name validation + if (!/^[a-zA-Z][a-zA-Z0-9_]*$/.test(centricEntity)) { + throw ErrorFactory.validation( + `Invalid centric entity format: ${centricEntity}`, + { centricEntity }, + [ + "Entity names must start with a letter", + "Use only letters, numbers, and underscores", + "Follow your schema's entity naming conventions", + "Examples: 'donor', 'specimen_data', 'sample_metadata'", + ] + ); + } + + Logger.debug`Centric entity validated: ${centricEntity}`; + } + + /** + * Prepare registration form data with enhanced validation + */ + private prepareRegistrationFormData( + params: DictionaryRegistrationParams + ): URLSearchParams { + const formData = new URLSearchParams(); + + // Add validated parameters + formData.append("categoryName", params.categoryName.trim()); + formData.append("dictionaryName", params.dictionaryName.trim()); + formData.append("dictionaryVersion", params.dictionaryVersion.trim()); + formData.append("defaultCentricEntity", params.defaultCentricEntity.trim()); + + Logger.debug`Form data prepared for registration`; + + return formData; + } + + /** + * Enhanced validation of registration response + */ + private validateRegistrationResponse( + responseData: any, + params: DictionaryRegistrationParams + ): void { + // Check for API-level errors in response + if (responseData?.error) { + throw ErrorFactory.connection( + `Lyric registration API error: ${responseData.error}`, + "Lyric", + this.config.url, + [ + "Check registration parameters format and values", + "Verify dictionary doesn't already exist", + "Ensure category is valid and accessible", + "Review Lyric service configuration", + "Check Lyric service logs for additional details", + ] + ); + } + + // Check for common response patterns that indicate issues + if (responseData?.success === false) { + const message = responseData.message || "Registration failed"; + throw ErrorFactory.validation( + `Dictionary registration rejected: ${message}`, + { responseData, params }, + [ + "Check if dictionary already exists in Lyric", + "Verify category permissions and access", + "Ensure centric entity is valid for the dictionary", + "Review registration parameters for correctness", + ] + ); + } + + Logger.debug`Registration response validated successfully`; + } + + /** + * Enhanced service error handling with Lyric-specific context + */ + protected handleServiceError(error: unknown, operation: string): never { + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + // Enhanced error handling with Lyric-specific guidance + const errorMessage = error instanceof Error ? error.message : String(error); + + let suggestions = [ + `Check that Lyric service is running and accessible`, + `Verify service URL: ${this.config.url}`, + "Check network connectivity and firewall settings", + "Confirm Lyric service configuration", + "Review Lyric service logs for additional details", + ]; + + // Add operation-specific suggestions + if (operation === "dictionary registration") { + suggestions = [ + "Verify all registration parameters are correct", + "Check if dictionary already exists in Lyric", + "Ensure category exists and is accessible", + "Verify centric entity matches dictionary schema", + "Confirm proper permissions for dictionary registration", + ...suggestions, + ]; + } else if (operation === "get dictionaries") { + suggestions = [ + "Lyric service may not have any dictionaries registered", + "Verify Lyric API endpoint is accessible", + "Check if authentication is required", + ...suggestions, + ]; + } else if (operation === "get categories") { + suggestions = [ + "Lyric service may not have any categories configured", + "Verify Lyric categories endpoint is accessible", + "Check if categories need to be created first", + ...suggestions, + ]; + } + + // Handle specific HTTP status codes + if (errorMessage.includes("409") || errorMessage.includes("conflict")) { + suggestions.unshift("Dictionary may already be registered in Lyric"); + suggestions.unshift( + "Check existing dictionaries or use a different name/version" + ); + } else if ( + errorMessage.includes("400") || + errorMessage.includes("validation") + ) { + suggestions.unshift("Registration parameters validation failed"); + suggestions.unshift("Check parameter format and required fields"); + } else if (errorMessage.includes("401") || errorMessage.includes("403")) { + suggestions.unshift("Authentication or authorization failed"); + suggestions.unshift("Check if API credentials are required"); + } + + throw ErrorFactory.connection( + `Lyric ${operation} failed: ${errorMessage}`, + "Lyric", + this.config.url, + suggestions + ); + } } diff --git a/apps/conductor/src/services/lyric/LyricSubmissionService.ts b/apps/conductor/src/services/lyric/LyricSubmissionService.ts index 751c5025..4529756a 100644 --- a/apps/conductor/src/services/lyric/LyricSubmissionService.ts +++ b/apps/conductor/src/services/lyric/LyricSubmissionService.ts @@ -1,8 +1,8 @@ -// src/services/lyric/LyricSubmissionService.ts +// src/services/lyric/LyricSubmissionService.ts - Enhanced with ErrorFactory patterns import { BaseService } from "../base/baseService"; import { ServiceConfig } from "../base/types"; import { Logger } from "../../utils/logger"; -import { ConductorError, ErrorCodes } from "../../utils/errors"; +import { ErrorFactory } from "../../utils/errors"; import * as fs from "fs"; import * as path from "path"; @@ -12,6 +12,7 @@ export interface DataSubmissionParams { dataDirectory: string; maxRetries?: number; retryDelay?: number; + [key: string]: any; } export interface DataSubmissionResult { @@ -36,22 +37,26 @@ export class LyricSubmissionService extends BaseService { /** * Complete data submission workflow: validate -> submit -> wait -> commit + * Enhanced with ErrorFactory patterns */ async submitDataWorkflow( params: DataSubmissionParams ): Promise { try { - // Step 1: Find and validate files + // Enhanced parameter validation + this.validateSubmissionParams(params); + + // Step 1: Find and validate files with enhanced feedback const validFiles = await this.findValidFiles(params.dataDirectory); - // Step 2: Submit files + // Step 2: Submit files with enhanced error handling const submission = await this.submitFiles({ categoryId: params.categoryId, organization: params.organization, files: validFiles, }); - // Step 3: Wait for validation + // Step 3: Wait for validation with enhanced progress tracking const finalStatus = await this.waitForValidation( submission.submissionId, params.maxRetries || 10, @@ -62,6 +67,8 @@ export class LyricSubmissionService extends BaseService { if (finalStatus === "VALID") { await this.commitSubmission(params.categoryId, submission.submissionId); + Logger.success`Lyric data submission workflow completed successfully`; + return { submissionId: submission.submissionId, status: "COMMITTED", @@ -70,38 +77,156 @@ export class LyricSubmissionService extends BaseService { }; } - throw new ConductorError( - `Submission validation failed with status: ${finalStatus}`, - ErrorCodes.VALIDATION_FAILED, - { submissionId: submission.submissionId, status: finalStatus } + throw ErrorFactory.validation( + `Lyric submission validation failed with status: ${finalStatus}`, + { submissionId: submission.submissionId, status: finalStatus }, + [ + "Check data format and content validity", + "Verify files match the registered dictionary schema", + "Review validation errors in Lyric service logs", + `Check submission status at ${this.config.url}/submission/${submission.submissionId}`, + "Ensure all required fields are present and properly formatted", + ] ); } catch (error) { this.handleServiceError(error, "data submission workflow"); } } + /** + * Enhanced parameter validation + */ + private validateSubmissionParams(params: DataSubmissionParams): void { + this.validateRequired( + params, + ["categoryId", "organization", "dataDirectory"], + "data submission" + ); + + // Enhanced category ID validation + if (!/^\d+$/.test(params.categoryId)) { + throw ErrorFactory.validation( + `Invalid category ID format: ${params.categoryId}`, + { categoryId: params.categoryId }, + [ + "Category ID must be a positive integer", + "Examples: '1', '2', '3'", + "Check with Lyric administrator for valid category IDs", + "Ensure the category exists in Lyric", + ] + ); + } + + // Enhanced organization validation + if ( + typeof params.organization !== "string" || + params.organization.trim() === "" + ) { + throw ErrorFactory.validation( + "Invalid organization for Lyric data submission", + { organization: params.organization, type: typeof params.organization }, + [ + "Organization must be a non-empty string", + "Use your institution's identifier", + "Examples: 'OICR', 'NIH', 'University-Toronto'", + "Match the organization used in dictionary registration", + ] + ); + } + + // Enhanced retry parameters validation + if (params.maxRetries !== undefined) { + if ( + !Number.isInteger(params.maxRetries) || + params.maxRetries < 1 || + params.maxRetries > 50 + ) { + throw ErrorFactory.validation( + `Invalid maxRetries value: ${params.maxRetries}`, + { maxRetries: params.maxRetries }, + [ + "Max retries must be an integer between 1 and 50", + "Recommended: 5-15 for most use cases", + "Higher values for unstable connections", + "Default is 10 if not specified", + ] + ); + } + } + + if (params.retryDelay !== undefined) { + if ( + !Number.isInteger(params.retryDelay) || + params.retryDelay < 1000 || + params.retryDelay > 300000 + ) { + throw ErrorFactory.validation( + `Invalid retryDelay value: ${params.retryDelay}ms`, + { retryDelay: params.retryDelay }, + [ + "Retry delay must be between 1000ms (1s) and 300000ms (5min)", + "Recommended: 10000-30000ms for most use cases", + "Longer delays for heavily loaded services", + "Default is 20000ms if not specified", + ] + ); + } + } + + Logger.debug`Submission parameters validated successfully`; + } + /** * Find valid CSV files that match the schema requirements + * Enhanced with detailed validation and feedback */ private async findValidFiles(dataDirectory: string): Promise { if (!fs.existsSync(dataDirectory)) { - throw new ConductorError( - `Data directory not found: ${dataDirectory}`, - ErrorCodes.FILE_NOT_FOUND + throw ErrorFactory.file( + `Data directory not found: ${path.basename(dataDirectory)}`, + dataDirectory, + [ + "Check that the directory path is correct", + "Ensure the directory exists", + "Verify permissions allow access", + `Current directory: ${process.cwd()}`, + "Create the directory if it doesn't exist", + ] ); } if (!fs.statSync(dataDirectory).isDirectory()) { - throw new ConductorError( - `Path is not a directory: ${dataDirectory}`, - ErrorCodes.INVALID_ARGS + throw ErrorFactory.file( + `Path is not a directory: ${path.basename(dataDirectory)}`, + dataDirectory, + [ + "Provide a directory path, not a file path", + "Check the path points to a directory", + "Ensure the path is correct", + ] + ); + } + + // Enhanced file discovery and validation + let allFiles: string[]; + try { + allFiles = fs.readdirSync(dataDirectory); + } catch (error) { + throw ErrorFactory.file( + `Cannot read data directory: ${path.basename(dataDirectory)}`, + dataDirectory, + [ + "Check directory permissions", + "Ensure directory is accessible", + "Verify directory is not corrupted", + "Try running with elevated permissions", + ] ); } - // Find all CSV files - const allFiles = fs - .readdirSync(dataDirectory) - .filter((file) => file.endsWith(".csv")) + // Filter and validate CSV files + const csvFiles = allFiles + .filter((file) => file.toLowerCase().endsWith(".csv")) .map((file) => path.join(dataDirectory, file)) .filter((filePath) => { try { @@ -112,120 +237,249 @@ export class LyricSubmissionService extends BaseService { } }); - if (allFiles.length === 0) { - throw new ConductorError( - `No valid CSV files found in ${dataDirectory}`, - ErrorCodes.FILE_NOT_FOUND, - { directory: dataDirectory } + if (csvFiles.length === 0) { + const nonCsvFiles = allFiles.filter( + (file) => !file.toLowerCase().endsWith(".csv") + ); + + throw ErrorFactory.file( + `No valid CSV files found in data directory: ${path.basename( + dataDirectory + )}`, + dataDirectory, + [ + "Ensure the directory contains CSV files", + "Check file extensions are .csv (case sensitive)", + "Verify files are not in subdirectories", + `Directory contains: ${allFiles.slice(0, 5).join(", ")}${ + allFiles.length > 5 ? "..." : "" + }`, + nonCsvFiles.length > 0 + ? `Non-CSV files found: ${nonCsvFiles.slice(0, 3).join(", ")}` + : "", + "Only CSV files are supported for Lyric upload", + ].filter(Boolean) ); } - Logger.info(`Found ${allFiles.length} valid CSV files`); - allFiles.forEach((file) => Logger.info(` - ${path.basename(file)}`)); + // Validate individual CSV files + const fileValidationErrors: string[] = []; + csvFiles.forEach((filePath) => { + try { + const stats = fs.statSync(filePath); + const fileName = path.basename(filePath); + + if (stats.size === 0) { + fileValidationErrors.push(`${fileName} (empty file)`); + } else if (stats.size > 100 * 1024 * 1024) { + // 100MB + Logger.warn`Large CSV file detected: ${fileName} (${( + stats.size / + 1024 / + 1024 + ).toFixed(1)}MB)`; + Logger.tipString("Large files may take longer to process"); + } + } catch (error) { + fileValidationErrors.push(`${path.basename(filePath)} (cannot read)`); + } + }); + + if (fileValidationErrors.length > 0) { + throw ErrorFactory.file( + `Invalid CSV files found in data directory`, + dataDirectory, + [ + `Fix these files: ${fileValidationErrors.join(", ")}`, + "Ensure all CSV files contain data", + "Check file permissions", + "Remove or fix empty or corrupted files", + ] + ); + } - return allFiles; + Logger.success`Found ${csvFiles.length} valid CSV file(s) for upload`; + csvFiles.forEach((file) => Logger.debug` - ${path.basename(file)}`); + + return csvFiles; } /** - * Submit files to Lyric + * Submit files to Lyric with enhanced error handling */ private async submitFiles(params: { categoryId: string; organization: string; files: string[]; }): Promise<{ submissionId: string }> { - Logger.info(`Submitting ${params.files.length} files to Lyric...`); + Logger.info`Submitting ${params.files.length} files to Lyric...`; - // Create FormData for file upload - const formData = new FormData(); + try { + // Create FormData for file upload + const formData = new FormData(); - // Add files - for (const filePath of params.files) { - const fileData = fs.readFileSync(filePath); - const blob = new Blob([fileData], { type: "text/csv" }); - formData.append("files", blob, path.basename(filePath)); - } + // Add files with enhanced validation + for (const filePath of params.files) { + const fileName = path.basename(filePath); - // Add organization - formData.append("organization", params.organization); + try { + const fileData = fs.readFileSync(filePath); + const blob = new Blob([fileData], { type: "text/csv" }); + formData.append("files", blob, fileName); + } catch (error) { + throw ErrorFactory.file( + `Cannot read file for upload: ${fileName}`, + filePath, + [ + "Check file permissions and accessibility", + "Ensure file is not locked by another process", + "Verify file is not corrupted", + "Try copying the file to a different location", + ] + ); + } + } - const response = await this.http.post<{ submissionId?: string }>( - `/submission/category/${params.categoryId}/data`, - formData, - { - headers: { - "Content-Type": "multipart/form-data", - }, + // Add organization + formData.append("organization", params.organization); + + const response = await this.http.post<{ submissionId?: string }>( + `/submission/category/${encodeURIComponent(params.categoryId)}/data`, + formData, + { + headers: { + "Content-Type": "multipart/form-data", + }, + } + ); + + const submissionId = response.data?.submissionId; + if (!submissionId) { + throw ErrorFactory.connection( + "Could not extract submission ID from Lyric response", + "Lyric", + this.config.url, + [ + "Lyric service may not be properly configured", + "Check Lyric API response format", + "Verify submission was successful", + "Review Lyric service logs for errors", + ] + ); } - ); - const submissionId = response.data?.submissionId; - if (!submissionId) { - throw new ConductorError( - "Could not extract submission ID from response", - ErrorCodes.CONNECTION_ERROR, - { response: response.data } + Logger.success`Submission created with ID: ${submissionId}`; + return { submissionId: submissionId.toString() }; + } catch (error) { + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + // Enhanced error handling for upload failures + const errorMessage = + error instanceof Error ? error.message : String(error); + + if (errorMessage.includes("413") || errorMessage.includes("too large")) { + throw ErrorFactory.validation( + "File upload too large for Lyric service", + { fileCount: params.files.length }, + [ + "Files may be too large for upload", + "Try uploading smaller batches of files", + "Check individual file sizes", + "Contact administrator about upload limits", + ] + ); + } else if ( + errorMessage.includes("400") || + errorMessage.includes("validation") + ) { + throw ErrorFactory.validation( + "File upload validation failed", + { categoryId: params.categoryId, organization: params.organization }, + [ + "Check category ID is valid and exists", + "Verify organization name is correct", + "Ensure files match the expected format", + "Review Lyric submission requirements", + ] + ); + } + + throw ErrorFactory.connection( + `File upload to Lyric failed: ${errorMessage}`, + "Lyric", + this.config.url, + [ + "Check Lyric service connectivity", + "Verify upload endpoint is accessible", + "Ensure proper network connectivity", + "Review file sizes and formats", + ] ); } - - Logger.success(`Submission created with ID: ${submissionId}`); - return { submissionId: submissionId.toString() }; } /** - * Wait for submission validation with progress updates + * Wait for submission validation with enhanced progress tracking */ private async waitForValidation( submissionId: string, maxRetries: number, retryDelay: number ): Promise { - Logger.info(`Waiting for submission ${submissionId} validation...`); - Logger.info( - "This may take a few minutes depending on file size and complexity." - ); + Logger.info`Waiting for submission ${submissionId} validation...`; + Logger.info`This may take a few minutes depending on file size and complexity.`; for (let attempt = 1; attempt <= maxRetries; attempt++) { try { const response = await this.http.get<{ status?: string }>( - `/submission/${submissionId}` + `/submission/${encodeURIComponent(submissionId)}` ); const status = response.data?.status; if (!status) { - throw new ConductorError( - "Could not extract status from response", - ErrorCodes.CONNECTION_ERROR, - { response: response.data } + throw ErrorFactory.connection( + "Could not extract status from Lyric validation response", + "Lyric", + this.config.url, + [ + "Lyric service may not be responding correctly", + "Check Lyric API response format", + "Verify submission ID is correct", + "Review Lyric service logs", + ] ); } - Logger.info(`Validation check ${attempt}/${maxRetries}: ${status}`); + Logger.info`Validation check ${attempt}/${maxRetries}: ${status}`; if (status === "VALID") { - Logger.success("Submission validation passed"); + Logger.success`Submission validation passed`; return status; } else if (status === "INVALID") { - throw new ConductorError( - "Submission validation failed", - ErrorCodes.VALIDATION_FAILED, - { - submissionId, - status, - suggestion: `Check validation details at ${this.config.url}/submission/${submissionId}`, - } + throw ErrorFactory.validation( + "Lyric submission validation failed", + { submissionId, status }, + [ + "Data validation failed - check CSV file format and content", + "Verify data matches the registered dictionary schema", + "Check for required fields and data types", + `Review validation details at ${this.config.url}/submission/${submissionId}`, + "Ensure all data follows the expected format", + ] ); } // Still processing, wait before next check if (attempt < maxRetries) { - Logger.info( - `Waiting ${retryDelay / 1000} seconds before next check...` - ); + Logger.info`Waiting ${ + retryDelay / 1000 + } seconds before next check...`; await this.delay(retryDelay); } } catch (error) { - if (error instanceof ConductorError) { + if (error instanceof Error && error.name === "ConductorError") { throw error; } @@ -233,43 +487,165 @@ export class LyricSubmissionService extends BaseService { this.handleServiceError(error, "validation status check"); } - Logger.warn( - `Status check failed, retrying... (${attempt}/${maxRetries})` - ); + Logger.warn`Status check failed, retrying... (${attempt}/${maxRetries})`; await this.delay(retryDelay); } } - throw new ConductorError( + throw ErrorFactory.connection( `Validation timed out after ${maxRetries} attempts`, - ErrorCodes.CONNECTION_ERROR, - { - submissionId, - attempts: maxRetries, - suggestion: `Check status manually at ${this.config.url}/submission/${submissionId}`, - } + "Lyric", + this.config.url, + [ + `Validation took longer than expected (${ + (maxRetries * retryDelay) / 1000 + }s)`, + "Large datasets may require more time to process", + `Check status manually at ${this.config.url}/submission/${submissionId}`, + "Consider increasing maxRetries or retryDelay for large datasets", + "Contact administrator if validation consistently times out", + ] ); } /** - * Commit a validated submission + * Commit a validated submission with enhanced error handling */ private async commitSubmission( categoryId: string, submissionId: string ): Promise { - Logger.info(`Committing submission: ${submissionId}`); + Logger.info`Committing submission: ${submissionId}`; - // Send empty object instead of null - await this.http.post( - `/submission/category/${categoryId}/commit/${submissionId}`, - {} - ); + try { + // Send empty object instead of null + await this.http.post( + `/submission/category/${encodeURIComponent( + categoryId + )}/commit/${encodeURIComponent(submissionId)}`, + {} + ); - Logger.success("Submission committed successfully"); + Logger.success`Submission committed successfully`; + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); + + if (errorMessage.includes("404")) { + throw ErrorFactory.validation( + `Submission not found for commit: ${submissionId}`, + { submissionId, categoryId }, + [ + "Submission may have already been committed", + "Verify submission ID is correct", + "Check that submission passed validation", + "Ensure submission belongs to the specified category", + ] + ); + } else if ( + errorMessage.includes("400") || + errorMessage.includes("conflict") + ) { + throw ErrorFactory.validation( + `Cannot commit submission: ${submissionId}`, + { submissionId, categoryId }, + [ + "Submission may not be in a committable state", + "Ensure submission has passed validation", + "Check that submission hasn't already been committed", + "Verify all validation steps completed successfully", + ] + ); + } + + throw ErrorFactory.connection( + `Failed to commit submission: ${errorMessage}`, + "Lyric", + this.config.url, + [ + "Check Lyric service connectivity", + "Verify commit endpoint is accessible", + "Ensure submission is in valid state", + "Review Lyric service logs for details", + ] + ); + } } + /** + * Utility delay function + */ private delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } + + /** + * Enhanced service error handling with Lyric data submission context + */ + protected handleServiceError(error: unknown, operation: string): never { + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + // Enhanced error handling with Lyric data submission specific guidance + const errorMessage = error instanceof Error ? error.message : String(error); + + let suggestions = [ + `Check that Lyric service is running and accessible`, + `Verify service URL: ${this.config.url}`, + "Check network connectivity and firewall settings", + "Confirm Lyric service configuration", + "Review Lyric service logs for additional details", + ]; + + // Add operation-specific suggestions + if (operation === "data submission workflow") { + suggestions = [ + "Verify CSV files format and content", + "Check data matches the registered dictionary schema", + "Ensure category ID exists and is accessible", + "Verify organization has proper permissions", + ...suggestions, + ]; + } else if (operation === "validation status check") { + suggestions = [ + "Lyric validation service may be overloaded", + "Check if submission ID is correct", + "Large datasets may require more time to validate", + "Consider increasing retry delay for better reliability", + ...suggestions, + ]; + } + + // Handle specific error patterns + if ( + errorMessage.includes("timeout") || + errorMessage.includes("ETIMEDOUT") + ) { + suggestions.unshift("Lyric service response timed out"); + suggestions.unshift("Large file uploads may take longer than expected"); + suggestions.unshift("Consider uploading smaller batches of files"); + } else if ( + errorMessage.includes("413") || + errorMessage.includes("too large") + ) { + suggestions.unshift("File upload size exceeds Lyric service limits"); + suggestions.unshift("Split large files into smaller chunks"); + suggestions.unshift("Contact administrator about upload size limits"); + } else if ( + errorMessage.includes("validation") || + errorMessage.includes("INVALID") + ) { + suggestions.unshift("Data validation failed against dictionary schema"); + suggestions.unshift("Check CSV format and required fields"); + suggestions.unshift("Verify data types match schema expectations"); + } + + throw ErrorFactory.connection( + `Lyric ${operation} failed: ${errorMessage}`, + "Lyric", + this.config.url, + suggestions + ); + } } diff --git a/apps/conductor/src/services/song-score/index.ts b/apps/conductor/src/services/song-score/index.ts index f5663247..2eb00ce7 100644 --- a/apps/conductor/src/services/song-score/index.ts +++ b/apps/conductor/src/services/song-score/index.ts @@ -1,5 +1,4 @@ // src/services/song/index.ts -export { SongService } from "./songService"; export { SongScoreService } from "./songScoreService"; export * from "./types"; // Note: validateSongSchema is only used internally by SongService diff --git a/apps/conductor/src/services/song-score/scoreService.ts b/apps/conductor/src/services/song-score/scoreService.ts index d7b388df..b86e10bc 100644 --- a/apps/conductor/src/services/song-score/scoreService.ts +++ b/apps/conductor/src/services/song-score/scoreService.ts @@ -1,8 +1,8 @@ -// src/services/score/ScoreService.ts +// src/services/score/ScoreService.ts - Enhanced with ErrorFactory patterns import { BaseService } from "../base/baseService"; import { ServiceConfig } from "../base/types"; import { Logger } from "../../utils/logger"; -import { ConductorError, ErrorCodes } from "../../utils/errors"; +import { ErrorFactory } from "../../utils/errors"; import { ScoreManifestUploadParams, ScoreManifestUploadResponse, @@ -33,33 +33,25 @@ export class ScoreService extends BaseService { /** * Complete manifest upload workflow: generate manifest -> upload files + * Enhanced with ErrorFactory patterns */ async uploadWithManifest( params: ScoreManifestUploadParams ): Promise { try { - this.validateRequired(params, ["analysisId", "dataDir", "manifestFile"]); + // Enhanced parameter validation + this.validateManifestUploadParams(params); - // Validate data directory exists - if (!fs.existsSync(params.dataDir)) { - throw new ConductorError( - `Data directory not found: ${params.dataDir}`, - ErrorCodes.FILE_NOT_FOUND - ); - } + // Enhanced data directory validation + this.validateDataDirectory(params.dataDir); - // Create output directory if needed + // Create output directory if needed with enhanced error handling const manifestDir = path.dirname(params.manifestFile); - if (!fs.existsSync(manifestDir)) { - fs.mkdirSync(manifestDir, { recursive: true }); - Logger.info(`Created directory: ${manifestDir}`); - } + this.ensureDirectoryExists(manifestDir); - Logger.info( - `Starting Score manifest upload for analysis: ${params.analysisId}` - ); + Logger.info`Starting Score manifest upload for analysis: ${params.analysisId}`; - // Step 1: Generate manifest + // Step 1: Generate manifest with enhanced error handling await this.generateManifest({ analysisId: params.analysisId, manifestFile: params.manifestFile, @@ -68,21 +60,16 @@ export class ScoreService extends BaseService { authToken: params.authToken, }); - // Step 2: Upload files using manifest + // Step 2: Upload files using manifest with enhanced error handling await this.uploadFiles({ manifestFile: params.manifestFile, authToken: params.authToken, }); - // Read manifest content for response - let manifestContent = ""; - try { - manifestContent = fs.readFileSync(params.manifestFile, "utf8"); - } catch (error) { - Logger.warn(`Could not read manifest file: ${error}`); - } + // Enhanced manifest content reading + const manifestContent = this.readManifestContent(params.manifestFile); - Logger.success(`Successfully uploaded files with Score`); + Logger.success`Successfully uploaded files with Score`; return { success: true, @@ -96,42 +83,170 @@ export class ScoreService extends BaseService { } } + /** + * Enhanced parameter validation + */ + private validateManifestUploadParams( + params: ScoreManifestUploadParams + ): void { + this.validateRequired( + params, + ["analysisId", "dataDir", "manifestFile"], + "manifest upload" + ); + + // Enhanced analysis ID validation + if (!/^[a-zA-Z0-9_-]+$/.test(params.analysisId)) { + throw ErrorFactory.validation( + `Invalid analysis ID format: ${params.analysisId}`, + { analysisId: params.analysisId }, + [ + "Analysis ID must contain only letters, numbers, hyphens, and underscores", + "Use the exact ID returned from SONG analysis submission", + "Check for typos or extra characters", + "Ensure the analysis exists in SONG", + ] + ); + } + + Logger.debug`Manifest upload parameters validated`; + } + + /** + * Enhanced data directory validation + */ + private validateDataDirectory(dataDir: string): void { + if (!fs.existsSync(dataDir)) { + throw ErrorFactory.file( + `Data directory not found: ${path.basename(dataDir)}`, + dataDir, + [ + "Check that the directory path is correct", + "Ensure the directory exists", + "Verify permissions allow access", + `Current directory: ${process.cwd()}`, + "Create the directory if it doesn't exist", + ] + ); + } + + const stats = fs.statSync(dataDir); + if (!stats.isDirectory()) { + throw ErrorFactory.file( + `Path is not a directory: ${path.basename(dataDir)}`, + dataDir, + [ + "Provide a directory path, not a file path", + "Check the path points to a directory", + "Ensure the path is correct", + ] + ); + } + + // Check for data files + const files = fs.readdirSync(dataDir); + const dataFiles = files.filter((file) => { + const ext = path.extname(file).toLowerCase(); + return [ + ".vcf", + ".bam", + ".fastq", + ".fq", + ".sam", + ".cram", + ".bed", + ".txt", + ".tsv", + ".csv", + ].includes(ext); + }); + + if (dataFiles.length === 0) { + Logger.warn`No common data file types found in directory: ${path.basename( + dataDir + )}`; + Logger.tipString( + "Ensure data files match those referenced in your analysis file" + ); + } else { + Logger.debug`Found ${dataFiles.length} data file(s) in directory`; + } + + Logger.debug`Data directory validated: ${dataDir}`; + } + + /** + * Enhanced directory creation with error handling + */ + private ensureDirectoryExists(dirPath: string): void { + if (!fs.existsSync(dirPath)) { + try { + fs.mkdirSync(dirPath, { recursive: true }); + Logger.info`Created directory: ${dirPath}`; + } catch (error) { + throw ErrorFactory.file( + `Cannot create manifest directory: ${path.basename(dirPath)}`, + dirPath, + [ + "Check directory permissions", + "Ensure parent directories exist", + "Verify disk space is available", + "Use a different output directory", + ] + ); + } + } + } + /** * Generate manifest file using SONG client or direct API approach + * Enhanced with detailed error handling */ private async generateManifest( params: ManifestGenerationParams ): Promise { - Logger.info(`Generating manifest for analysis: ${params.analysisId}`); + Logger.info`Generating manifest for analysis: ${params.analysisId}`; - // Check if Docker song-client is available - const useSongDocker = await this.checkIfDockerContainerRunning( - "song-client" - ); + try { + // Check if Docker song-client is available + const useSongDocker = await this.checkIfDockerContainerRunning( + "song-client" + ); - if (useSongDocker) { - Logger.info(`Using Song Docker client to generate manifest`); - await this.generateManifestWithSongClient(params); - } else { - Logger.info(`Using direct API approach to generate manifest`); - await this.generateManifestDirect(params); - } + if (useSongDocker) { + Logger.info`Using SONG Docker client to generate manifest`; + await this.generateManifestWithSongClient(params); + } else { + Logger.info`Using direct API approach to generate manifest`; + await this.generateManifestDirect(params); + } + + // Enhanced manifest verification + this.verifyManifestGenerated(params.manifestFile); - // Verify manifest was created - if (!fs.existsSync(params.manifestFile)) { - throw new ConductorError( - `Manifest file not generated at expected path: ${params.manifestFile}`, - ErrorCodes.FILE_NOT_FOUND + Logger.success`Successfully generated manifest at ${params.manifestFile}`; + } catch (error) { + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + throw ErrorFactory.connection( + `Failed to generate manifest for analysis ${params.analysisId}`, + "Score/SONG", + undefined, + [ + "Check that SONG service is accessible", + "Verify analysis exists and contains file references", + "Ensure Docker is available for SONG client operations", + "Check network connectivity to SONG service", + "Review analysis file structure and content", + ] ); } - - const manifestContent = fs.readFileSync(params.manifestFile, "utf8"); - Logger.debug(`Generated manifest content:\n${manifestContent}`); - Logger.success(`Successfully generated manifest at ${params.manifestFile}`); } /** - * Generate manifest using SONG Docker client + * Generate manifest using SONG Docker client with enhanced error handling */ private async generateManifestWithSongClient( params: ManifestGenerationParams @@ -148,97 +263,172 @@ export class ScoreService extends BaseService { `sh -c "sing manifest -a ${params.analysisId} -f ${containerManifestPath} -d ${containerDataDir}"`, ].join(" "); - Logger.debug(`Executing: ${command}`); + Logger.debug`Executing SONG client command: ${command}`; - // Execute the command + // Execute the command with enhanced error handling const { stdout, stderr } = await execPromise(command, { timeout: this.SONG_EXEC_TIMEOUT, }); - // Log output - if (stdout) Logger.debug(`SONG manifest stdout: ${stdout}`); - if (stderr) Logger.warn(`SONG manifest stderr: ${stderr}`); - } catch (error: any) { - Logger.error(`SONG client manifest generation failed`); + // Enhanced output logging + if (stdout) Logger.debug`SONG manifest stdout: ${stdout}`; + if (stderr) Logger.warn`SONG manifest stderr: ${stderr}`; - if (error.stdout) Logger.debug(`Stdout: ${error.stdout}`); - if (error.stderr) Logger.debug(`Stderr: ${error.stderr}`); + Logger.debug`SONG client manifest generation completed`; + } catch (error: any) { + Logger.error`SONG client manifest generation failed`; + + if (error.stdout) Logger.debug`Stdout: ${error.stdout}`; + if (error.stderr) Logger.debug`Stderr: ${error.stderr}`; + + if (error.code === "ETIMEDOUT") { + throw ErrorFactory.connection( + "SONG client manifest generation timed out", + "SONG", + undefined, + [ + `Operation timed out after ${ + this.SONG_EXEC_TIMEOUT / 1000 + } seconds`, + "Large analyses may require more time to process", + "Check SONG service performance and connectivity", + "Consider using direct API approach if Docker issues persist", + ] + ); + } - throw new ConductorError( - `Failed to generate manifest: ${error.message || "Unknown error"}`, - ErrorCodes.CONNECTION_ERROR, - error + throw ErrorFactory.connection( + `SONG client manifest generation failed: ${ + error.message || "Unknown error" + }`, + "SONG", + undefined, + [ + "Check that song-client Docker container is running", + "Verify Docker is properly configured", + "Ensure SONG service is accessible from Docker container", + "Check analysis ID exists and has file references", + "Review Docker container logs for additional details", + ] ); } } /** - * Generate manifest directly using SONG API + * Generate manifest directly using SONG API with enhanced error handling */ private async generateManifestDirect( params: ManifestGenerationParams ): Promise { try { - // We need to find the analysis in SONG first - // This requires importing SongService - for now we'll make direct HTTP calls + Logger.info`Fetching analysis ${params.analysisId} details from SONG API`; - Logger.info( - `Fetching analysis ${params.analysisId} details from SONG API` - ); - - // Create a temporary HTTP client for SONG + // Enhanced SONG service configuration const songConfig = { url: params.songUrl || "http://localhost:8080", timeout: 10000, authToken: params.authToken, }; - // This is a simplified approach - in practice, you'd want to use SongService - // But to avoid circular dependencies, we'll make direct HTTP calls here - const axios = require("axios"); - const baseUrl = songConfig.url.endsWith("/") - ? songConfig.url.slice(0, -1) - : songConfig.url; + // Validate SONG URL + try { + new URL(songConfig.url); + } catch (error) { + throw ErrorFactory.config( + `Invalid SONG URL for manifest generation: ${songConfig.url}`, + "songUrl", + [ + "Use a valid URL format: http://localhost:8080", + "Include protocol (http:// or https://)", + "Check for typos in the URL", + "Verify SONG service is accessible", + ] + ); + } + + const analysis = await this.fetchAnalysisFromSong( + songConfig, + params.analysisId + ); + + // Enhanced manifest content generation + const manifestContent = this.generateManifestContent( + analysis, + params.analysisId + ); + + // Write the manifest to file with enhanced error handling + this.writeManifestFile(params.manifestFile, manifestContent); + Logger.info`Successfully generated manifest using direct API approach`; + } catch (error: any) { + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + throw ErrorFactory.connection( + `Direct manifest generation failed: ${ + error.message || "Unknown error" + }`, + "SONG", + params.songUrl, + [ + "Check SONG service connectivity and availability", + "Verify analysis ID exists and contains files", + "Ensure proper authentication credentials", + "Check network connectivity to SONG service", + ] + ); + } + } + + /** + * Enhanced analysis fetching from SONG + */ + private async fetchAnalysisFromSong( + songConfig: any, + analysisId: string + ): Promise { + const axios = require("axios"); + const baseUrl = songConfig.url.endsWith("/") + ? songConfig.url.slice(0, -1) + : songConfig.url; + + try { // Get all studies to find which one contains our analysis const studiesResponse = await axios.get(`${baseUrl}/studies/all`, { headers: { Accept: "application/json", - Authorization: params.authToken?.startsWith("Bearer ") - ? params.authToken - : `Bearer ${params.authToken}`, + Authorization: songConfig.authToken?.startsWith("Bearer ") + ? songConfig.authToken + : `Bearer ${songConfig.authToken}`, }, + timeout: songConfig.timeout, }); const studies = Array.isArray(studiesResponse.data) ? studiesResponse.data : [studiesResponse.data]; - let analysis = null; - let studyId = null; - // Search for the analysis across all studies for (const study of studies) { try { const analysisResponse = await axios.get( - `${baseUrl}/studies/${study}/analysis/${params.analysisId}`, + `${baseUrl}/studies/${study}/analysis/${analysisId}`, { headers: { Accept: "application/json", - Authorization: params.authToken?.startsWith("Bearer ") - ? params.authToken - : `Bearer ${params.authToken}`, + Authorization: songConfig.authToken?.startsWith("Bearer ") + ? songConfig.authToken + : `Bearer ${songConfig.authToken}`, }, + timeout: songConfig.timeout, } ); if (analysisResponse.status === 200) { - analysis = analysisResponse.data; - studyId = study; - Logger.info( - `Found analysis ${params.analysisId} in study ${studyId}` - ); - break; + Logger.info`Found analysis ${analysisId} in study ${study}`; + return analysisResponse.data; } } catch (error) { // Continue to next study if analysis not found @@ -246,88 +436,191 @@ export class ScoreService extends BaseService { } } - if (!analysis || !studyId) { - throw new ConductorError( - `Analysis ${params.analysisId} not found in any study`, - ErrorCodes.CONNECTION_ERROR - ); + throw ErrorFactory.validation( + `Analysis not found in any study: ${analysisId}`, + { analysisId, studiesChecked: studies.length }, + [ + "Verify analysis ID is correct", + "Check that analysis was successfully submitted to SONG", + "Ensure analysis exists and is in UNPUBLISHED state", + "Confirm you have access to the study containing the analysis", + ] + ); + } catch (error) { + if (error instanceof Error && error.name === "ConductorError") { + throw error; } - // Extract file information from the analysis - const files = analysis.files || []; - - if (files.length === 0) { - throw new ConductorError( - `No files found in analysis ${params.analysisId}`, - ErrorCodes.VALIDATION_FAILED + const errorMessage = + error instanceof Error ? error.message : String(error); + + if (errorMessage.includes("401") || errorMessage.includes("403")) { + throw ErrorFactory.connection( + "SONG API authentication failed", + "SONG", + songConfig.url, + [ + "Check authentication token is valid", + "Verify API credentials and permissions", + "Ensure token hasn't expired", + "Confirm access to SONG studies and analyses", + ] ); } - Logger.info( - `Found ${files.length} files in analysis ${params.analysisId}` + throw ErrorFactory.connection( + `Failed to fetch analysis from SONG: ${errorMessage}`, + "SONG", + songConfig.url, + [ + "Check SONG service connectivity", + "Verify analysis ID exists", + "Ensure proper authentication", + "Check network connectivity", + ] + ); + } + } + + /** + * Enhanced manifest content generation + */ + private generateManifestContent(analysis: any, analysisId: string): string { + // Extract file information from the analysis + const files = analysis.files || []; + + if (files.length === 0) { + throw ErrorFactory.validation( + `No files found in analysis ${analysisId}`, + { analysisId, analysis: Object.keys(analysis) }, + [ + "Analysis must contain file references", + "Check that files were properly added to the analysis", + "Verify analysis structure includes 'files' array", + "Ensure files have required objectId, fileName, and fileMd5sum", + ] ); + } - // Generate manifest content - // First line: analysis ID followed by two tabs - let manifestContent = `${params.analysisId}\t\t\n`; + Logger.info`Found ${files.length} files in analysis ${analysisId}`; - for (const file of files) { - const objectId = file.objectId; - const fileName = file.fileName; - const fileMd5sum = file.fileMd5sum; + // Generate manifest content + // First line: analysis ID followed by two tabs + let manifestContent = `${analysisId}\t\t\n`; - if (!objectId || !fileName || !fileMd5sum) { - Logger.warn( - `Missing required fields for file: ${JSON.stringify(file)}` - ); - continue; - } + for (const file of files) { + const objectId = file.objectId; + const fileName = file.fileName; + const fileMd5sum = file.fileMd5sum; - // Use container path for Docker compatibility - const containerFilePath = `/data/fileData/${fileName}`; - manifestContent += `${objectId}\t${containerFilePath}\t${fileMd5sum}\n`; + if (!objectId || !fileName || !fileMd5sum) { + Logger.warn`Missing required fields for file: ${JSON.stringify(file)}`; + continue; } - // Write the manifest to file - Logger.debug( - `Writing manifest content to ${params.manifestFile}:\n${manifestContent}` + // Use container path for Docker compatibility + const containerFilePath = `/data/fileData/${fileName}`; + manifestContent += `${objectId}\t${containerFilePath}\t${fileMd5sum}\n`; + } + + return manifestContent; + } + + /** + * Enhanced manifest file writing + */ + private writeManifestFile(manifestFile: string, content: string): void { + try { + Logger.debug`Writing manifest content to ${manifestFile}`; + fs.writeFileSync(manifestFile, content); + } catch (error) { + throw ErrorFactory.file( + `Failed to write manifest file: ${path.basename(manifestFile)}`, + manifestFile, + [ + "Check directory permissions", + "Ensure sufficient disk space", + "Verify file path is accessible", + "Try using a different output directory", + ] ); - fs.writeFileSync(params.manifestFile, manifestContent); + } + } - Logger.info(`Successfully generated manifest at ${params.manifestFile}`); - } catch (error: any) { - Logger.error(`Direct manifest generation failed`); + /** + * Enhanced manifest verification + */ + private verifyManifestGenerated(manifestFile: string): void { + if (!fs.existsSync(manifestFile)) { + throw ErrorFactory.file( + `Manifest file not generated at expected path: ${path.basename( + manifestFile + )}`, + manifestFile, + [ + "Check manifest generation process completed successfully", + "Verify output directory is writable", + "Ensure no errors occurred during generation", + "Try running manifest generation again", + ] + ); + } - throw new ConductorError( - `Failed to generate manifest: ${error.message || "Unknown error"}`, - ErrorCodes.CONNECTION_ERROR, - error + const stats = fs.statSync(manifestFile); + if (stats.size === 0) { + throw ErrorFactory.file( + `Generated manifest file is empty: ${path.basename(manifestFile)}`, + manifestFile, + [ + "Check that analysis contains file references", + "Verify manifest generation process worked correctly", + "Ensure analysis has valid file entries", + "Review SONG analysis structure", + ] ); } + + const manifestContent = fs.readFileSync(manifestFile, "utf8"); + Logger.debug`Generated manifest content:\n${manifestContent}`; } /** - * Upload files using score-client + * Enhanced manifest content reading + */ + private readManifestContent(manifestFile: string): string { + try { + return fs.readFileSync(manifestFile, "utf8"); + } catch (error) { + Logger.warn`Could not read manifest file for response: ${error}`; + return ""; + } + } + + /** + * Upload files using score-client with enhanced error handling */ private async uploadFiles(params: { manifestFile: string; authToken?: string; }): Promise { - Logger.info(`Uploading files with Score client`); + Logger.info`Uploading files with Score client`; - // Check if Docker score-client is available + // Enhanced Docker availability check const useScoreDocker = await this.checkIfDockerContainerRunning( "score-client" ); if (!useScoreDocker) { - throw new ConductorError( - "Score client Docker container not available. Please ensure score-client container is running.", - ErrorCodes.INVALID_ARGS, - { - suggestion: - "Install Docker and ensure score-client container is running", - } + throw ErrorFactory.validation( + "Score client Docker container not available", + { container: "score-client" }, + [ + "Install Docker and ensure it's running", + "Pull the score-client Docker image", + "Start the score-client container", + "Verify Docker container is properly configured", + "Check Docker daemon is accessible", + ] ); } @@ -342,69 +635,147 @@ export class ScoreService extends BaseService { `sh -c "score-client upload --manifest ${containerManifestPath}"`, ].join(" "); - Logger.debug(`Executing: ${command}`); + Logger.debug`Executing Score client command: ${command}`; - // Execute the command + // Execute the command with enhanced error handling const { stdout, stderr } = await execPromise(command, { timeout: this.SCORE_EXEC_TIMEOUT, }); - // Log output - if (stdout) Logger.debug(`SCORE upload stdout: ${stdout}`); - if (stderr) Logger.warn(`SCORE upload stderr: ${stderr}`); + // Enhanced output logging + if (stdout) Logger.debug`SCORE upload stdout: ${stdout}`; + if (stderr) Logger.warn`SCORE upload stderr: ${stderr}`; - Logger.success(`Files uploaded successfully with Score client`); + Logger.success`Files uploaded successfully with Score client`; } catch (error: any) { - Logger.error(`Score client upload failed`); - - if (error.stdout) Logger.debug(`Stdout: ${error.stdout}`); - if (error.stderr) Logger.debug(`Stderr: ${error.stderr}`); + Logger.error`Score client upload failed`; + + if (error.stdout) Logger.debug`Stdout: ${error.stdout}`; + if (error.stderr) Logger.debug`Stderr: ${error.stderr}`; + + if (error.code === "ETIMEDOUT") { + throw ErrorFactory.connection( + "Score client upload timed out", + "Score", + this.config.url, + [ + `Upload timed out after ${this.SCORE_EXEC_TIMEOUT / 1000} seconds`, + "Large files may require more time to upload", + "Check Score service performance and connectivity", + "Consider uploading smaller batches of files", + "Verify network stability and bandwidth", + ] + ); + } - throw new ConductorError( - `Failed to upload with Score: ${error.message || "Unknown error"}`, - ErrorCodes.CONNECTION_ERROR, - error + throw ErrorFactory.connection( + `Score client upload failed: ${error.message || "Unknown error"}`, + "Score", + this.config.url, + [ + "Check that score-client Docker container is running", + "Verify Docker is properly configured", + "Ensure Score service is accessible from Docker container", + "Check manifest file format and content", + "Verify all referenced files exist in data directory", + "Review Docker container logs for additional details", + ] ); } } /** - * Check if a Docker container is running + * Check if a Docker container is running with enhanced error handling */ private async checkIfDockerContainerRunning( containerName: string ): Promise { try { const command = `docker ps -q -f name=${containerName}`; - Logger.debug(`Checking if container is running: ${command}`); + Logger.debug`Checking if container is running: ${command}`; - const { stdout } = await execPromise(command); - return stdout.trim().length > 0; + const { stdout } = await execPromise(command, { timeout: 5000 }); + const isRunning = stdout.trim().length > 0; + + Logger.debug`Container ${containerName} ${ + isRunning ? "is" : "is not" + } running`; + + return isRunning; } catch (error) { - Logger.debug( - `Docker container check failed: ${ - error instanceof Error ? error.message : String(error) - }` - ); + Logger.debug`Docker container check failed: ${ + error instanceof Error ? error.message : String(error) + }`; return false; } } /** - * Validate Docker availability + * Validate Docker availability with enhanced error handling */ async validateDockerAvailability(): Promise { try { - await execPromise("docker --version"); + await execPromise("docker --version", { timeout: 5000 }); + Logger.debug`Docker is available`; } catch (error) { - throw new ConductorError( + throw ErrorFactory.validation( "Docker is required for Score operations but is not available", - ErrorCodes.INVALID_ARGS, - { - suggestion: - "Install Docker and ensure it's running before using Score services", - } + { error: error instanceof Error ? error.message : String(error) }, + [ + "Install Docker and ensure it's running", + "Check Docker daemon is started", + "Verify Docker is accessible from command line", + "Ensure proper Docker permissions", + "Test with: docker --version", + ] + ); + } + } + + /** + * Enhanced service error handling with Score-specific context + */ + protected handleServiceError(error: unknown, operation: string): never { + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + // Enhanced error handling with Score-specific guidance + const errorMessage = error instanceof Error ? error.message : String(error); + + let suggestions = [ + `Check that Score service is running and accessible`, + `Verify service URL: ${this.config.url}`, + "Check network connectivity and firewall settings", + "Confirm Score service configuration", + "Review Score service logs for additional details", + ]; + + // Add operation-specific suggestions + if (operation === "manifest upload workflow") { + suggestions = [ + "Verify analysis exists and contains file references", + "Check that data directory contains referenced files", + "Ensure Docker is available for Score operations", + "Verify SONG service connectivity for manifest generation", + ...suggestions, + ]; + } + + // Handle Docker-specific errors + if (errorMessage.includes("Docker") || errorMessage.includes("container")) { + suggestions.unshift("Docker is required for Score operations"); + suggestions.unshift("Ensure Docker is installed and running"); + suggestions.unshift( + "Check that score-client and song-client containers are available" ); } + + throw ErrorFactory.connection( + `Score ${operation} failed: ${errorMessage}`, + "Score", + this.config.url, + suggestions + ); } } diff --git a/apps/conductor/src/services/song-score/songSchemaValidator.ts b/apps/conductor/src/services/song-score/songSchemaValidator.ts index 2e665e37..9d5dddd9 100644 --- a/apps/conductor/src/services/song-score/songSchemaValidator.ts +++ b/apps/conductor/src/services/song-score/songSchemaValidator.ts @@ -2,8 +2,9 @@ * SONG Schema Validator * * Validates schema files against SONG-specific requirements based on SONG documentation. + * Enhanced with ErrorFactory patterns for consistent error handling. */ -import { ConductorError, ErrorCodes } from "../../utils/errors"; +import { ErrorFactory } from "../../utils/errors"; /** * Required fields for SONG analysis schemas @@ -25,37 +26,52 @@ export function validateSongSchema(schema: any): { // Check if schema is an object if (!schema || typeof schema !== "object") { - throw new ConductorError( + throw ErrorFactory.validation( "Invalid schema format: Schema must be a JSON object", - ErrorCodes.INVALID_FILE + { schema, type: typeof schema }, + [ + "Ensure the schema file contains a valid JSON object", + "Check JSON syntax for errors (missing commas, brackets, quotes)", + "Verify the file is properly formatted", + "Use a JSON validator to check structure", + ] ); } // Check for required fields for (const field of REQUIRED_FIELDS) { if (typeof schema[field] === "undefined" || schema[field] === null) { - throw new ConductorError( + throw ErrorFactory.validation( `Invalid schema: Missing required field '${field}'`, - ErrorCodes.INVALID_FILE, { - details: `The SONG server requires '${field}' to be present`, - suggestion: `Add a '${field}' field to your schema`, - } + schema: Object.keys(schema), + missingField: field, + requiredFields: REQUIRED_FIELDS, + }, + [ + `Add a '${field}' field to your schema`, + `The SONG server requires '${field}' to be present`, + `Required fields for SONG schemas: ${REQUIRED_FIELDS.join(", ")}`, + "Check SONG documentation for schema format requirements", + ] ); } } // Validate the "schema" field is an object if (typeof schema.schema !== "object") { - throw new ConductorError( + throw ErrorFactory.validation( "Invalid schema: The 'schema' field must be an object", - ErrorCodes.INVALID_FILE, { - details: - "The 'schema' field defines the JSON schema for this analysis type", - suggestion: - "Make sure 'schema' is an object containing at least 'type' and 'properties'", - } + schemaFieldType: typeof schema.schema, + schemaField: schema.schema, + }, + [ + "The 'schema' field defines the JSON schema for this analysis type", + "Make sure 'schema' is an object containing at least 'type' and 'properties'", + "Use JSON Schema format for the 'schema' field", + 'Example: { "type": "object", "properties": { ... } }', + ] ); } @@ -166,8 +182,71 @@ export function validateSongSchema(schema: any): { warnings.push("The 'properties' field should be an object"); } + // Additional SONG-specific validations + validateSongSpecificRequirements(schema, warnings); + return { isValid: true, warnings, }; } + +/** + * Validates SONG-specific schema requirements + */ +function validateSongSpecificRequirements( + schema: any, + warnings: string[] +): void { + // Check for SONG-specific analysis types + if (schema.name) { + const commonAnalysisTypes = [ + "sequencingRead", + "variantCall", + "sequencingExperiment", + "alignment", + "transcriptome", + "copy_number_variation", + "structural_variation", + ]; + + if (!commonAnalysisTypes.includes(schema.name)) { + warnings.push( + `Analysis type '${schema.name}' is not a common SONG analysis type. ` + + `Common types include: ${commonAnalysisTypes + .slice(0, 3) + .join(", ")}, etc.` + ); + } + } + + // Check for required properties in certain analysis types + if (schema.name === "sequencingRead" && schema.schema.properties) { + const requiredSequencingFields = ["fileName", "fileMd5sum", "fileSize"]; + const schemaProperties = Object.keys(schema.schema.properties); + + const missingFields = requiredSequencingFields.filter( + (field) => !schemaProperties.includes(field) + ); + + if (missingFields.length > 0) { + warnings.push( + `Sequencing read schemas typically require: ${missingFields.join(", ")}` + ); + } + } + + // Check for version field + if (!schema.version) { + warnings.push( + "Consider adding a 'version' field to track schema evolution" + ); + } + + // Check for description field + if (!schema.description) { + warnings.push( + "Consider adding a 'description' field to document the schema purpose" + ); + } +} diff --git a/apps/conductor/src/services/song-score/songScoreService.ts b/apps/conductor/src/services/song-score/songScoreService.ts index 61b01a11..f2d279af 100644 --- a/apps/conductor/src/services/song-score/songScoreService.ts +++ b/apps/conductor/src/services/song-score/songScoreService.ts @@ -1,7 +1,8 @@ -// src/services/song/SongScoreService.ts +// src/services/song-score/songScoreService.ts - Enhanced with ErrorFactory patterns import { BaseService } from "../base/baseService"; import { ServiceConfig } from "../base/types"; import { Logger } from "../../utils/logger"; +import { ErrorFactory } from "../../utils/errors"; import { SongService } from "./songService"; import { ScoreService } from "./scoreService"; import { SongScoreWorkflowParams, SongScoreWorkflowResponse } from "./types"; @@ -56,17 +57,12 @@ export class SongScoreService extends BaseService { let analysisId = ""; try { - this.validateRequired(params, [ - "analysisContent", - "studyId", - "dataDir", - "manifestFile", - ]); + this.validateWorkflowParams(params); - Logger.info(`Starting SONG/Score workflow for study: ${params.studyId}`); + Logger.info`Starting SONG/Score workflow for study: ${params.studyId}`; // Step 1: Submit analysis to SONG - Logger.info(`Step 1: Submitting analysis to SONG`); + Logger.info`Step 1: Submitting analysis to SONG`; const analysisResponse = await this.songService.submitAnalysis({ analysisContent: params.analysisContent, studyId: params.studyId, @@ -75,10 +71,10 @@ export class SongScoreService extends BaseService { analysisId = analysisResponse.analysisId; steps.submitted = true; - Logger.success(`Analysis submitted with ID: ${analysisId}`); + Logger.success`Analysis submitted with ID: ${analysisId}`; // Step 2: Generate manifest and upload files to Score - Logger.info(`Step 2: Generating manifest and uploading files to Score`); + Logger.info`Step 2: Generating manifest and uploading files to Score`; await this.scoreService.uploadWithManifest({ analysisId, dataDir: params.dataDir, @@ -88,10 +84,10 @@ export class SongScoreService extends BaseService { }); steps.uploaded = true; - Logger.success(`Files uploaded successfully to Score`); + Logger.success`Files uploaded successfully to Score`; // Step 3: Publish analysis in SONG - Logger.info(`Step 3: Publishing analysis in SONG`); + Logger.info`Step 3: Publishing analysis in SONG`; await this.songService.publishAnalysis({ analysisId, studyId: params.studyId, @@ -99,9 +95,9 @@ export class SongScoreService extends BaseService { }); steps.published = true; - Logger.success(`Analysis published successfully`); + Logger.success`Analysis published successfully`; - Logger.success(`SONG/Score workflow completed successfully`); + Logger.success`SONG/Score workflow completed successfully`; return { success: true, @@ -113,36 +109,140 @@ export class SongScoreService extends BaseService { message: "Workflow completed successfully", }; } catch (error) { - // Determine the status based on which steps completed - let status: "COMPLETED" | "PARTIAL" | "FAILED" = "FAILED"; + return this.handleWorkflowError(error, analysisId, params, steps); + } + } - if (steps.submitted && steps.uploaded && !steps.published) { - status = "PARTIAL"; - } else if (steps.submitted && !steps.uploaded) { - status = "PARTIAL"; - } + /** + * Validate workflow parameters + */ + private validateWorkflowParams(params: SongScoreWorkflowParams): void { + this.validateRequired( + params, + ["analysisContent", "studyId", "dataDir", "manifestFile"], + "SONG/Score workflow" + ); + + // Validate study ID format + if (!/^[a-zA-Z0-9_-]+$/.test(params.studyId)) { + throw ErrorFactory.validation( + `Invalid study ID format: ${params.studyId}`, + { studyId: params.studyId }, + [ + "Study ID must contain only letters, numbers, hyphens, and underscores", + "Use the same study ID used when creating the study", + "Check for typos or extra characters", + "Ensure the study exists in SONG", + ] + ); + } + + // Validate analysis content is valid JSON + try { + JSON.parse(params.analysisContent); + } catch (error) { + throw ErrorFactory.validation( + "Invalid JSON format in analysis content", + { error: error instanceof Error ? error.message : String(error) }, + [ + "Check JSON syntax for errors (missing commas, brackets, quotes)", + "Validate JSON structure using a JSON validator", + "Ensure file encoding is UTF-8", + "Try viewing the analysis file in a JSON editor", + ] + ); + } - const errorMessage = - error instanceof Error ? error.message : String(error); + // Validate data directory exists + const fs = require("fs"); + if (!fs.existsSync(params.dataDir)) { + throw ErrorFactory.file( + `Data directory not found: ${params.dataDir}`, + params.dataDir, + [ + "Check that the directory path is correct", + "Ensure the directory exists", + "Verify permissions allow access", + `Current directory: ${process.cwd()}`, + "Create the directory if it doesn't exist", + ] + ); + } - Logger.error(`SONG/Score workflow failed: ${errorMessage}`); + Logger.debug`Workflow parameters validated`; + } - // Log which steps completed - Logger.info(`Workflow status:`); - Logger.info(` - Analysis submitted: ${steps.submitted ? "✓" : "✗"}`); - Logger.info(` - Files uploaded: ${steps.uploaded ? "✓" : "✗"}`); - Logger.info(` - Analysis published: ${steps.published ? "✓" : "✗"}`); + /** + * Handle workflow errors with detailed context + */ + private handleWorkflowError( + error: unknown, + analysisId: string, + params: SongScoreWorkflowParams, + steps: { submitted: boolean; uploaded: boolean; published: boolean } + ): SongScoreWorkflowResponse { + // Determine the status based on which steps completed + let status: "COMPLETED" | "PARTIAL" | "FAILED" = "FAILED"; - return { - success: false, - analysisId, - studyId: params.studyId, - manifestFile: params.manifestFile, - status, - steps, - message: `Workflow failed: ${errorMessage}`, - }; + if (steps.submitted && steps.uploaded && !steps.published) { + status = "PARTIAL"; + } else if (steps.submitted && !steps.uploaded) { + status = "PARTIAL"; } + + const errorMessage = error instanceof Error ? error.message : String(error); + + Logger.error`SONG/Score workflow failed: ${errorMessage}`; + + // Log which steps completed + Logger.info`Workflow status:`; + Logger.info` - Analysis submitted: ${steps.submitted ? "✓" : "✗"}`; + Logger.info` - Files uploaded: ${steps.uploaded ? "✓" : "✗"}`; + Logger.info` - Analysis published: ${steps.published ? "✓" : "✗"}`; + + // Provide specific guidance based on failure point + let suggestions: string[] = []; + + if (!steps.submitted) { + suggestions = [ + "Analysis submission failed - check SONG service connectivity", + "Verify analysis file format and content", + "Ensure study exists in SONG", + "Check SONG service authentication", + ]; + } else if (!steps.uploaded) { + suggestions = [ + "File upload failed - check Score service and Docker requirements", + "Verify data files exist in specified directory", + "Ensure Docker containers are running (song-client, score-client)", + "Check Score service connectivity", + ]; + } else if (!steps.published) { + suggestions = [ + "Analysis publication failed - files uploaded but publication incomplete", + "Run songPublishAnalysis command manually to complete", + "Check analysis validation status in SONG", + "Verify all required files were uploaded successfully", + ]; + } else { + suggestions = [ + "Unexpected workflow failure", + "Check all service connectivities", + "Review service logs for detailed errors", + "Contact support if issue persists", + ]; + } + + return { + success: false, + analysisId, + studyId: params.studyId, + manifestFile: params.manifestFile, + status, + steps, + message: `Workflow failed: ${errorMessage}`, + suggestions, + }; } /** @@ -164,13 +264,20 @@ export class SongScoreService extends BaseService { const scoreHealthy = scoreHealth.status === "fulfilled" && scoreHealth.value.healthy; + if (!songHealthy) { + Logger.warn`SONG service health check failed`; + } + if (!scoreHealthy) { + Logger.warn`Score service health check failed`; + } + return { song: songHealthy, score: scoreHealthy, overall: songHealthy && scoreHealthy, }; } catch (error) { - Logger.warn(`Error checking services health: ${error}`); + Logger.warn`Error checking services health: ${error}`; return { song: false, score: false, @@ -180,13 +287,72 @@ export class SongScoreService extends BaseService { } /** - * Validate Docker availability for Score operations + * Validate Docker requirements for Score operations */ async validateDockerRequirements(): Promise { try { await this.scoreService.validateDockerAvailability(); + Logger.debug`Docker requirements validated for Score operations`; } catch (error) { - this.handleServiceError(error, "Docker validation"); + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + throw ErrorFactory.validation( + "Docker validation failed for SONG/Score workflow", + { error: error instanceof Error ? error.message : String(error) }, + [ + "Docker is required for Score file upload operations", + "Install Docker and ensure it's running", + "Check Docker daemon is accessible", + "Verify Docker permissions are correct", + "Test with: docker --version", + ] + ); } } + + /** + * Enhanced service error handling for combined workflow + */ + protected handleServiceError(error: unknown, operation: string): never { + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + const errorMessage = error instanceof Error ? error.message : String(error); + + let suggestions = [ + "Check that both SONG and Score services are running", + "Verify all service URLs and connectivity", + "Ensure proper authentication for all services", + "Check Docker is available for Score operations", + "Review service logs for detailed errors", + ]; + + // Add operation-specific suggestions + if (operation === "workflow execution") { + suggestions = [ + "Workflow involves multiple services - check each component", + "SONG: verify analysis format and study existence", + "Score: ensure Docker containers and file accessibility", + "Check network connectivity to all services", + ...suggestions, + ]; + } else if (operation === "Docker validation") { + suggestions = [ + "Docker is required for Score file upload operations", + "Install Docker and ensure it's running", + "Check Docker daemon is accessible", + "Verify score-client and song-client containers are available", + ]; + } + + throw ErrorFactory.connection( + `SONG/Score ${operation} failed: ${errorMessage}`, + "SONG/Score", + this.config.url, + suggestions + ); + } } diff --git a/apps/conductor/src/services/song-score/songService.ts b/apps/conductor/src/services/song-score/songService.ts index 5c8b5068..616eb59d 100644 --- a/apps/conductor/src/services/song-score/songService.ts +++ b/apps/conductor/src/services/song-score/songService.ts @@ -1,8 +1,8 @@ -// src/services/song/SongService.ts +// src/services/song-score/songService.ts - Enhanced with ErrorFactory patterns import { BaseService } from "../base/baseService"; import { ServiceConfig } from "../base/types"; import { Logger } from "../../utils/logger"; -import { ConductorError, ErrorCodes } from "../../utils/errors"; +import { ErrorFactory } from "../../utils/errors"; import { SongSchemaUploadParams, SongSchemaUploadResponse, @@ -14,7 +14,6 @@ import { SongPublishResponse, } from "./types"; import { validateSongSchema } from "./songSchemaValidator"; -import * as fs from "fs"; export class SongService extends BaseService { constructor(config: ServiceConfig) { @@ -43,44 +42,49 @@ export class SongService extends BaseService { try { schemaData = JSON.parse(params.schemaContent); } catch (error) { - throw new ConductorError( - `Invalid schema format: ${ - error instanceof Error ? error.message : String(error) - }`, - ErrorCodes.INVALID_FILE, - error + throw ErrorFactory.validation( + "Invalid JSON format in SONG schema", + { error: error instanceof Error ? error.message : String(error) }, + [ + "Check JSON syntax for errors (missing commas, brackets, quotes)", + "Validate JSON structure using a JSON validator", + "Ensure file encoding is UTF-8", + "Try viewing the file in a JSON editor", + error instanceof Error ? `JSON error: ${error.message}` : "", + ].filter(Boolean) ); } // Validate against SONG-specific requirements const { isValid, warnings } = validateSongSchema(schemaData); - // Log any warnings if (warnings.length > 0) { - Logger.warn("Schema validation warnings:"); - warnings.forEach((warning) => { - Logger.warn(` - ${warning}`); - }); + Logger.warn`Schema validation warnings:`; + warnings.forEach((warning) => Logger.warn` - ${warning}`); } - Logger.info(`Uploading schema: ${schemaData.name}`); + Logger.info`Uploading schema: ${schemaData.name}`; - // Upload to SONG schemas endpoint const response = await this.http.post( "/schemas", schemaData ); - // Check for errors in response if (response.data?.error) { - throw new ConductorError( + throw ErrorFactory.connection( `SONG API error: ${response.data.error}`, - ErrorCodes.CONNECTION_ERROR + "SONG", + this.config.url, + [ + "Check schema format and structure", + "Verify SONG service is properly configured", + "Review schema for required fields and valid values", + "Check SONG service logs for additional details", + ] ); } - Logger.success(`Schema "${schemaData.name}" uploaded successfully`); - + Logger.success`Schema "${schemaData.name}" uploaded successfully`; return response.data; } catch (error) { this.handleServiceError(error, "schema upload"); @@ -94,12 +98,12 @@ export class SongService extends BaseService { try { this.validateRequired(params, ["studyId", "name", "organization"]); - Logger.info(`Creating study: ${params.studyId}`); + Logger.info`Creating study: ${params.studyId}`; // Check if study already exists const studyExists = await this.checkStudyExists(params.studyId); if (studyExists && !params.force) { - Logger.warn(`Study ID ${params.studyId} already exists`); + Logger.warn`Study ID ${params.studyId} already exists`; return { studyId: params.studyId, name: params.name, @@ -109,7 +113,6 @@ export class SongService extends BaseService { }; } - // Prepare study payload const studyPayload = { description: params.description || "string", info: {}, @@ -118,13 +121,12 @@ export class SongService extends BaseService { studyId: params.studyId, }; - // Create study const response = await this.http.post( `/studies/${params.studyId}/`, studyPayload ); - Logger.success(`Study created successfully`); + Logger.success`Study created successfully`; return { ...response.data, @@ -134,7 +136,6 @@ export class SongService extends BaseService { status: "CREATED", }; } catch (error) { - // Handle 409 conflict for existing studies if (this.isConflictError(error)) { return { studyId: params.studyId, @@ -163,20 +164,30 @@ export class SongService extends BaseService { try { analysisData = JSON.parse(params.analysisContent); } catch (error) { - throw new ConductorError( - `Invalid analysis format: ${ - error instanceof Error ? error.message : String(error) - }`, - ErrorCodes.INVALID_FILE, - error + throw ErrorFactory.validation( + "Invalid JSON format in analysis file", + { error: error instanceof Error ? error.message : String(error) }, + [ + "Check JSON syntax for errors (missing commas, brackets, quotes)", + "Validate JSON structure using a JSON validator", + "Ensure file encoding is UTF-8", + "Try viewing the file in a JSON editor", + error instanceof Error ? `JSON error: ${error.message}` : "", + ].filter(Boolean) ); } // Basic validation of analysis structure if (!analysisData.analysisType || !analysisData.analysisType.name) { - throw new ConductorError( - "Invalid analysis format: Missing required field 'analysisType.name'", - ErrorCodes.INVALID_FILE + throw ErrorFactory.validation( + "Missing required field 'analysisType.name' in analysis file", + { analysisData: Object.keys(analysisData) }, + [ + "Analysis must have 'analysisType' object with 'name' field", + "Check SONG analysis schema requirements", + "Ensure analysis type is properly defined", + "Review SONG documentation for analysis structure", + ] ); } @@ -185,16 +196,21 @@ export class SongService extends BaseService { !Array.isArray(analysisData.files) || analysisData.files.length === 0 ) { - throw new ConductorError( - "Invalid analysis format: 'files' must be a non-empty array", - ErrorCodes.INVALID_FILE + throw ErrorFactory.validation( + "Missing or empty 'files' array in analysis file", + { filesCount: analysisData.files?.length || 0 }, + [ + "Analysis must include 'files' array with at least one file", + "Each file should have objectId, fileName, and fileMd5sum", + "Ensure files are properly defined in the analysis", + "Check that file references match actual data files", + ] ); } - Logger.info(`Submitting analysis to study: ${params.studyId}`); - Logger.info(`Analysis type: ${analysisData.analysisType.name}`); + Logger.info`Submitting analysis to study: ${params.studyId}`; + Logger.info`Analysis type: ${analysisData.analysisType.name}`; - // Submit analysis const submitUrl = `/submit/${params.studyId}?allowDuplicates=${ params.allowDuplicates || false }`; @@ -202,9 +218,7 @@ export class SongService extends BaseService { submitUrl, params.analysisContent, { - headers: { - "Content-Type": "application/json", - }, + headers: { "Content-Type": "application/json" }, } ); @@ -220,13 +234,20 @@ export class SongService extends BaseService { } if (!analysisId) { - throw new ConductorError( + throw ErrorFactory.connection( "No analysis ID returned from SONG API", - ErrorCodes.CONNECTION_ERROR + "SONG", + this.config.url, + [ + "SONG service may not be responding correctly", + "Check SONG API response format", + "Verify analysis submission was successful", + "Review SONG service logs for errors", + ] ); } - Logger.success(`Analysis submitted successfully with ID: ${analysisId}`); + Logger.success`Analysis submitted successfully with ID: ${analysisId}`; return { analysisId, @@ -248,23 +269,19 @@ export class SongService extends BaseService { try { this.validateRequired(params, ["analysisId", "studyId"]); - Logger.info(`Publishing analysis: ${params.analysisId}`); + Logger.info`Publishing analysis: ${params.analysisId}`; - // Construct the publish endpoint URL const publishUrl = `/studies/${params.studyId}/analysis/publish/${params.analysisId}`; - - // Set up query parameters const queryParams: Record = {}; if (params.ignoreUndefinedMd5) { queryParams.ignoreUndefinedMd5 = true; } - // Make the PUT request to publish const response = await this.http.put(publishUrl, null, { params: queryParams, }); - Logger.success(`Analysis published successfully`); + Logger.success`Analysis published successfully`; return { analysisId: params.analysisId, @@ -326,14 +343,13 @@ export class SongService extends BaseService { return { studyId, analysis }; } } catch (error) { - // Continue to next study if analysis not found continue; } } return null; } catch (error) { - Logger.warn(`Could not find analysis ${analysisId}: ${error}`); + Logger.warn`Could not find analysis ${analysisId}: ${error}`; return null; } } @@ -346,11 +362,9 @@ export class SongService extends BaseService { const response = await this.http.get(`/studies/${studyId}`); return response.status === 200; } catch (error: any) { - // If we get a 404, study doesn't exist if (error.response && error.response.status === 404) { return false; } - // For other errors, assume study doesn't exist return false; } } @@ -361,4 +375,83 @@ export class SongService extends BaseService { private isConflictError(error: any): boolean { return error.response && error.response.status === 409; } + + /** + * Enhanced service error handling with SONG-specific context + */ + protected handleServiceError(error: unknown, operation: string): never { + if (error instanceof Error && error.name === "ConductorError") { + throw error; + } + + const errorMessage = error instanceof Error ? error.message : String(error); + + let suggestions = [ + `Check that SONG service is running and accessible`, + `Verify service URL: ${this.config.url}`, + "Check network connectivity and firewall settings", + "Confirm SONG service configuration", + "Review SONG service logs for additional details", + ]; + + // Add operation-specific suggestions + if (operation === "schema upload") { + suggestions = [ + "Verify schema format follows SONG requirements", + "Ensure schema has required 'name' and 'schema' fields", + "Check for valid JSON structure and syntax", + ...suggestions, + ]; + } else if (operation === "study creation") { + suggestions = [ + "Check if study ID already exists", + "Verify study parameters are valid", + "Ensure organization name is correct", + ...suggestions, + ]; + } else if (operation === "analysis submission") { + suggestions = [ + "Verify analysis format and structure", + "Check that study exists in SONG", + "Ensure analysis type is properly defined", + "Verify file references in analysis", + ...suggestions, + ]; + } else if (operation === "analysis publication") { + suggestions = [ + "Check that analysis exists and is in UNPUBLISHED state", + "Verify all required files are uploaded", + "Ensure analysis passed validation checks", + ...suggestions, + ]; + } + + // Handle specific HTTP status codes + if (errorMessage.includes("404")) { + suggestions.unshift("Resource not found in SONG"); + suggestions.unshift("Check that the specified resource exists"); + } else if ( + errorMessage.includes("409") || + errorMessage.includes("conflict") + ) { + suggestions.unshift("Resource conflict - may already exist"); + suggestions.unshift("Check for duplicate IDs or existing resources"); + } else if ( + errorMessage.includes("400") || + errorMessage.includes("validation") + ) { + suggestions.unshift("Request validation failed"); + suggestions.unshift("Check request parameters and format"); + } else if (errorMessage.includes("401") || errorMessage.includes("403")) { + suggestions.unshift("Authentication or authorization failed"); + suggestions.unshift("Check API credentials and permissions"); + } + + throw ErrorFactory.connection( + `SONG ${operation} failed: ${errorMessage}`, + "SONG", + this.config.url, + suggestions + ); + } } diff --git a/apps/conductor/src/tree.txt b/apps/conductor/src/tree.txt new file mode 100644 index 00000000..e04e0530 --- /dev/null +++ b/apps/conductor/src/tree.txt @@ -0,0 +1,71 @@ +. +├── cli +│   ├── commandOptions.ts +│   ├── environment.ts +│   ├── index.ts +│   └── serviceConfigManager.ts +├── commands +│   ├── baseCommand.ts +│   ├── commandRegistry.ts +│   ├── lecternUploadCommand.ts +│   ├── lyricRegistrationCommand.ts +│   ├── lyricUploadCommand.ts +│   ├── maestroIndexCommand.ts +│   ├── songCreateStudyCommand.ts +│   ├── songPublishAnalysisCommand.ts +│   ├── songSubmitAnalysisCommand.ts +│   ├── songUploadSchemaCommand.ts +│   └── uploadCsvCommand.ts +├── main.ts +├── services +│   ├── base +│   │   ├── baseService.ts +│   │   ├── HttpService.ts +│   │   └── types.ts +│   ├── csvProcessor +│   │   ├── csvParser.ts +│   │   ├── index.ts +│   │   ├── logHandler.ts +│   │   ├── metadata.ts +│   │   └── progressBar.ts +│   ├── elasticsearch +│   │   ├── bulk.ts +│   │   ├── client.ts +│   │   └── index.ts +│   ├── lectern +│   │   ├── index.ts +│   │   ├── lecternService.ts +│   │   └── types.ts +│   ├── lyric +│   │   ├── LyricRegistrationService.ts +│   │   ├── LyricSubmissionService.ts +│   │   └── types.ts +│   ├── song-score +│   │   ├── index.ts +│   │   ├── scoreService.ts +│   │   ├── songSchemaValidator.ts +│   │   ├── songScoreService.ts +│   │   ├── songService.ts +│   │   └── types.ts +│   └── tree.txt +├── tree.txt +├── types +│   ├── cli.ts +│   ├── constants.ts +│   ├── elasticsearch.ts +│   ├── index.ts +│   └── validations.ts +├── utils +│   ├── errors.ts +│   ├── fileUtils.ts +│   └── logger.ts +└── validations + ├── constants.ts + ├── csvValidator.ts + ├── elasticsearchValidator.ts + ├── environmentValidator.ts + ├── fileValidator.ts + ├── index.ts + └── utils.ts + +13 directories, 56 files diff --git a/apps/conductor/src/types/cli.ts b/apps/conductor/src/types/cli.ts index 861100e2..98f14cbd 100644 --- a/apps/conductor/src/types/cli.ts +++ b/apps/conductor/src/types/cli.ts @@ -60,7 +60,7 @@ interface Config { // Keep this as it's used in CLI setup interface CLIOutput { - profile: Profile; + profile: string; debug?: boolean; filePaths: string[]; config: Config; diff --git a/apps/conductor/src/utils/errors.ts b/apps/conductor/src/utils/errors.ts index 7223c5a6..868ee6c0 100644 --- a/apps/conductor/src/utils/errors.ts +++ b/apps/conductor/src/utils/errors.ts @@ -1,8 +1,13 @@ -// src/utils/errors.ts - Remove unused exports +// src/utils/errors.ts - Updated to match composer 1:1 import { Logger } from "./logger"; export class ConductorError extends Error { - constructor(message: string, public code: string, public details?: any) { + constructor( + message: string, + public code: string, + public details?: any, + public suggestions?: string[] // CHANGED: Added direct suggestions property + ) { super(message); this.name = "ConductorError"; } @@ -14,28 +19,248 @@ export class ConductorError extends Error { } } +// CHANGED: Removed brackets from error codes export const ErrorCodes = { - INVALID_ARGS: "[INVALID_ARGS]", - FILE_NOT_FOUND: "[FILE_NOT_FOUND]", - INVALID_FILE: "[INVALID_FILE]", - VALIDATION_FAILED: "[VALIDATION_FAILED]", - ENV_ERROR: "[ENV_ERROR]", - PARSING_ERROR: "[PARSING_ERROR]", - FILE_ERROR: "[FILE_ERROR]", - FILE_WRITE_ERROR: "[FILE_WRITE_ERROR]", - CONNECTION_ERROR: "[CONNECTION_ERROR]", - AUTH_ERROR: "[AUTH_ERROR]", - INDEX_NOT_FOUND: "[INDEX_NOT_FOUND]", - TRANSFORM_ERROR: "[TRANSFORM_ERROR]", - CLI_ERROR: "[CLI_ERROR]", - CSV_ERROR: "[CSV_ERROR]", - ES_ERROR: "[ES_ERROR]", - UNKNOWN_ERROR: "[UNKNOWN_ERROR]", - USER_CANCELLED: "[USER_CANCELLED]", + INVALID_ARGS: "INVALID_ARGS", + FILE_NOT_FOUND: "FILE_NOT_FOUND", + INVALID_FILE: "INVALID_FILE", + VALIDATION_FAILED: "VALIDATION_FAILED", + ENV_ERROR: "ENV_ERROR", + PARSING_ERROR: "PARSING_ERROR", + FILE_ERROR: "FILE_ERROR", + FILE_WRITE_ERROR: "FILE_WRITE_ERROR", + CONNECTION_ERROR: "CONNECTION_ERROR", + AUTH_ERROR: "AUTH_ERROR", + INDEX_NOT_FOUND: "INDEX_NOT_FOUND", + TRANSFORM_ERROR: "TRANSFORM_ERROR", + CLI_ERROR: "CLI_ERROR", + CSV_ERROR: "CSV_ERROR", + ES_ERROR: "ES_ERROR", + UNKNOWN_ERROR: "UNKNOWN_ERROR", + USER_CANCELLED: "USER_CANCELLED", } as const; -// Remove the exported type - just use typeof if needed internally -// type ErrorCode = (typeof ErrorCodes)[keyof typeof ErrorCodes]; +type ErrorCodes = (typeof ErrorCodes)[keyof typeof ErrorCodes]; + +/** + * Factory for creating consistent, user-friendly errors with actionable suggestions + */ +export class ErrorFactory { + /** + * Create a file-related error with helpful suggestions + */ + static file( + message: string, + filePath?: string, + suggestions: string[] = [] + ): ConductorError { + const details: any = {}; + if (filePath) { + details.filePath = filePath; + details.currentDirectory = process.cwd(); + } + + const defaultSuggestions = [ + "Check that the file path is correct", + "Ensure the file exists and is readable", + "Verify file permissions allow access", + ]; + + if (filePath && !suggestions.length) { + defaultSuggestions.push(`Current directory: ${process.cwd()}`); + } + + // CHANGED: Pass suggestions as 4th parameter instead of embedding in details + return new ConductorError( + message, + ErrorCodes.FILE_NOT_FOUND, + details, + suggestions.length ? suggestions : defaultSuggestions + ); + } + + /** + * Create a validation error with specific field guidance + */ + static validation( + message: string, + details?: any, + suggestions: string[] = [] + ): ConductorError { + const defaultSuggestions = [ + "Check the input format and structure", + "Verify all required fields are present", + "Ensure data types match expected values", + ]; + + return new ConductorError( + message, + ErrorCodes.VALIDATION_FAILED, + details, + suggestions.length ? suggestions : defaultSuggestions + ); + } + + /** + * Create a connection error with service-specific troubleshooting + */ + static connection( + message: string, + service?: string, + url?: string, + suggestions: string[] = [] + ): ConductorError { + const details: any = { service }; + if (url) details.url = url; + + const defaultSuggestions = service + ? [ + `Check that ${service} is running and accessible`, + "Verify network connectivity and firewall settings", + "Confirm authentication credentials are correct", + ...(url ? [`Try: curl ${url}/health`] : []), + ] + : [ + "Check network connectivity", + "Verify service is running", + "Confirm connection parameters", + ]; + + // CHANGED: Pass suggestions as 4th parameter + return new ConductorError( + message, + ErrorCodes.CONNECTION_ERROR, + details, + suggestions.length ? suggestions : defaultSuggestions + ); + } + + /** + * Create a configuration error with parameter-specific guidance + */ + static config( + message: string, + parameter?: string, + suggestions: string[] = [] + ): ConductorError { + const details: any = {}; + if (parameter) details.parameter = parameter; + + const defaultSuggestions = parameter + ? [ + `Check the ${parameter} configuration value`, + "Verify environment variables are set correctly", + "Ensure configuration file syntax is valid", + ] + : [ + "Check configuration values", + "Verify environment variables", + "Ensure all required settings are provided", + ]; + + // CHANGED: Pass suggestions as 4th parameter + return new ConductorError( + message, + ErrorCodes.ENV_ERROR, + details, + suggestions.length ? suggestions : defaultSuggestions + ); + } + + /** + * Create an invalid arguments error with usage guidance + */ + static args( + message: string, + command?: string, + suggestions: string[] = [] + ): ConductorError { + const details: any = {}; + if (command) details.command = command; + + const defaultSuggestions = command + ? [ + `Check the syntax for '${command}' command`, + `Use: conductor ${command} --help for usage information`, + "Verify all required parameters are provided", + ] + : [ + "Check command syntax and parameters", + "Use: conductor --help for available commands", + "Verify all required arguments are provided", + ]; + + // CHANGED: Pass suggestions as 4th parameter + return new ConductorError( + message, + ErrorCodes.INVALID_ARGS, + details, + suggestions.length ? suggestions : defaultSuggestions + ); + } + + /** + * Create a CSV-specific error with format guidance + */ + static csv( + message: string, + filePath?: string, + row?: number, + suggestions: string[] = [] + ): ConductorError { + const details: any = {}; + if (filePath) details.filePath = filePath; + if (row !== undefined) details.row = row; + + const defaultSuggestions = [ + "Check CSV file format and structure", + "Verify headers are properly formatted", + "Ensure delimiter is correct (comma, tab, etc.)", + "Check for special characters in data", + ]; + + // CHANGED: Pass suggestions as 4th parameter + return new ConductorError( + message, + ErrorCodes.CSV_ERROR, + details, + suggestions.length ? suggestions : defaultSuggestions + ); + } + + /** + * Create an index/database error with specific guidance + */ + static index( + message: string, + indexName?: string, + suggestions: string[] = [] + ): ConductorError { + const details: any = {}; + if (indexName) details.indexName = indexName; + + const defaultSuggestions = indexName + ? [ + `Check that index '${indexName}' exists`, + "Verify Elasticsearch is running and accessible", + "Confirm index permissions and mappings", + `Try: GET /${indexName}/_mapping to check index structure`, + ] + : [ + "Check index exists and is accessible", + "Verify database connection", + "Confirm index permissions", + ]; + + // CHANGED: Pass suggestions as 4th parameter + return new ConductorError( + message, + ErrorCodes.INDEX_NOT_FOUND, + details, + suggestions.length ? suggestions : defaultSuggestions + ); + } +} function formatErrorDetails(details: any): string { if (typeof details === "string") { @@ -51,48 +276,63 @@ function formatErrorDetails(details: any): string { } } -export function handleError( - error: unknown, - showAvailableProfiles?: () => void -): never { +/** + * Centralized error handler for the application + * @param error - The error to handle + * @param showHelp - Optional callback to show help information + */ +export function handleError(error: unknown, showHelp?: () => void): never { if (error instanceof ConductorError) { - // Basic error message for all users - Logger.error(`${error.message}`); + Logger.error`[${error.code}] ${error.message}`; - // Detailed error only in debug mode + // CHANGED: Read suggestions from direct property and use Logger.section + Logger.tipString + if (error.suggestions && error.suggestions.length > 0) { + Logger.section("\nSuggestions\n"); + error.suggestions.forEach((suggestion) => { + Logger.tipString(suggestion); + }); + } + + // Show help if callback provided + if (showHelp) { + showHelp(); + } + + // Show details in debug mode if (process.argv.includes("--debug")) { if (error.details) { - Logger.debug("Error details:"); - Logger.debug(formatErrorDetails(error.details)); + const formattedDetails = formatErrorDetails(error.details); + Logger.debugString("Error details:"); + Logger.debugString(formattedDetails); } - Logger.debug("Stack trace:"); - Logger.debug(error.stack || "No stack trace available"); - } - - if (showAvailableProfiles) { - showAvailableProfiles(); + Logger.debugString("Stack trace:"); + Logger.debugString(error.stack || "No stack trace available"); } } else { - // For unexpected errors, just output the message - Logger.error( - `Unexpected error: ${ - error instanceof Error ? error.message : String(error) - }` - ); + Logger.debugString("Unexpected error occurred"); - if (process.argv.includes("--debug") && error instanceof Error) { - Logger.debug("Stack trace:"); - Logger.debug(error.stack || "No stack trace available"); + if (error instanceof Error) { + Logger.debugString(error.message); + if (process.argv.includes("--debug")) { + Logger.debugString("Stack trace:"); + Logger.debugString(error.stack || "No stack trace available"); + } + } else { + Logger.debugString(String(error)); } } process.exit(1); } +// Backward compatibility - can be removed after migration export function createValidationError( message: string, details?: any ): ConductorError { - return new ConductorError(message, ErrorCodes.VALIDATION_FAILED, details); + Logger.warnString( + "createValidationError is deprecated, use ErrorFactory.validation instead" + ); + return ErrorFactory.validation(message, details); } diff --git a/apps/conductor/src/utils/fileUtils.ts b/apps/conductor/src/utils/fileUtils.ts new file mode 100644 index 00000000..5b9084b5 --- /dev/null +++ b/apps/conductor/src/utils/fileUtils.ts @@ -0,0 +1,469 @@ +/** + * Enhanced File Utilities + * + * Centralized file and directory operations with consistent error handling. + * Eliminates code duplication while maintaining command-specific flexibility. + */ + +import * as fs from "fs"; +import * as path from "path"; +import { ErrorFactory } from "./errors"; +import { Logger } from "./logger"; + +/** + * Core file validation with consistent error handling + */ +export function validateFileAccess( + filePath: string, + fileType: string = "file" +): void { + const fileName = path.basename(filePath); + + if (!filePath || typeof filePath !== "string" || filePath.trim() === "") { + throw ErrorFactory.args(`${fileType} path not specified`, undefined, [ + `Provide a ${fileType} path`, + "Check command line arguments", + `Example: --${fileType + .toLowerCase() + .replace(/\s+/g, "-")}-file example.json`, + ]); + } + + if (!fs.existsSync(filePath)) { + throw ErrorFactory.file(`${fileType} not found: ${fileName}`, filePath, [ + "Check that the file path is correct", + "Ensure the file exists at the specified location", + "Verify file permissions allow read access", + `Current directory: ${process.cwd()}`, + "Use absolute path if relative path is not working", + ]); + } + + // Check if file is readable + try { + fs.accessSync(filePath, fs.constants.R_OK); + } catch (error) { + throw ErrorFactory.file( + `${fileType} is not readable: ${fileName}`, + filePath, + [ + "Check file permissions", + "Ensure the file is not locked by another process", + "Verify you have read access to the file", + "Try copying the file to a different location", + ] + ); + } + + // Check file size + const stats = fs.statSync(filePath); + if (stats.size === 0) { + throw ErrorFactory.file(`${fileType} is empty: ${fileName}`, filePath, [ + `Ensure the ${fileType.toLowerCase()} contains data`, + "Check if the file was properly created", + "Verify the file is not corrupted", + "Try recreating the file with valid content", + ]); + } + + Logger.debug`${fileType} validated: ${fileName}`; +} + +/** + * Core directory validation with consistent error handling + */ +export function validateDirectoryAccess( + dirPath: string, + dirType: string = "directory" +): void { + const dirName = path.basename(dirPath); + + if (!dirPath || typeof dirPath !== "string" || dirPath.trim() === "") { + throw ErrorFactory.args(`${dirType} path not specified`, undefined, [ + `Provide a ${dirType} path`, + "Check command line arguments", + `Example: --${dirType.toLowerCase().replace(/\s+/g, "-")} ./data`, + ]); + } + + if (!fs.existsSync(dirPath)) { + throw ErrorFactory.file(`${dirType} not found: ${dirName}`, dirPath, [ + "Check that the directory path is correct", + "Ensure the directory exists", + "Verify permissions allow access", + `Current directory: ${process.cwd()}`, + "Use absolute path if relative path is not working", + ]); + } + + const stats = fs.statSync(dirPath); + if (!stats.isDirectory()) { + throw ErrorFactory.file(`Path is not a directory: ${dirName}`, dirPath, [ + "Provide a directory path, not a file path", + "Check the path points to a directory", + "Ensure the path is correct", + ]); + } + + Logger.debug`${dirType} validated: ${dirName}`; +} + +/** + * Find files by extension with filtering options + */ +export function findFilesByExtension( + dirPath: string, + extensions: string[], + options: { + recursive?: boolean; + minSize?: number; + maxSize?: number; + } = {} +): string[] { + const { recursive = false, minSize = 1, maxSize } = options; + const normalizedExts = extensions.map((ext) => + ext.toLowerCase().startsWith(".") + ? ext.toLowerCase() + : `.${ext.toLowerCase()}` + ); + + let foundFiles: string[] = []; + + function scanDirectory(currentDir: string): void { + try { + const entries = fs.readdirSync(currentDir); + + for (const entry of entries) { + const fullPath = path.join(currentDir, entry); + + try { + const stats = fs.statSync(fullPath); + + if (stats.isDirectory() && recursive) { + scanDirectory(fullPath); + } else if (stats.isFile()) { + const ext = path.extname(entry).toLowerCase(); + + if (normalizedExts.includes(ext)) { + // Check size constraints + if ( + stats.size >= minSize && + (!maxSize || stats.size <= maxSize) + ) { + foundFiles.push(fullPath); + } + } + } + } catch (error) { + // Skip files we can't access + Logger.debug`Skipping inaccessible file: ${fullPath}`; + } + } + } catch (error) { + throw ErrorFactory.file( + `Cannot read directory: ${path.basename(currentDir)}`, + currentDir, + [ + "Check directory permissions", + "Ensure directory is accessible", + "Verify directory is not corrupted", + ] + ); + } + } + + scanDirectory(dirPath); + return foundFiles; +} + +/** + * Fluent API for file validation + */ +export class FileValidator { + private filePath: string; + private fileType: string; + private requiredExtensions?: string[]; + private minSizeBytes?: number; + private maxSizeBytes?: number; + private shouldExist: boolean = true; + + constructor(filePath: string, fileType: string = "file") { + this.filePath = filePath; + this.fileType = fileType; + } + + /** + * Require specific file extensions + */ + requireExtension(extensions: string | string[]): this { + this.requiredExtensions = Array.isArray(extensions) + ? extensions + : [extensions]; + return this; + } + + /** + * Require minimum file size + */ + requireMinSize(bytes: number): this { + this.minSizeBytes = bytes; + return this; + } + + /** + * Require maximum file size + */ + requireMaxSize(bytes: number): this { + this.maxSizeBytes = bytes; + return this; + } + + /** + * Allow file to not exist (for optional files) + */ + optional(): this { + this.shouldExist = false; + return this; + } + + /** + * Execute validation + */ + validate(): boolean { + // If file is optional and doesn't exist, that's fine + if (!this.shouldExist && !fs.existsSync(this.filePath)) { + return false; + } + + // Standard file access validation + validateFileAccess(this.filePath, this.fileType); + + const stats = fs.statSync(this.filePath); + const fileName = path.basename(this.filePath); + + // Extension validation + if (this.requiredExtensions) { + const fileExt = path.extname(this.filePath).toLowerCase(); + const normalizedExts = this.requiredExtensions.map((ext) => + ext.startsWith(".") ? ext.toLowerCase() : `.${ext.toLowerCase()}` + ); + + if (!normalizedExts.includes(fileExt)) { + throw ErrorFactory.validation( + `Invalid ${this.fileType} extension: ${fileName}`, + { + actualExtension: fileExt, + allowedExtensions: normalizedExts, + filePath: this.filePath, + }, + [ + `${ + this.fileType + } must have one of these extensions: ${normalizedExts.join(", ")}`, + `Found extension: ${fileExt}`, + "Check the file format and rename if necessary", + ] + ); + } + } + + // Size validation + if (this.minSizeBytes !== undefined && stats.size < this.minSizeBytes) { + throw ErrorFactory.file( + `${this.fileType} is too small: ${fileName} (${stats.size} bytes)`, + this.filePath, + [ + `Minimum size required: ${this.minSizeBytes} bytes`, + `Current size: ${stats.size} bytes`, + "Ensure the file contains sufficient data", + ] + ); + } + + if (this.maxSizeBytes !== undefined && stats.size > this.maxSizeBytes) { + throw ErrorFactory.file( + `${this.fileType} is too large: ${fileName} (${stats.size} bytes)`, + this.filePath, + [ + `Maximum size allowed: ${this.maxSizeBytes} bytes`, + `Current size: ${stats.size} bytes`, + "Consider compressing or splitting the file", + ] + ); + } + + return true; + } +} + +/** + * Fluent API for directory validation + */ +export class DirectoryValidator { + private dirPath: string; + private dirType: string; + private requiredFiles?: string[]; + private requiredExtensions?: string[]; + private minFileCount?: number; + private maxFileCount?: number; + + constructor(dirPath: string, dirType: string = "directory") { + this.dirPath = dirPath; + this.dirType = dirType; + } + + /** + * Require specific files to exist in directory + */ + requireFiles(fileNames: string[]): this { + this.requiredFiles = fileNames; + return this; + } + + /** + * Require files with specific extensions + */ + requireFilesWithExtensions(extensions: string[]): this { + this.requiredExtensions = extensions; + return this; + } + + /** + * Require minimum number of files + */ + requireMinFileCount(count: number): this { + this.minFileCount = count; + return this; + } + + /** + * Require maximum number of files + */ + requireMaxFileCount(count: number): this { + this.maxFileCount = count; + return this; + } + + /** + * Execute validation + */ + validate(): string[] { + // Standard directory access validation + validateDirectoryAccess(this.dirPath, this.dirType); + + const allFiles = fs + .readdirSync(this.dirPath) + .map((file) => path.join(this.dirPath, file)) + .filter((filePath) => { + try { + return fs.statSync(filePath).isFile(); + } catch { + return false; + } + }); + + // Required files validation + if (this.requiredFiles) { + const missingFiles = this.requiredFiles.filter((fileName) => { + const fullPath = path.join(this.dirPath, fileName); + return !fs.existsSync(fullPath); + }); + + if (missingFiles.length > 0) { + throw ErrorFactory.file( + `Required files missing in ${this.dirType}: ${path.basename( + this.dirPath + )}`, + this.dirPath, + [ + `Missing files: ${missingFiles.join(", ")}`, + "Ensure all required files are present", + "Check file names and spelling", + ] + ); + } + } + + // Extension filtering and validation + let relevantFiles = allFiles; + if (this.requiredExtensions) { + const normalizedExts = this.requiredExtensions.map((ext) => + ext.startsWith(".") ? ext.toLowerCase() : `.${ext.toLowerCase()}` + ); + + relevantFiles = allFiles.filter((filePath) => { + const ext = path.extname(filePath).toLowerCase(); + return normalizedExts.includes(ext); + }); + + if (relevantFiles.length === 0) { + throw ErrorFactory.file( + `No files with required extensions found in ${ + this.dirType + }: ${path.basename(this.dirPath)}`, + this.dirPath, + [ + `Required extensions: ${normalizedExts.join(", ")}`, + `Directory contains: ${ + allFiles.map((f) => path.extname(f)).join(", ") || "no files" + }`, + "Check file extensions and directory contents", + ] + ); + } + } + + // File count validation + if ( + this.minFileCount !== undefined && + relevantFiles.length < this.minFileCount + ) { + throw ErrorFactory.file( + `Insufficient files in ${this.dirType}: ${path.basename(this.dirPath)}`, + this.dirPath, + [ + `Minimum files required: ${this.minFileCount}`, + `Files found: ${relevantFiles.length}`, + "Add more files to the directory", + ] + ); + } + + if ( + this.maxFileCount !== undefined && + relevantFiles.length > this.maxFileCount + ) { + throw ErrorFactory.file( + `Too many files in ${this.dirType}: ${path.basename(this.dirPath)}`, + this.dirPath, + [ + `Maximum files allowed: ${this.maxFileCount}`, + `Files found: ${relevantFiles.length}`, + "Remove some files or use a different directory", + ] + ); + } + + Logger.debug`${this.dirType} validation passed: ${relevantFiles.length} files found`; + return relevantFiles; + } +} + +/** + * Validation builder for fluent API + */ +export class ValidationBuilder { + /** + * Start file validation + */ + static file(filePath: string, fileType?: string): FileValidator { + return new FileValidator(filePath, fileType); + } + + /** + * Start directory validation + */ + static directory(dirPath: string, dirType?: string): DirectoryValidator { + return new DirectoryValidator(dirPath, dirType); + } +} diff --git a/apps/conductor/src/utils/logger.ts b/apps/conductor/src/utils/logger.ts index 0a81a420..f81b695b 100644 --- a/apps/conductor/src/utils/logger.ts +++ b/apps/conductor/src/utils/logger.ts @@ -1,7 +1,8 @@ -// src/utils/logger.ts - Remove unused exports +// src/utils/logger.ts - Standardized logger with consistent template literal usage import chalk from "chalk"; -enum LogLevel { +// Make LogLevel public for use in other modules +const enum LogLevel { DEBUG = 0, INFO = 1, SUCCESS = 2, @@ -18,6 +19,52 @@ interface LoggerConfig { debug: boolean; } +// Centralized configuration for icons and colors +const LOG_CONFIG = { + icons: { + [LogLevel.DEBUG]: "🔍", + [LogLevel.INFO]: "▸", + [LogLevel.SUCCESS]: "✓", + [LogLevel.WARN]: "⚠", + [LogLevel.ERROR]: "✗", + [LogLevel.TIP]: "", + [LogLevel.GENERIC]: "", + [LogLevel.SECTION]: "", + [LogLevel.INPUT]: "❔", + } as const, + + colors: { + [LogLevel.DEBUG]: chalk.bold.gray, + [LogLevel.INFO]: chalk.bold.cyan, + [LogLevel.SUCCESS]: chalk.bold.green, + [LogLevel.WARN]: chalk.bold.yellow, + [LogLevel.ERROR]: chalk.bold.red, + [LogLevel.TIP]: chalk.bold.yellow, + [LogLevel.GENERIC]: chalk.white, + [LogLevel.SECTION]: chalk.bold.blue, + [LogLevel.INPUT]: chalk.bold.yellow, + } as const, + + labels: { + [LogLevel.DEBUG]: "Debug", + [LogLevel.INFO]: "Info", + [LogLevel.SUCCESS]: "Success", + [LogLevel.WARN]: "Warn", + [LogLevel.ERROR]: "Error", + [LogLevel.TIP]: "", + [LogLevel.GENERIC]: "", + [LogLevel.SECTION]: "", + [LogLevel.INPUT]: "User Input", + } as const, + + needsNewLine: new Set([ + LogLevel.ERROR, + LogLevel.INPUT, + LogLevel.WARN, + LogLevel.SUCCESS, + ]), +} as const; + export class Logger { private static config: LoggerConfig = { level: LogLevel.INFO, @@ -25,61 +72,20 @@ export class Logger { }; private static formatMessage(message: string, level: LogLevel): string { - const icons = { - [LogLevel.DEBUG]: "🔍", - [LogLevel.INFO]: "▸", - [LogLevel.SUCCESS]: "✓", - [LogLevel.WARN]: "⚠", - [LogLevel.ERROR]: "✗", - [LogLevel.TIP]: "\n💡", - [LogLevel.GENERIC]: "", - [LogLevel.SECTION]: "", - [LogLevel.INPUT]: "❔", - }; - - const colors: Record string> = { - [LogLevel.DEBUG]: chalk.bold.gray, - [LogLevel.INFO]: chalk.bold.cyan, - [LogLevel.SUCCESS]: chalk.bold.green, - [LogLevel.WARN]: chalk.bold.yellow, - [LogLevel.ERROR]: chalk.bold.red, - [LogLevel.TIP]: chalk.bold.yellow, - [LogLevel.GENERIC]: chalk.white, - [LogLevel.SECTION]: chalk.bold.green, - [LogLevel.INPUT]: chalk.bold.yellow, - }; - - const levelLabels = { - [LogLevel.DEBUG]: "Debug", - [LogLevel.INFO]: "Info", - [LogLevel.SUCCESS]: "Success", - [LogLevel.WARN]: "Warn", - [LogLevel.ERROR]: "Error", - [LogLevel.TIP]: "Tip", - [LogLevel.GENERIC]: "", - [LogLevel.SECTION]: "", - [LogLevel.INPUT]: "User Input", - }; - - const needsNewLine = [ - LogLevel.ERROR, - LogLevel.INPUT, - LogLevel.WARN, - LogLevel.SUCCESS, - ].includes(level); - - const prefix = needsNewLine ? "\n" : ""; + const { icons, colors, labels, needsNewLine } = LOG_CONFIG; + + const prefix = needsNewLine.has(level) ? "\n" : ""; if (level === LogLevel.GENERIC) { return colors[level](message); } if (level === LogLevel.SECTION) { - return `${prefix}\n${colors[level](`\n${icons[level]} ${message}\n`)}`; + return `${prefix}${colors[level](`${icons[level]} ${message}`)}`; } return `${prefix}${colors[level]( - `${icons[level]} ${levelLabels[level]} ` + `${icons[level]} ${labels[level]} ` )}${message}`; } @@ -90,11 +96,12 @@ export class Logger { static enableDebug(): void { this.config.debug = true; this.config.level = LogLevel.DEBUG; - console.log(chalk.gray("🔍 **Debug profile enabled**")); + console.log(chalk.gray("🔍 **Debug mode enabled**")); } /** - * Tagged template helper that automatically bolds interpolated values. + * Formats template literal strings with highlighted variables + * Standardized approach for all logging methods */ static formatVariables( strings: TemplateStringsArray, @@ -108,20 +115,34 @@ export class Logger { } /** - * Core log function that accepts either a tagged template literal or a plain string. + * Internal logging method with standardized template literal support */ private static log( level: LogLevel, - strings: TemplateStringsArray | string, + strings: TemplateStringsArray, ...values: any[] ): void { if (this.config.level > level && level !== LogLevel.DEBUG) return; if (!this.config.debug && level === LogLevel.DEBUG) return; - const message = - typeof strings === "string" - ? strings - : this.formatVariables(strings, ...values); + const message = this.formatVariables(strings, ...values); + const formattedMessage = this.formatMessage(message, level); + + if (level === LogLevel.WARN) { + console.warn(formattedMessage); + } else if (level === LogLevel.ERROR) { + console.error(formattedMessage); + } else { + console.log(formattedMessage); + } + } + + /** + * Overloaded logging method for backwards compatibility with string arguments + */ + private static logString(level: LogLevel, message: string): void { + if (this.config.level > level && level !== LogLevel.DEBUG) return; + if (!this.config.debug && level === LogLevel.DEBUG) return; const formattedMessage = this.formatMessage(message, level); @@ -134,33 +155,59 @@ export class Logger { } } - static debug(strings: TemplateStringsArray | string, ...values: any[]): void { + // Standardized template literal methods + static debug(strings: TemplateStringsArray, ...values: any[]): void { this.log(LogLevel.DEBUG, strings, ...values); } - static info(strings: TemplateStringsArray | string, ...values: any[]): void { + static info(strings: TemplateStringsArray, ...values: any[]): void { this.log(LogLevel.INFO, strings, ...values); } - static success( - strings: TemplateStringsArray | string, - ...values: any[] - ): void { + static success(strings: TemplateStringsArray, ...values: any[]): void { this.log(LogLevel.SUCCESS, strings, ...values); } - static warn(strings: TemplateStringsArray | string, ...values: any[]): void { + static warn(strings: TemplateStringsArray, ...values: any[]): void { this.log(LogLevel.WARN, strings, ...values); } - static error(strings: TemplateStringsArray | string, ...values: any[]): void { + static error(strings: TemplateStringsArray, ...values: any[]): void { this.log(LogLevel.ERROR, strings, ...values); } - static tip(strings: TemplateStringsArray | string, ...values: any[]): void { + static tip(strings: TemplateStringsArray, ...values: any[]): void { this.log(LogLevel.TIP, strings, ...values); + console.log(); + } + + // String-based methods for backwards compatibility + static debugString(message: string): void { + this.logString(LogLevel.DEBUG, message); + } + + static infoString(message: string): void { + this.logString(LogLevel.INFO, message); + } + + static successString(message: string): void { + this.logString(LogLevel.SUCCESS, message); + } + + static warnString(message: string): void { + this.logString(LogLevel.WARN, message); } + static errorString(message: string): void { + this.logString(LogLevel.ERROR, message); + } + + static tipString(message: string): void { + this.logString(LogLevel.TIP, message); + console.log(); + } + + // Special purpose methods static generic(message: string): void { console.log(this.formatMessage(message, LogLevel.GENERIC)); } @@ -174,16 +221,16 @@ export class Logger { } static header(text: string): void { - const separator = "═".repeat(text.length + 6); - console.log(`\n${chalk.bold.magenta(separator)}`); - console.log(`${chalk.bold.magenta(" " + text + " ")}`); - console.log(`${chalk.bold.magenta(separator)}\n`); + console.log(`${chalk.bold.magenta("=".repeat(text.length))}`); + console.log(`${chalk.bold.magenta(text)}`); + console.log(`${chalk.bold.magenta("=".repeat(text.length))}`); } static commandInfo(command: string, description: string): void { console.log`${chalk.bold.blue(command)}: ${description}`; } + // Enhanced default value methods with consistent template literal support static defaultValueInfo(message: string, overrideCommand: string): void { if (this.config.level <= LogLevel.INFO) { console.log(this.formatMessage(message, LogLevel.INFO)); @@ -191,13 +238,14 @@ export class Logger { } } - static commandValueTip(message: string, overrideCommand: string): void { - if (this.config.level <= LogLevel.TIP) { - console.log(this.formatMessage(message, LogLevel.TIP)); + static defaultValueWarning(message: string, overrideCommand: string): void { + if (this.config.level <= LogLevel.WARN) { + console.warn(this.formatMessage(message, LogLevel.WARN)); console.log(chalk.gray` Override with: ${overrideCommand}\n`); } } + // Debug object logging with standardized formatting static debugObject(label: string, obj: any): void { if (this.config.debug) { console.log(chalk.gray`🔍 ${label}:`); @@ -213,6 +261,7 @@ export class Logger { } } + // Timing utility with template literal support static timing(label: string, timeMs: number): void { const formattedTime = timeMs < 1000 @@ -222,17 +271,18 @@ export class Logger { console.log(chalk.gray`⏱ ${label}: ${formattedTime}`); } - static warnfileList(title: string, files: string[]): void { + // File list utilities + static fileList(title: string, files: string[]): void { if (files.length === 0) return; - Logger.warn`${title}:`; + this.warnString(`${title}:\n`); files.forEach((file) => { console.log(chalk.gray` - ${file}`); }); } - static infofileList(title: string, files: string[]): void { + static errorFileList(title: string, files: string[]): void { if (files.length === 0) return; - Logger.info`${title}:`; + this.errorString(`${title}:\n`); files.forEach((file) => { console.log(chalk.gray` - ${file}`); }); @@ -402,8 +452,8 @@ export class Logger { ); this.generic(""); - // SONG Upload commands - this.generic(chalk.bold.magenta("SONG Schema Upload Commands:")); + // Song Upload commands + this.generic(chalk.bold.magenta("Song Schema Upload Commands:")); this.generic(chalk.white("conductor songUploadSchema -s schema.json")); this.generic(chalk.gray("Options:")); this.generic( @@ -413,7 +463,7 @@ export class Logger { ); this.generic( chalk.gray( - "-u, --song-url SONG server URL (default: http://localhost:8080)" + "-u, --song-url Song server URL (default: http://localhost:8080)" ) ); this.generic( @@ -432,15 +482,15 @@ export class Logger { ); this.generic(""); - // SONG Create Study commands - this.generic(chalk.bold.magenta("SONG Create Study Commands:")); + // Song Create Study commands + this.generic(chalk.bold.magenta("Song Create Study Commands:")); this.generic( chalk.white("conductor songCreateStudy -i study-id -n study-name") ); this.generic(chalk.gray("Options:")); this.generic( chalk.gray( - "-u, --song-url SONG server URL (default: http://localhost:8080)" + "-u, --song-url Song server URL (default: http://localhost:8080)" ) ); this.generic( @@ -476,5 +526,104 @@ export class Logger { ) ); this.generic(""); + + // Song Submit Analysis commands + this.generic(chalk.bold.magenta("Song Submit Analysis Commands:")); + this.generic(chalk.white("conductor songSubmitAnalysis -a analysis.json")); + this.generic(chalk.gray("Options:")); + this.generic( + chalk.gray( + "-a, --analysis-file Analysis JSON file to submit (required)" + ) + ); + this.generic( + chalk.gray( + "-u, --song-url Song server URL (default: http://localhost:8080)" + ) + ); + this.generic( + chalk.gray( + "-s, --score-url Score server URL (default: http://localhost:8087)" + ) + ); + this.generic( + chalk.gray("-i, --study-id Study ID (default: demo)") + ); + this.generic( + chalk.gray( + "--allow-duplicates Allow duplicate analysis submissions" + ) + ); + this.generic( + chalk.gray( + "-d, --data-dir Directory containing data files (default: ./data)" + ) + ); + this.generic( + chalk.gray( + "--output-dir Directory for manifest file output (default: ./output)" + ) + ); + this.generic( + chalk.gray("-m, --manifest-file Path for manifest file") + ); + this.generic( + chalk.gray( + "-t, --auth-token Authentication token (default: 123)" + ) + ); + this.generic( + chalk.gray( + "--ignore-undefined-md5 Ignore files with undefined MD5 checksums" + ) + ); + this.generic( + chalk.gray( + "--force Force studyId from command line instead of from file" + ) + ); + this.generic(""); + this.generic( + chalk.gray( + "Example: conductor songSubmitAnalysis -a analysis.json -i my-study -d ./data" + ) + ); + this.generic(""); + + // Song Publish Analysis commands + this.generic(chalk.bold.magenta("Song Publish Analysis Commands:")); + this.generic(chalk.white("conductor songPublishAnalysis -a analysis-id")); + this.generic(chalk.gray("Options:")); + this.generic( + chalk.gray("-a, --analysis-id Analysis ID to publish (required)") + ); + this.generic( + chalk.gray("-i, --study-id Study ID (default: demo)") + ); + this.generic( + chalk.gray( + "-u, --song-url Song server URL (default: http://localhost:8080)" + ) + ); + this.generic( + chalk.gray( + "-t, --auth-token Authentication token (default: 123)" + ) + ); + this.generic( + chalk.gray( + "--ignore-undefined-md5 Ignore files with undefined MD5 checksums" + ) + ); + this.generic( + chalk.gray("-o, --output Output directory for logs") + ); + this.generic(""); + this.generic( + chalk.gray( + "Example: conductor songPublishAnalysis -a analysis-123 -i my-study" + ) + ); + this.generic(""); } } diff --git a/apps/conductor/src/validations/csvValidator.ts b/apps/conductor/src/validations/csvValidator.ts index 6b0a55f8..189ca424 100644 --- a/apps/conductor/src/validations/csvValidator.ts +++ b/apps/conductor/src/validations/csvValidator.ts @@ -1,275 +1,170 @@ import * as fs from "fs"; -import { Client } from "@elastic/elasticsearch"; -import { ConductorError, ErrorCodes } from "../utils/errors"; -import { parseCSVLine } from "../services/csvProcessor/csvParser"; -import { VALIDATION_CONSTANTS } from "./constants"; +import { ErrorFactory } from "../utils/errors"; +import { parseCSVLine } from "../services/csvProcessor/csvParser"; // Updated import import { Logger } from "../utils/logger"; /** - * Module for validating CSV files against structural and naming rules. - * Includes validation for headers, content structure, and naming conventions. - */ - -/** - * Validates CSV headers against naming conventions and rules. - * Checks: - * - Special character restrictions - * - Maximum length limits - * - Reserved word restrictions - * - GraphQL naming conventions - * - Duplicate prevention + * Validates the header structure of a CSV file. + * Reads the first line of the file and validates the headers. * - * @param headers - Array of header strings to validate - * @returns Promise resolving to true if all headers are valid - * @throws ConductorError with details if validation fails + * @param filePath - Path to the CSV file + * @param delimiter - Character used to separate values in the CSV + * @returns Promise resolving to true if headers are valid + * @throws ConductorError if headers are invalid or file can't be read */ -export async function validateCSVStructure( - headers: string[] +export async function validateCSVHeaders( + filePath: string, + delimiter: string ): Promise { - Logger.debug`Validating CSV structure with ${headers.length} headers`; - try { - // Clean and filter headers - const cleanedHeaders = headers - .map((header) => header.trim()) - .filter((header) => header !== ""); - - // Validate basic header presence - if (cleanedHeaders.length === 0) { - throw new ConductorError( - "No valid headers found in CSV file", - ErrorCodes.VALIDATION_FAILED - ); + Logger.debug`Validating CSV headers for file: ${filePath}`; + Logger.debug`Using delimiter: '${delimiter}'`; + + const fileContent = fs.readFileSync(filePath, "utf-8"); + const [headerLine] = fileContent.split("\n"); + + if (!headerLine) { + Logger.debug`CSV file is empty or has no headers`; + throw ErrorFactory.file("CSV file is empty or has no headers", filePath, [ + "Ensure the CSV file contains at least one row of headers", + "Check that the file is not corrupted", + "Verify the file encoding is UTF-8", + ]); } - if (cleanedHeaders.length !== headers.length) { - Logger.warn`Empty or whitespace-only headers detected`; - throw new ConductorError( - "Empty or whitespace-only headers detected", - ErrorCodes.VALIDATION_FAILED - ); + const headers = parseCSVLine(headerLine, delimiter, true)[0]; + if (!headers) { + Logger.debug`Failed to parse CSV headers`; + throw ErrorFactory.file("Failed to parse CSV headers", filePath, [ + "Check that the delimiter is correct", + "Ensure headers don't contain unescaped quotes", + "Verify the CSV format is valid", + ]); } - // Validate headers against all rules - const invalidHeaders = cleanedHeaders.filter((header: string) => { - const hasInvalidChars = VALIDATION_CONSTANTS.INVALID_CHARS.some((char) => - header.includes(char) - ); - const isTooLong = - Buffer.from(header).length > VALIDATION_CONSTANTS.MAX_HEADER_LENGTH; - const isReserved = VALIDATION_CONSTANTS.RESERVED_WORDS.includes( - header.toLowerCase() - ); - const isValidGraphQLName = - VALIDATION_CONSTANTS.GRAPHQL_NAME_PATTERN.test(header); - - return hasInvalidChars || isTooLong || isReserved || !isValidGraphQLName; - }); - - if (invalidHeaders.length > 0) { - Logger.error`Invalid header names detected: ${invalidHeaders.join(", ")}`; - throw new ConductorError( - "Invalid header names detected", - ErrorCodes.VALIDATION_FAILED, - { invalidHeaders } - ); - } - - // Check for duplicate headers - const headerCounts: Record = cleanedHeaders.reduce( - (acc: Record, header: string) => { - acc[header] = (acc[header] || 0) + 1; - return acc; - }, - {} - ); - - const duplicates = Object.entries(headerCounts) - .filter(([_, count]) => count > 1) - .map(([header]) => header); - - if (duplicates.length > 0) { - Logger.error`Duplicate headers found in CSV file: ${duplicates.join( - ", " - )}`; - throw new ConductorError( - "Duplicate headers found in CSV file", - ErrorCodes.VALIDATION_FAILED, - { duplicates, counts: headerCounts } - ); - } - - // Optional: Check for generic headers - const genericHeaders = cleanedHeaders.filter((header) => - ["col1", "col2", "column1", "column2", "0", "1", "2"].includes( - header.toLowerCase() - ) - ); - - if (genericHeaders.length > 0) { - Logger.warn`Generic headers detected:`; - genericHeaders.forEach((header) => { - Logger.warn`Generic header: "${header}"`; - }); - Logger.tip`Consider using more descriptive column names`; - } - - Logger.debug`CSV header structure matches valid`; - - // Log all headers in debug mode - Logger.debugObject("CSV Headers", cleanedHeaders); - - return true; + Logger.debug`Parsed headers: ${headers.join(", ")}`; + return validateCSVStructure(headers); } catch (error) { - if (error instanceof ConductorError) { + Logger.debug`Error during CSV header validation`; + Logger.debugObject("Error details", error); + + if (error instanceof Error && error.name === "ConductorError") { throw error; } - Logger.error`Error validating CSV structure: ${ - error instanceof Error ? error.message : String(error) - }`; - throw new ConductorError( - "Error validating CSV structure", - ErrorCodes.VALIDATION_FAILED, - error - ); + throw ErrorFactory.validation("Error validating CSV headers", error, [ + "Check that the file exists and is readable", + "Verify the CSV format is correct", + "Ensure proper file permissions", + ]); } } /** - * Validates CSV headers against Elasticsearch index mappings. - * Ensures CSV structure matches expected index fields. + * Validates CSV headers against naming conventions and rules. + * Enhanced with ErrorFactory patterns for consistent error handling. * - * @param client - Elasticsearch client instance - * @param headers - Array of CSV headers to validate - * @param indexName - Target Elasticsearch index name - * @returns Promise resolving to true if headers match mappings - * @throws ConductorError if validation fails + * @param headers - Array of header strings from CSV file + * @returns boolean indicating if headers are valid + * @throws ConductorError if headers fail validation */ -export async function validateHeadersMatchMappings( - client: Client, - headers: string[], - indexName: string -): Promise { - Logger.debug`Validating headers against index ${indexName} mappings`; - +export function validateCSVStructure(headers: string[]): boolean { try { - // Try to get mappings from the existing index - const { body } = await client.indices.getMapping({ - index: indexName, - }); - - // Type-safe navigation - const mappings = body[indexName]?.mappings; - if (!mappings) { - Logger.error`No mappings found for index ${indexName}`; - throw new ConductorError( - "No mappings found for the specified index", - ErrorCodes.VALIDATION_FAILED - ); + Logger.debug`Validating CSV structure with ${headers.length} headers`; + + // Enhanced validation: Check for empty headers + if (!headers || headers.length === 0) { + throw ErrorFactory.csv("CSV file has no headers", undefined, 1, [ + "Ensure the CSV file contains column headers", + "Check that the first row is not empty", + "Verify the CSV format is correct", + ]); } - // Navigate to the nested properties - const expectedFields = mappings.properties?.data?.properties - ? Object.keys(mappings.properties.data.properties) - : []; - - Logger.debug`Found ${expectedFields.length} fields in existing index mapping`; - - // Clean up headers for comparison - const cleanedHeaders = headers - .map((header: string) => header.trim()) - .filter((header: string) => header !== ""); - - if (cleanedHeaders.length === 0) { - Logger.error`No valid headers found`; - throw new ConductorError( - "No valid headers found", - ErrorCodes.VALIDATION_FAILED + // Enhanced validation: Check for duplicate headers + const duplicateHeaders = headers.filter( + (header, index) => headers.indexOf(header) !== index + ); + if (duplicateHeaders.length > 0) { + throw ErrorFactory.csv( + `Duplicate headers found: ${[...new Set(duplicateHeaders)].join(", ")}`, + undefined, + 1, + [ + "Ensure all column headers are unique", + "Remove or rename duplicate headers", + "Check for extra spaces in header names", + ] ); } - // Check for extra headers not in the mapping - const extraHeaders = cleanedHeaders.filter( - (header: string) => !expectedFields.includes(header) + // Enhanced validation: Check for empty/whitespace headers + const emptyHeaders = headers.filter( + (header, index) => !header || header.trim() === "" ); - - // Check for fields in the mapping that aren't in the headers - const missingRequiredFields = expectedFields.filter( - (field: string) => - field !== "submission_metadata" && !cleanedHeaders.includes(field) - ); - - // Log appropriate warnings - if (extraHeaders.length > 0) { - Logger.warn`Extra headers not in index mapping: ${extraHeaders.join( - ", " - )}`; - Logger.tip`These fields will be added to documents but may not be properly indexed`; + if (emptyHeaders.length > 0) { + throw ErrorFactory.csv( + `Empty headers detected (${emptyHeaders.length} of ${headers.length})`, + undefined, + 1, + [ + "Ensure all columns have header names", + "Remove empty columns from the CSV", + "Check for extra delimiters in the header row", + ] + ); } - if (missingRequiredFields.length > 0) { - Logger.warn`Missing fields from index mapping: ${missingRequiredFields.join( + // Enhanced validation: Check for headers with special characters that might cause issues + const problematicHeaders = headers.filter((header) => { + const trimmed = header.trim(); + return ( + trimmed.includes(",") || + trimmed.includes(";") || + trimmed.includes("\t") || + trimmed.includes("\n") || + trimmed.includes("\r") + ); + }); + + if (problematicHeaders.length > 0) { + Logger.warn`Headers contain special characters: ${problematicHeaders.join( ", " )}`; - Logger.tip`Data for these fields will be null in the indexed documents`; - } - - // Raise error if there's a significant mismatch between the headers and mapping - if ( - extraHeaders.length > expectedFields.length * 0.5 || - missingRequiredFields.length > expectedFields.length * 0.5 - ) { - Logger.error`Significant header/field mismatch detected`; - throw new ConductorError( - "Significant header/field mismatch detected - the CSV structure doesn't match the index mapping", - ErrorCodes.VALIDATION_FAILED, - { - extraHeaders, - missingRequiredFields, - details: { - expected: expectedFields, - found: cleanedHeaders, - }, - } + Logger.tipString( + "Consider renaming headers to avoid commas, semicolons, tabs, or line breaks" ); } - Logger.debug`Headers validated against index mapping`; - return true; - } catch (error: any) { - // If the index doesn't exist, provide a clear error - if ( - error.meta && - error.meta.body && - error.meta.body.error.type === "index_not_found_exception" - ) { - Logger.error`Index ${indexName} does not exist`; - throw new ConductorError( - `Index ${indexName} does not exist - create it first or use a different index name`, - ErrorCodes.INDEX_NOT_FOUND + // Enhanced validation: Check for very long headers + const longHeaders = headers.filter((header) => header.length > 100); + if (longHeaders.length > 0) { + Logger.warn`Very long headers detected (>100 chars): ${longHeaders + .map((h) => h.substring(0, 50) + "...") + .join(", ")}`; + Logger.tipString( + "Consider shortening header names for better readability" ); } - // Type-safe error handling for other errors - if (error instanceof ConductorError) { + Logger.debug`CSV structure validation passed: ${headers.length} valid headers`; + return true; + } catch (error) { + if (error instanceof Error && error.name === "ConductorError") { throw error; } - // Add more detailed error logging - Logger.error`Error validating headers against index: ${ - error instanceof Error ? error.message : String(error) - }`; - Logger.debug`Error details: ${ - error instanceof Error ? error.stack : "No stack trace available" - }`; - - throw new ConductorError( - "Error validating headers against index", - ErrorCodes.VALIDATION_FAILED, - { - originalError: error instanceof Error ? error.message : String(error), - errorType: error instanceof Error ? error.name : "Unknown Error", - } + throw ErrorFactory.csv( + `CSV structure validation failed: ${ + error instanceof Error ? error.message : String(error) + }`, + undefined, + 1, + [ + "Check CSV header format and structure", + "Ensure headers are properly formatted", + "Verify no special characters in headers", + "Review CSV file for formatting issues", + ] ); } } diff --git a/apps/conductor/src/validations/elasticsearchValidator.ts b/apps/conductor/src/validations/elasticsearchValidator.ts index ec629226..65340cc4 100644 --- a/apps/conductor/src/validations/elasticsearchValidator.ts +++ b/apps/conductor/src/validations/elasticsearchValidator.ts @@ -1,132 +1,476 @@ +/** + * Elasticsearch Validator + * + * Validates Elasticsearch connections, indices, and configurations. + * Enhanced with ErrorFactory patterns for consistent error handling. + */ + import { Client } from "@elastic/elasticsearch"; import { - ConductorError, - ErrorCodes, - createValidationError, -} from "../utils/errors"; + ConnectionValidationResult, + IndexValidationResult, +} from "../types/validations"; import { Logger } from "../utils/logger"; -import { ConnectionValidationResult, IndexValidationResult } from "../types"; +import { ErrorFactory } from "../utils/errors"; /** - * Validates Elasticsearch connection by making a ping request + * Enhanced Elasticsearch connection validation with detailed error analysis */ export async function validateElasticsearchConnection( client: Client, - config: any + url: string ): Promise { try { - Logger.info`Testing connection to Elasticsearch at ${config.elasticsearch.url}`; - + Logger.debug`Testing Elasticsearch connection to ${url}`; const startTime = Date.now(); - const response = await client.ping(); + + const response = await client.info(); const responseTime = Date.now() - startTime; - Logger.info`Connected to Elasticsearch successfully in ${responseTime}ms`; + Logger.success`Connected to Elasticsearch cluster: ${response.body.cluster_name}`; + Logger.debug`Response time: ${responseTime}ms`; return { valid: true, errors: [], + version: response.body.version?.number, + clusterName: response.body.cluster_name, responseTimeMs: responseTime, }; - } catch (error: any) { - const errorMessage = error instanceof Error ? error.message : String(error); - - // Log the error message - Logger.error`Failed to connect to Elasticsearch: ${errorMessage}`; + } catch (error) { + const enhancedError = createConnectionError(error, url); - // Add a warning with the override command info - Logger.commandValueTip( - "Check Elasticsearch is running and that the correct URL and auth params are in use", - "--url -u -p " - ); - - throw new ConductorError( - `Failed to connect to Elasticsearch at ${config.elasticsearch.url}`, - ErrorCodes.CONNECTION_ERROR, - error - ); + return { + valid: false, + errors: [enhancedError.message], + responseTimeMs: undefined, + }; } } /** - * Validates that an index exists + * Enhanced index validation with detailed error handling */ export async function validateIndex( client: Client, indexName: string ): Promise { try { - Logger.info`Checking if index ${indexName} exists`; + Logger.debug`Checking if index exists: ${indexName}`; - // Use the more reliable get method with a try/catch + const existsResponse = await client.indices.exists({ + index: indexName, + }); + + if (!existsResponse.body) { + return { + valid: false, + exists: false, + errors: [`Index '${indexName}' does not exist`], + }; + } + + // Get index details if it exists try { - const { body } = await client.indices.get({ index: indexName }); - - // Check if we actually got back information about the requested index - if (!body || !body[indexName]) { - Logger.error`Index ${indexName} not found in response`; - throw new ConductorError( - `Index ${indexName} not found`, - ErrorCodes.INDEX_NOT_FOUND - ); - } + Logger.debug`Getting index details for: ${indexName}`; - Logger.info`Index ${indexName} exists`; + const [mappingsResponse, settingsResponse] = await Promise.all([ + client.indices.getMapping({ index: indexName }), + client.indices.getSettings({ index: indexName }), + ]); return { valid: true, - errors: [], exists: true, + errors: [], + mappings: mappingsResponse.body[indexName]?.mappings, + settings: settingsResponse.body[indexName]?.settings, }; - } catch (indexError: any) { - // Check if the error is specifically about the index not existing - if ( - indexError.meta && - indexError.meta.body && - (indexError.meta.body.error.type === "index_not_found_exception" || - indexError.meta.body.status === 404) - ) { - Logger.error`Index ${indexName} does not exist`; - Logger.commandValueTip( - "Create the index first or use a different index name", - "-i " + } catch (detailsError) { + // Enhanced error handling for index details retrieval + throw ErrorFactory.connection( + `Failed to get index details: ${ + detailsError instanceof Error + ? detailsError.message + : String(detailsError) + }`, + "Elasticsearch", + undefined, // URL not available in this context + [ + `Check read permissions for index '${indexName}'`, + "Verify user has necessary cluster privileges", + "Ensure index is not in a locked state", + "Check Elasticsearch cluster health", + "Try accessing index through Kibana or other tools to verify accessibility", + ] + ); + } + } catch (error) { + // Enhanced error handling for index existence check + if (error instanceof Error && error.name === "ConductorError") { + throw error; // Re-throw enhanced errors + } + + throw ErrorFactory.connection( + `Failed to check index existence: ${ + error instanceof Error ? error.message : String(error) + }`, + "Elasticsearch", + undefined, // URL not available in this context + [ + `Verify index name '${indexName}' is correct`, + "Check Elasticsearch connection is stable", + "Ensure user has index read permissions", + "Check if index was recently deleted or renamed", + "Verify cluster is healthy and responsive", + ] + ); + } +} + +/** + * Validates that CSV headers match the Elasticsearch index mapping + * Enhanced with ErrorFactory patterns for detailed guidance + */ +export async function validateHeadersMatchMappings( + client: Client, + headers: string[], + indexName: string +): Promise { + try { + Logger.debug`Validating CSV headers against Elasticsearch index mapping`; + Logger.debug`Headers: ${headers.join(", ")}`; + Logger.debug`Index: ${indexName}`; + + // Get index mapping + let mappingResponse; + try { + mappingResponse = await client.indices.getMapping({ index: indexName }); + } catch (mappingError) { + throw ErrorFactory.index( + `Failed to retrieve mapping for index '${indexName}'`, + indexName, + [ + "Check that the index exists", + "Verify user has read permissions on the index", + "Ensure Elasticsearch is accessible", + `Test manually: GET /${indexName}/_mapping`, + "Create the index with proper mapping if it doesn't exist", + ] + ); + } + + const indexMapping = mappingResponse.body[indexName]?.mappings?.properties; + + if (!indexMapping) { + Logger.warn`No mapping properties found for index '${indexName}' - proceeding with dynamic mapping`; + return; + } + + // Get field names from mapping + const mappingFields = Object.keys(indexMapping); + Logger.debug`Index mapping fields: ${mappingFields.join(", ")}`; + + // Clean headers (remove whitespace, handle case sensitivity) + const cleanHeaders = headers.map((header) => header.trim()); + + // Check for missing fields in mapping + const unmappedHeaders = cleanHeaders.filter((header) => { + // Check for exact match first + if (mappingFields.includes(header)) return false; + + // Check for case-insensitive match + const lowerHeader = header.toLowerCase(); + return !mappingFields.some( + (field) => field.toLowerCase() === lowerHeader + ); + }); + + // Check for potential field naming issues + const potentialMismatches: string[] = []; + unmappedHeaders.forEach((header) => { + const lowerHeader = header.toLowerCase(); + const similarFields = mappingFields.filter( + (field) => + field.toLowerCase().includes(lowerHeader) || + lowerHeader.includes(field.toLowerCase()) + ); + + if (similarFields.length > 0) { + potentialMismatches.push( + `'${header}' might match: ${similarFields.join(", ")}` ); + } + }); + + // Report validation results + if (unmappedHeaders.length === 0) { + Logger.success`All CSV headers match index mapping fields`; + return; + } - throw new ConductorError( - `Index ${indexName} does not exist. Create the index first or use a different index name.`, - ErrorCodes.INDEX_NOT_FOUND, - indexError + // Handle unmapped headers + if (unmappedHeaders.length > headers.length * 0.5) { + // More than 50% of headers don't match - likely a serious issue + // UPDATED: Create a directly-visible error message that includes all key information + const errorMessage = `Many CSV headers don't match index mapping (${unmappedHeaders.length} of ${headers.length})`; + + // Create a more comprehensive list of suggestions with all the details + const enhancedSuggestions = [ + "Check that you're using the correct index", + "Verify CSV headers match expected field names", + "Consider updating index mapping to include new fields", + "Check for case sensitivity issues in field names", + // Add CSV headers directly in suggestions so they're always visible + `CSV headers (${headers.length}): ${headers.join(", ")}`, + // Add unmapped headers directly in suggestions so they're always visible + `Unmapped headers (${unmappedHeaders.length}): ${unmappedHeaders.join( + ", " + )}`, + // Add expected fields directly in suggestions so they're always visible + `Expected fields in mapping (${ + mappingFields.length + }): ${mappingFields.join(", ")}`, + ]; + + // Add potential matches if available + if (potentialMismatches.length > 0) { + enhancedSuggestions.push("Potential matches:"); + potentialMismatches.forEach((match) => { + enhancedSuggestions.push(` ${match}`); + }); + } + + // Add a direct recommendation based on the situation + if (unmappedHeaders.length === headers.length) { + enhancedSuggestions.push( + "All headers are unmapped - you may be using the wrong index or need to create a mapping first" ); - } else { - // Some other error occurred - throw indexError; } + + throw ErrorFactory.validation( + errorMessage, + { + unmappedHeaders, + mappingFields, + indexName, + potentialMismatches, + }, + enhancedSuggestions + ); + } else { + // Some headers don't match - warn but continue + Logger.warn`Some CSV headers don't match index mapping:`; + unmappedHeaders.forEach((header) => { + Logger.warn` - ${header}`; + }); + + if (potentialMismatches.length > 0) { + Logger.info`Potential matches found:`; + potentialMismatches.slice(0, 3).forEach((match) => { + Logger.info` - ${match}`; + }); + } + + Logger.tipString("Unmapped fields will use dynamic mapping if enabled"); + Logger.tipString( + "Consider updating your index mapping or CSV headers for better consistency" + ); } - } catch (error: any) { - const errorMessage = error instanceof Error ? error.message : String(error); - Logger.error`Index check failed: ${errorMessage}`; - - throw new ConductorError( - `Failed to check if index ${indexName} exists`, - ErrorCodes.INDEX_NOT_FOUND, - error + } catch (validateError) { + if ( + validateError instanceof Error && + validateError.name === "ConductorError" + ) { + throw validateError; + } + + throw ErrorFactory.validation( + `Failed to validate headers against mapping: ${ + validateError instanceof Error + ? validateError.message + : String(validateError) + }`, + { headers, indexName }, + [ + "Check Elasticsearch connectivity", + "Verify index exists and is accessible", + "Ensure proper authentication credentials", + "Check index permissions", + "Review network connectivity", + ] ); } } /** - * Validates that batch size is a positive number + * Enhanced batch size validation with helpful guidance */ export function validateBatchSize(batchSize: number): void { - if (!batchSize || isNaN(batchSize) || batchSize <= 0) { - throw createValidationError("Batch size must be a positive number", { - provided: batchSize, - }); + if (!Number.isInteger(batchSize) || batchSize <= 0) { + throw ErrorFactory.validation( + `Invalid batch size: ${batchSize}`, + { batchSize }, + [ + "Batch size must be a positive integer", + "Recommended range: 100-1000 for most use cases", + "Use smaller batch sizes (100-500) for large documents", + "Use larger batch sizes (500-1000) for small documents", + "Start with 500 and adjust based on performance", + ] + ); + } + + // Provide guidance for suboptimal batch sizes + if (batchSize < 10) { + Logger.warn`Batch size ${batchSize} is very small - this may impact performance`; + Logger.tipString( + "Consider using batch sizes of 100-1000 for better throughput" + ); + } else if (batchSize > 5000) { + Logger.warn`Batch size ${batchSize} is very large - this may cause memory issues`; + Logger.tipString( + "Consider using smaller batch sizes (500-2000) to avoid timeouts" + ); } - if (batchSize > 10000) { - Logger.warn`Batch size ${batchSize} is quite large and may cause performance issues`; - } else { - Logger.debug`Batch size validated: ${batchSize}`; + Logger.debug`Batch size validated: ${batchSize}`; +} + +/** + * Enhanced connection error analysis and categorization + */ +function createConnectionError(error: unknown, url: string): Error { + const errorMessage = error instanceof Error ? error.message : String(error); + + // Authentication errors + if ( + errorMessage.includes("401") || + errorMessage.includes("authentication") || + errorMessage.includes("unauthorized") + ) { + return ErrorFactory.connection( + "Elasticsearch authentication failed", + "Elasticsearch", + url, + [ + "Check username and password", + "Verify API key or token is valid", + "Ensure authentication method is correct", + "Check if credentials have expired", + "Verify service account permissions", + ] + ); + } + + // Authorization errors + if ( + errorMessage.includes("403") || + errorMessage.includes("forbidden") || + errorMessage.includes("permission") + ) { + return ErrorFactory.connection( + "Elasticsearch search access forbidden - insufficient permissions", + "Elasticsearch", + undefined, + [ + "User lacks necessary cluster or index permissions", + "Check user roles and privileges in Elasticsearch", + "Verify cluster-level permissions", + "Contact Elasticsearch administrator for access", + "Review security policy and user roles", + ] + ); + } + + // SSL/TLS errors + if ( + errorMessage.includes("SSL") || + errorMessage.includes("certificate") || + errorMessage.includes("CERT") + ) { + return ErrorFactory.connection( + "Elasticsearch SSL/TLS connection error", + "Elasticsearch", + undefined, + [ + "Check SSL certificate validity and trust", + "Verify TLS configuration matches server settings", + "Ensure proper SSL/TLS version compatibility", + "Check if HTTPS is required for this instance", + "Try HTTP if HTTPS is causing issues (development only)", + "Verify certificate authority and trust chain", + ] + ); + } + + // DNS resolution errors + if ( + errorMessage.includes("ENOTFOUND") || + errorMessage.includes("getaddrinfo") + ) { + return ErrorFactory.connection( + "Cannot resolve Elasticsearch hostname", + "Elasticsearch", + undefined, + [ + "Check hostname spelling in the URL", + "Verify DNS resolution is working", + "Try using IP address instead of hostname", + "Check network connectivity and DNS servers", + "Test with: nslookup ", + "Verify hosts file doesn't have conflicting entries", + ] + ); } + + // Connection refused errors + if (errorMessage.includes("ECONNREFUSED")) { + return ErrorFactory.connection( + "Connection refused by Elasticsearch server", + "Elasticsearch", + url, + [ + "Check that Elasticsearch is running", + "Verify correct port is being used", + "Ensure firewall is not blocking connection", + "Check network connectivity", + "Verify cluster status and health", + ] + ); + } + + // Timeout errors + if ( + errorMessage.includes("ETIMEDOUT") || + errorMessage.includes("timeout") || + errorMessage.includes("ESOCKETTIMEDOUT") + ) { + return ErrorFactory.connection( + "Elasticsearch connection timed out", + "Elasticsearch", + url, + [ + "Check network latency and connectivity", + "Verify Elasticsearch server is not overloaded", + "Increase connection timeout settings", + "Check if the server is responding to other requests", + "Verify server resource utilization", + ] + ); + } + + // Generic connection error + return ErrorFactory.connection( + `Failed to connect to Elasticsearch: ${errorMessage}`, + "Elasticsearch", + url, + [ + "Check that Elasticsearch is running and accessible", + "Verify network connectivity", + "Check authentication credentials", + "Ensure correct URL and port", + "Verify cluster health and status", + ] + ); } diff --git a/apps/conductor/src/validations/environment.ts b/apps/conductor/src/validations/environment.ts deleted file mode 100644 index 4379017a..00000000 --- a/apps/conductor/src/validations/environment.ts +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Environment Validation - * - * Validates the runtime environment configuration and requirements. - */ - -import { createValidationError } from "../utils/errors"; -import { Logger } from "../utils/logger"; - -interface EnvironmentValidationParams { - elasticsearchUrl: string; - // Add other relevant environment parameters -} - -/** - * Validates the environment configuration and requirements - */ -export async function validateEnvironment( - params: EnvironmentValidationParams -): Promise { - Logger.debug("Environment Validation"); - - // Validate Elasticsearch URL is provided - if (!params.elasticsearchUrl) { - throw createValidationError("Elasticsearch URL is required", { - parameter: "elasticsearchUrl", - expected: "valid URL", - }); - } - Logger.debug`Elasticsearch URL is provided: ${params.elasticsearchUrl}`; - - // Add additional environment validations as needed - - Logger.debug`Environment validation passed`; -} diff --git a/apps/conductor/src/validations/environmentValidator.ts b/apps/conductor/src/validations/environmentValidator.ts new file mode 100644 index 00000000..9e3163fe --- /dev/null +++ b/apps/conductor/src/validations/environmentValidator.ts @@ -0,0 +1,87 @@ +/** + * Environment Validation + * + * Validates the runtime environment configuration and requirements. + * Enhanced with ErrorFactory patterns for consistent error handling. + */ + +import { ErrorFactory } from "../utils/errors"; +import { Logger } from "../utils/logger"; + +interface EnvironmentValidationParams { + elasticsearchUrl: string; + // Add other relevant environment parameters +} + +/** + * Validates the environment configuration and requirements + * Enhanced with ErrorFactory for consistent error handling + */ +export async function validateEnvironment( + params: EnvironmentValidationParams +): Promise { + Logger.debug`Starting environment validation`; + + // Enhanced Elasticsearch URL validation + if (!params.elasticsearchUrl) { + Logger.warn`No Elasticsearch URL provided defaulting to http://localhost:9200`; + Logger.tip`Set Elasticsearch URL: conductor upload --url http://localhost:9200`; + } + + // Enhanced URL format validation + try { + const url = new URL(params.elasticsearchUrl); + if (!["http:", "https:"].includes(url.protocol)) { + throw ErrorFactory.config( + `Invalid Elasticsearch URL protocol: ${url.protocol}`, + "elasticsearchUrl", + [ + "Use HTTP or HTTPS protocol", + "Example: http://localhost:9200", + "Example: https://elasticsearch.company.com:9200", + "Check if SSL/TLS is required for your Elasticsearch instance", + ] + ); + } + Logger.debug`Elasticsearch URL validated: ${params.elasticsearchUrl}`; + } catch (urlError) { + if (urlError instanceof Error && urlError.name === "ConductorError") { + throw urlError; + } + + throw ErrorFactory.config( + `Invalid Elasticsearch URL format: ${params.elasticsearchUrl}`, + "elasticsearchUrl", + [ + "Use a valid URL format with protocol", + "Example: http://localhost:9200", + "Example: https://elasticsearch.company.com:9200", + "Check for typos in the URL", + "Ensure proper protocol (http:// or https://)", + ] + ); + } + + // Add additional environment validations as needed + // Example: Node.js version check + const nodeVersion = process.version; + const majorVersion = parseInt(nodeVersion.slice(1).split(".")[0]); + + if (majorVersion < 14) { + Logger.warn`Node.js version ${nodeVersion} is quite old`; + Logger.tipString( + "Consider upgrading to Node.js 16+ for better performance and security" + ); + } + + // Example: Memory check for large operations + const totalMemory = Math.round(require("os").totalmem() / 1024 / 1024 / 1024); + if (totalMemory < 2) { + Logger.warn`Low system memory detected: ${totalMemory}GB`; + Logger.tipString( + "Large CSV uploads may require more memory - consider using smaller batch sizes" + ); + } + + Logger.debug`Environment validation passed`; +} diff --git a/apps/conductor/src/validations/fileValidator.ts b/apps/conductor/src/validations/fileValidator.ts index bbc0a505..3d594f4a 100644 --- a/apps/conductor/src/validations/fileValidator.ts +++ b/apps/conductor/src/validations/fileValidator.ts @@ -3,6 +3,8 @@ * * Validates file existence, permissions, and basic properties * before processing CSV files into Elasticsearch. + * Enhanced with ErrorFactory patterns while maintaining original scope. + * Updated to use centralized file utilities. */ import * as fs from "fs"; @@ -14,12 +16,18 @@ import { ALLOWED_EXTENSIONS } from "./constants"; /** * Validates that files exist, have an extension, and that the extension is allowed. * Returns a structured result with a validity flag and error messages. + * Enhanced with better error messages but maintains original return structure. */ export async function validateFiles( filePaths: string[] ): Promise { if (!filePaths || filePaths.length === 0) { - return { valid: false, errors: ["No input files specified"] }; + return { + valid: false, + errors: [ + "No input files specified - use -f or --file to specify CSV files", + ], + }; } const notFoundFiles: string[] = []; @@ -28,6 +36,7 @@ export async function validateFiles( for (const filePath of filePaths) { const extension = path.extname(filePath).toLowerCase(); + if (!extension) { // File extension is missing, so record that as a warning. missingExtensions.push(filePath); @@ -36,30 +45,29 @@ export async function validateFiles( // Check if the extension is allowed. if (!ALLOWED_EXTENSIONS.includes(extension)) { - invalidExtensions.push(`${filePath} (${extension})`); + invalidExtensions.push(`${path.basename(filePath)} (${extension})`); continue; } // Check file existence. if (!fs.existsSync(filePath)) { - notFoundFiles.push(filePath); + notFoundFiles.push(path.basename(filePath)); continue; } } const errors: string[] = []; - // Log missing extension files as warnings + // Log missing extension files as warnings (maintain original behavior) if (missingExtensions.length > 0) { - Logger.warn( - `Missing file extension for: ${missingExtensions.join( - ", " - )}. Allowed extensions: ${ALLOWED_EXTENSIONS.join(", ")}` - ); + const missingList = missingExtensions + .map((f) => path.basename(f)) + .join(", "); + const allowedList = ALLOWED_EXTENSIONS.join(", "); + Logger.warn`Missing file extension for: ${missingList}. Allowed extensions: ${allowedList}`; } - // Only generate the error messages but don't log them directly - // Let the error handling system do the logging + // Enhanced error messages but same structure if (invalidExtensions.length > 0) { errors.push( `Invalid file extensions: ${invalidExtensions.join( @@ -69,7 +77,11 @@ export async function validateFiles( } if (notFoundFiles.length > 0) { - errors.push(`Files not found: ${notFoundFiles.join(", ")}`); + errors.push( + `Files not found: ${notFoundFiles.join( + ", " + )}. Check file paths and permissions.` + ); } return { valid: errors.length === 0, errors }; diff --git a/apps/conductor/src/validations/index.ts b/apps/conductor/src/validations/index.ts index 3d26fdfe..c7f9df63 100644 --- a/apps/conductor/src/validations/index.ts +++ b/apps/conductor/src/validations/index.ts @@ -8,4 +8,4 @@ export * from "./csvValidator"; export * from "./elasticsearchValidator"; export * from "./fileValidator"; -export * from "./environment"; +export * from "./environmentValidator"; diff --git a/apps/conductor/src/validations/utils.ts b/apps/conductor/src/validations/utils.ts index ddb27fdc..d7832e9b 100644 --- a/apps/conductor/src/validations/utils.ts +++ b/apps/conductor/src/validations/utils.ts @@ -2,19 +2,48 @@ * Common Validation Utilities * * Simple validators for common primitive values and configurations. + * Enhanced with ErrorFactory patterns for consistent error handling. */ -import { createValidationError } from "../utils/errors"; +import { ErrorFactory } from "../utils/errors"; import { Logger } from "../utils/logger"; /** - * Validates that a delimiter is a single character + * Validates that a delimiter is a single character with enhanced error handling */ export function validateDelimiter(delimiter: string): void { - if (!delimiter || delimiter.length !== 1) { - throw createValidationError("Delimiter must be a single character", { - provided: delimiter, - }); + if (!delimiter) { + throw ErrorFactory.config("CSV delimiter not specified", "delimiter", [ + "Provide a delimiter: conductor upload -f data.csv --delimiter ';'", + "Use common delimiters: ',' (comma), '\\t' (tab), ';' (semicolon)", + "Set CSV_DELIMITER environment variable", + ]); } - Logger.debug`Delimiter validated: '${delimiter}'`; + + if (typeof delimiter !== "string") { + throw ErrorFactory.config( + `Invalid delimiter type: ${typeof delimiter}`, + "delimiter", + [ + "Delimiter must be a string", + "Use a single character like ',' or ';'", + "Check command line argument format", + ] + ); + } + + if (delimiter.length !== 1) { + throw ErrorFactory.config( + `Invalid delimiter length: '${delimiter}' (${delimiter.length} characters)`, + "delimiter", + [ + "Delimiter must be exactly one character", + "Common delimiters: ',' (comma), ';' (semicolon), '\\t' (tab)", + "For tab delimiter, use: --delimiter $'\\t'", + "Check for extra spaces or quotes around the delimiter", + ] + ); + } + + Logger.debug`Delimiter validated: '${delimiter.replace("\t", "\\t")}'`; } diff --git a/generatedConfigs/elasticsearchConfigs/mapping.json b/generatedConfigs/elasticsearchConfigs/mapping.json new file mode 100644 index 00000000..bd812525 --- /dev/null +++ b/generatedConfigs/elasticsearchConfigs/mapping.json @@ -0,0 +1,120 @@ +{ + "index_patterns": [ + "datatable1-*" + ], + "aliases": { + "datatable1_centric": {} + }, + "mappings": { + "properties": { + "data": { + "type": "object", + "properties": { + "donor_id": { + "type": "keyword" + }, + "gender": { + "type": "keyword" + }, + "primary_site": { + "type": "keyword" + }, + "vital_status": { + "type": "keyword" + }, + "diagnosis_id": { + "type": "keyword" + }, + "age_at_diagnosis": { + "type": "integer" + }, + "cancer_type": { + "type": "keyword" + }, + "staging_system": { + "type": "keyword" + }, + "stage": { + "type": "keyword" + }, + "specimen_id": { + "type": "keyword" + }, + "specimen_type": { + "type": "keyword" + }, + "tissue_source": { + "type": "keyword" + }, + "sample_id": { + "type": "keyword" + }, + "sample_type": { + "type": "keyword" + }, + "treatment_id": { + "type": "keyword" + }, + "treatment_type": { + "type": "keyword" + }, + "treatment_start": { + "type": "integer" + }, + "treatment_duration": { + "type": "integer" + }, + "treatment_response": { + "type": "keyword" + }, + "drug_name": { + "type": "keyword" + }, + "followup_id": { + "type": "keyword" + }, + "followup_interval": { + "type": "integer" + }, + "disease_status": { + "type": "keyword" + }, + "submission_metadata": { + "type": "object", + "properties": { + "submitter_id": { + "type": "keyword", + "null_value": "No Data" + }, + "processing_started": { + "type": "date" + }, + "processed_at": { + "type": "date" + }, + "source_file": { + "type": "keyword", + "null_value": "No Data" + }, + "record_number": { + "type": "integer" + }, + "hostname": { + "type": "keyword", + "null_value": "No Data" + }, + "username": { + "type": "keyword", + "null_value": "No Data" + } + } + } + } + } + } + }, + "settings": { + "number_of_shards": 1, + "number_of_replicas": 0 + } +} \ No newline at end of file