refactor: Linting

bb_tasks/tasks/callable.py

@@ -2,14 +2,15 @@
 
 from __future__ import absolute_import, unicode_literals
 
+import json
 import subprocess
 
 from django.contrib.sites.models import Site
 
 from celery import shared_task
 
 from bb_vm.models import (
-    PortTunnel, VirtualBrickOwner, VirtualBrick,
+    PortTunnel, VirtualBrickOwner, VirtualBrick, RentedGPU
 )
 
 DIR = '/opt/brickbox/bb_vm/bash_scripts/'
@@ -30,11 +31,13 @@ def new_vm_subprocess(instance_id):
     catch_clone_errors.apply_async((instance_id,), countdown=60)
     remove_stale_clone.apply_async((instance_id,), countdown=180)
 
+    gpu_xml = RentedGPU.objects.filter(virt_brick=brick)[0].gpu.xml
+
     new_vm_script = [
                         f'{DIR}brick_connect.sh',
                         f'{str(host.ssh_username)}', f'{str(host.ssh_port)}',
                         'brick_img', f'{str(Site.objects.get_current().domain)}',
-                        f'{str(instance_id)}', f'{str(brick.assigned_gpus[0])}'
+                        f'{str(instance_id)}', f'{str(gpu_xml)}'
                     ]
 
     with subprocess.Popen(new_vm_script) as script:
@@ -145,9 +148,17 @@ def remove_stale_clone(instance_id):
     '''
     Last resort to remove clones that did not sucessfully start and never given a port.
     '''
-    brick = VirtualBrick.objects.get(id=instance_id)
+    try:
+        brick = VirtualBrick.objects.get(id=instance_id)
 
-    if brick.ssh_port is None and brick.is_on is False:
-        VirtualBrickOwner.objects.filter(virt_brick=instance_id).delete()
-        brick.delete()
-        destroy_vm_subprocess(instance_id)
+        if brick.ssh_port is None and brick.is_on is False:
+            VirtualBrickOwner.objects.filter(virt_brick=instance_id).delete()
+            brick.delete()
+            destroy_vm_subprocess(instance_id)
+
+        return True
+
+    except VirtualBrick.DoesNotExist as err:
+        return json.dumps({
+            'error':f'{err}'
+        })

bb_tasks/tasks/periodic.py

@@ -12,6 +12,7 @@
 
 from bb_vm.models import PortTunnel, HostFoundation, GPU, VirtualBrick
 
+# Script directory on server.
 DIR = '/opt/brickbox/bb_vm/bash_scripts/'
 
 @shared_task
@@ -59,24 +60,28 @@ def verify_brick_connectivity():
     command = "No Command Ran"
 
     for brick in bricks:
-        if brick.ssh_port.port_number is not None:
-            command = ['lsof', '-i', f'tcp:{brick.ssh_port.port_number}']
-            with Popen(command, stdout=PIPE) as script:
-
-                port_result = f"{script.stdout.read().decode('ascii')}"
-
-                if port_result and not brick.is_online:
-                    PortTunnel.objects.filter(
-                        port_number=brick.ssh_port.port_number
-                    ).update(is_alive=True)
-                    # brick.ssh_port.is_alive = True
-                    # brick.save()
-                elif not port_result and brick.is_online:
-                    PortTunnel.objects.filter(
-                        port_number=brick.ssh_port.port_number
-                    ).update(is_alive=False)
-                    # brick.ssh_port.is_alive = False
-                    # brick.save()
+        try:
+            if brick.ssh_port.port_number is not None:
+                command = ['lsof', '-i', f'tcp:{brick.ssh_port.port_number}']
+                with Popen(command, stdout=PIPE) as script:
+
+                    port_result = f"{script.stdout.read().decode('ascii')}"
+
+                    if port_result and not brick.is_online:
+                        PortTunnel.objects.filter(
+                            port_number=brick.ssh_port.port_number
+                        ).update(is_alive=True)
+                        # brick.ssh_port.is_alive = True
+                        # brick.save()
+                    elif not port_result and brick.is_online:
+                        PortTunnel.objects.filter(
+                            port_number=brick.ssh_port.port_number
+                        ).update(is_alive=False)
+                        # brick.ssh_port.is_alive = False
+                        # brick.save()
+        except AttributeError as err:
+            port_result = f"{err}"
+
 
     return json.dumps({
         'bricks':f'{bricks.values()}',
@@ -110,7 +115,7 @@ def reconnect_host(host):
         except AttributeError:
             prep_script_result = 'No output'
 
-    reconnect_script = "Not GPUs"
+    reconnect_script = "No GPUs"
     reconnect_script_result = "Not Ran"
 
     for gpu in GPU.objects.filter(host=host):

bb_vm/bash_scripts/brick_connect.sh

@@ -26,9 +26,13 @@ xml_data=$6
 # Check that port is active before trying to connect.
 if lsof -i tcp:"$port"; then
 
-    sshpass -p "" ssh -i /opt/brickbox/bb_vm/keys/"$host_user" -o StrictHostKeyChecking=no -p "$port" "$host_user"@localhost \
-    'bash -s' < /opt/brickbox/bb_vm/bash_scripts/"$action".sh "$url" "$instance" \""$xml_data"\" 2>> \
-    /opt/brickbox/bb_vm/bash_scripts/logs/brick_connect_errors.log
+    # curl -X POST https://"$url"/api/vmlog/ -d "level=20&virt_brick=$instance&message=$action%20sending%20over%20ssh%20tunnel&command=ssh"
+
+    sudo cat /opt/brickbox/bb_vm/bash_scripts/"$action".sh "$url" "$instance" \""$xml_data"\" | sudo ssh -i /opt/brickbox/bb_vm/keys/"$host_user" -o StrictHostKeyChecking=no -p "$port" "$host_user"@localhost 'sudo bash -s' 2>> /opt/brickbox/bb_vm/bash_scripts/logs/brick_connect_errors.log
+    # sudo ssh -i /opt/brickbox/bb_vm/keys/"$host_user" -o StrictHostKeyChecking=no -p "$port" "$host_user"@localhost 'sudo bash -s' < /opt/brickbox/bb_vm/bash_scripts/"$action".sh "$url" "$instance" \""$xml_data"\" 2>> /opt/brickbox/bb_vm/bash_scripts/logs/brick_connect_errors.log
+
+    # curl -X POST https://"$url"/api/vmlog/ -d "level=20&virt_brick=$instance&message=$action%20Sent%20over%20ssh%20tunnel&command=ssh&command_output=$last_command_output"
+
 
 else
 

bb_vm/bash_scripts/brick_img.sh

@@ -18,7 +18,7 @@ url=$1
 instance=$2
 xml_data=$3
 
-echo "$xml_data" >> bash_errors.log
+sudo echo "$xml_data" | sudo tee -a bash_errors.log > /dev/null
 
 # Logging
 curl -X POST https://"$url"/api/vmlog/ -d "level=20&virt_brick=$instance&message=Successfully%20SSH%20connection%20to%20host,%20creating%20$instance."
@@ -36,18 +36,18 @@ if sudo virsh domblklist "$instance" | grep "\/var\/lib\/libvirt\/images\/$insta
 
     curl -X POST https://dev.brickbox.io/vm/state/ -d "instance=$instance&verify=clone" &
 
-    rm /home/bb_dev/GPU.xml 2>> bash_errors.log
+    sudo rm GPU.xml 2>> bash_errors.log
 
     sleep 1
 
-    echo "$xml_data" >> /home/bb_dev/GPU.xml 2>> bash_errors.log
+    sudo echo "$xml_data" | sudo tee -a GPU.xml > /dev/null 2>> bash_errors.log
 
 
     # Logging
     curl -X POST https://"$url"/api/vmlog/ -d "level=20&virt_brick=$instance&message=GPU%20XML%0D%0A$xml_data"
 
 
-    last_command_output=$(sudo virsh attach-device "$instance" /home/bb_dev/GPU.xml --persistent 2>> bash_errors.log)
+    last_command_output=$(sudo virsh attach-device "$instance" GPU.xml --persistent 2>> bash_errors.log)
     last_command=!!
 
     sleep 20
@@ -66,15 +66,15 @@ if sudo virsh domblklist "$instance" | grep "\/var\/lib\/libvirt\/images\/$insta
 
         sleep 20
 
-        sudo virsh attach-device "$instance" /home/bb_dev/GPU.xml --persistent 2>> bash_errors.log
+        sudo virsh attach-device "$instance" GPU.xml --persistent 2>> bash_errors.log
 
         sleep 20
 
         sudo virsh reboot "$instance" 2>> bash_errors.log
 
     fi
 
-
+    # Verify that the GPU was attached sucessfully
     if sudo virsh dumpxml "$instance" | grep "hostdev"; then
 
         curl -X POST https://"$url"/api/vmlog/ -d "level=20&virt_brick=$instance&message=GPU%20attached%20successfully."  # Logging
@@ -88,7 +88,7 @@ if sudo virsh domblklist "$instance" | grep "\/var\/lib\/libvirt\/images\/$insta
 
     sudo virsh start "$instance" 2>> bash_errors.log
 
-    rm /home/bb_dev/GPU.xml 2>> bash_errors.log
+    sudo rm GPU.xml 2>> bash_errors.log
 
     curl https://dev.brickbox.io/vm/register/"$instance"/"$(sudo virsh domuuid "$instance")"/ &
 

bb_vm/bash_scripts/brick_prep.sh

@@ -7,26 +7,38 @@
 # Verify that QEMU is installed.
 if [ ! -f /usr/bin/qemu-system-x86_64 ]; then
     echo "QEMU is not installed. Attempting to install."
-    sudo apt-get install qemu-kvm libvirt-daemon-system libvirt-clients bridge-utils -y
+    # sudo apt-get install qemu-kvm libvirt-daemon-system libvirt-clients bridge-utils -y
+    sudo apt install qemu-kvm qemu-utils libvirt-daemon-system libvirt-clients bridge-utils virt-manager ovmf -y
     # apt-get install qemu-kvm -y
     # sudo apt-get install qemu -y
     exit 1
 fi
 
 # Check if directory exists.
-if [ ! -d "/vfio-pci-bind" ]; then
+if [ ! -d "vfio-pci-bind/" ]; then
     sudo git clone https://github.com/andre-richter/vfio-pci-bind.git
 fi
 
 # Check that file is executable, if not, make it executable.
 if [ ! -x /vfio-pci-bind/vfio-pci-bind ]; then
-    sudo chmod +x /vfio-pci-bind/vfio-pci-bind.sh
+    sudo chmod +x vfio-pci-bind/vfio-pci-bind.sh
 fi
 
-sudo ip link add name br0 type bridge
-sudo ip link set dev br0 up
-sudo ip link set dev enp3s0f1 master br0
+# check if br0 exists, if not, create it.
+if [ ! -d /sys/class/net/br0 ]; then
+    sudo ip link add name br0 type bridge
 
-sudo dhclient -r br0 && sudo dhclient br0
+    sudo ip link set dev br0 up
+    sudo ip link set dev enp3s0f1 master br0
+
+    sudo dhclient -r br0 && sudo dhclient br0
+fi
+
+sudo service gdm3 stop
+
+sudo rmmod nvidia_uvm
+sudo rmmod nvidia_drm
+sudo rmmod nvidia_modeset
+sudo rmmod nvidia
 
 exit

bb_vm/bash_scripts/brick_reconnect.sh

@@ -1,12 +1,23 @@
 #!/bin/bash
 
-# url=$1
+url=$1
 device=$2
 pcie=$3
 
 # cd /vfio-pci-bind || exit 1
 
-sudo ./vfio-pci-bind/vfio-pci-bind.sh "$device" "$pcie" 2>> bash_errors.log
+command_output=$(sudo ./vfio-pci-bind/vfio-pci-bind.sh "$device" "$pcie" 2>> bash_errors.log)
+
+# Verify that the GPU was attached sucessfully
+if sudo lspci -s "$pcie" -k | grep "vfio-pci"; then
+
+    curl -X POST https://"$url"/api/vmlog/ -d "level=20&virt_brick=NA&message=$pcie%20Driver%20set%20to%20VFIO&command_output=$command_output"  # Logging
+
+else
+
+    curl -X POST https://"$url"/api/vmlog/ -d "level=40&virt_brick=NA&message=Failed%20set%20$pcie%20driver%20set%20to%20VFIO&command_output=$command_output"    # Logging
+
+fi
 
 # sudo ip link add name br0 type bridge
 # sudo ip link set dev br0 up