automation_validation

#!/usr/bin/python import re import subprocess import sys import shutil import time if len(sys.argv) != 4: print('Please pass the site\nEx: python compare_connections.py <SITE> <RAC> <TYPE>\n') exit(1) width = shutil.get_terminal_size((70, 30)).columns if width < 120: input("Please maximize the terminal window for better display and Enter:") site_name = sys.argv[1] rac_name = sys.argv[2] dcu_type = sys.argv[3] valid_racs = [f'R{i}' for i in range(1,11)] if rac_name not in valid_racs: print(f"RAC name should be in {valid_racs}") exit(1) if dcu_type.upper() not in ["VRR", "GF"]: print("Dcu Type should be either VRR or GF") exit(1) dcu_type = dcu_type.upper() def get_console_out(command, shell=False): result = subprocess.run(command, shell=shell, stdout=subprocess.PIPE) lines = result.stdout.decode('utf-8').split("\n") return lines hostname = get_console_out("hostname", shell=True)[0] host = hostname[-4:].lower() #To get last 4 char i.e c001, c002, c003 etc host_no = int(host.lower().replace("c", "")) #----------- software validations reference data ----------- BIOS_VERSION_REQD="2.22.2" CPLD_VERSION_REQD="1.1.4" DIAG_APP_VERSION_REQD="6" BACKPLANE_VERSION_REQD="2.52" INTEL_X710_VERSION_REQD="22.5.7" INTEL_I350_VERSION_REQD="22.5.7" MELLANOX_X5_VERSION_REQD="16.32.20.04" MELLANOX_X6_VERSION_REQD="22.36.10.10" RAID_VERSION_REQD="25.5.9.0001" IDRAC_VERSION_REQD="7.00.00.173" NIC_VERSIONS = {"MT28800": MELLANOX_X5_VERSION_REQD, "MT28841": MELLANOX_X6_VERSION_REQD, "Ethernet": INTEL_I350_VERSION_REQD} #----------- hardware validations reference data ----------- MEMORY = "84GB" NIV_VALID_TYPE = ["NIC.Slot.3-1-1", "NIC.Slot.3-2-1", "NIC.Slot.8-1-1", "NIC.Slot.8-2-1"] #------------------ Connectivity --------------------------- RACS = { "VRR": { "R1": ["401to1", "402to1"], 'R2': ["403to1", "404to1"], "R3": ["405to1", "406to1"], 'R4': ["407to1", "408to1"], }, "GF": { "R1": ["401to2", "402to2"], 'R2': ["403to2", "404to2"] } } RAC = RACS[dcu_type][rac_name] connectivity_reference = { "VRR": { "c001": [12, 13], "c002": [14, 15], "c003": [16, 17], "c004": [18, 19], "c005": [20, 21], "c006": [22, 23], "c007": [24, 25], "c008": [26, 27], "c009": [28, 29], "c010": [30, 31] }, "GF": { "c001": [8, 9], "c002": [10, 11], "c003": [12, 13], "c004": [14, 15], "c005": [16, 17], "c006": [28, 29], "c007": [30, 31], "c008": [32, 33], "c009": [34, 35], "c010": [36, 37] }, "dest_itf": {"VRR": "TenGigE", "GF": "HundredGigE"} } connectivity_itf = connectivity_reference[dcu_type][host] dest_itf_ref = connectivity_reference["dest_itf"][dcu_type] sources = { 'idrac': {'dest': f'{site_name}{RAC[0]}', 'dest_itf': 'GigabitEthernet0/0/0/0', "enabled": False}, 'eno3': {'dest': f'{site_name}{RAC[1]}', 'dest_itf': f'GigabitEthernet0/0/0/{host_no}', "enabled": True}, 'enp94s0f0': {'dest': f'{site_name}{RAC[0]}', 'dest_itf': f'{dest_itf_ref}0/0/0/{connectivity_itf[0]}'}, 'enp94s0f1': {'dest': f'{site_name}{RAC[0]}', 'dest_itf': f'{dest_itf_ref}0/0/0/{connectivity_itf[1]}'}, 'enp216s0f0': {'dest': f'{site_name}{RAC[1]}', 'dest_itf': f'{dest_itf_ref}0/0/0/{connectivity_itf[0]}'}, 'enp216s0f1': {'dest': f'{site_name}{RAC[1]}', 'dest_itf': f'{dest_itf_ref}0/0/0/{connectivity_itf[1]}'}, } if dcu_type == "GF": sources = { 'idrac': {'dest': f'NA', 'dest_itf': 'NA', "enabled": False}, 'eno3': {'dest': f'NA', 'dest_itf': 'NA', "enabled": False}, 'enp94s0f0': {'dest': f'{site_name}{RAC[0]}', 'dest_itf': f'{dest_itf_ref}0/0/0/{connectivity_itf[0]}'}, 'enp94s0f1': {'dest': f'{site_name}{RAC[1]}', 'dest_itf': f'{dest_itf_ref}0/0/0/{connectivity_itf[0]}'}, 'enp216s0f0': {'dest': f'{site_name}{RAC[0]}', 'dest_itf': f'{dest_itf_ref}0/0/0/{connectivity_itf[1]}'}, 'enp216s0f1': {'dest': f'{site_name}{RAC[1]}', 'dest_itf': f'{dest_itf_ref}0/0/0/{connectivity_itf[1]}'}, } GF_PORT_NOS = ["P3P1", "P3P2", "P8P1", "P8P2"] #----------------------------------------------------------- RED = "\033[31m" GREEN = "\033[32m" YELLOW = "\033[33m" RESET = "\033[0m" def print_headers(): print(f"{'#':<5} {'Check':<25} {'Expected':<40} {'Installed':<40} {'Result':<10}") print("-" * 120) counter = 0 def print_row(ch, ex, ins, res): global counter counter += 1 print(f"{counter:<5} {ch:<25} {ex:<40} {ins:<40} {res:<10}") print('\n') print('-'*120) print("Hardware Validations") print_headers() cpu_lines = get_console_out(["racadm", "get", "BIOS.ProcSettings"]) for line in cpu_lines: if "NumCores" in line: cpu_cores = int(line.split('=')[1]) if cpu_cores > 22: print_row("CPU Cores", "Min 22", cpu_cores, YELLOW + "WARNING" + RESET) elif cpu_cores == 22: print_row("CPU Cores", "Min 22", cpu_cores, GREEN + "PASSED" + RESET) else: print_row("CPU Cores", "Min 22", cpu_cores, RED + "FAILED" + RESET) # if re.findall('Proc[0-9]+Brand', line): # print(line) memory_lines = get_console_out(["racadm", "get", "BIOS.MemSettings"]) for line in memory_lines: if "SysMemSize" in line: if "384 GB" not in line: print_row("Memory", "384GB", line, RED + "FAILED" + RESET) else: print_row("Memory", "384GB", "384 GB", GREEN + "PASSED" + RESET) slots = get_console_out(["racadm", "get", "NIC.NICConfig"]) valid_slots = ["NIC.Slot.3-1-1", "NIC.Slot.3-2-1", "NIC.Slot.8-1-1", "NIC.Slot.8-2-1"] slots_present = [] for line in slots: if "Slot" in line: cur_slot = re.findall("NIC\.Slot\.[0-9]+-[0-9]+-[0-9]+", line) if cur_slot: slots_present.append(cur_slot[0]) nn = 1 for slot in valid_slots: if slot not in slots_present: print_row("NIC Slot", slot, "Not Present", RED + "FAILED" + RESET) else: print_row("NIC Slot", slot, slot, GREEN + "PASSED" + RESET) nn += 1 ''' card_out = get_console_out("/usr/bin/lshw -businfo | /bin/grep 5e:00.0 |/usr/bin/awk '{print $4}'|/bin/sed 's/^ *//g'", shell=True) if card_out and card_out[0].strip() in ["MT28800", "MT28841"]: print_row("Correct Card", "MT28800 / MT28841", card_out[0].strip(), GREEN + "PASSED" + RESET) else: print_row("Correct Card", "MT28800 / MT28841", card_out[0].strip(), RED + "FAILED" + RESET) ''' dim_lines = get_console_out("dmidecode -t 17", shell=True) dims = [] cur_dim = None for dim_line in dim_lines: cur_line = dim_line.strip() if "Handle" in cur_line and "DMI type" in cur_line: cur_dim = [cur_line] if "Size" in cur_line and "No Module Installed" not in cur_line: cur_dim.append(cur_line) elif "Size" in cur_line and "No Module Installed" in cur_line: cur_dim = None if cur_dim and "Locator" in cur_line: cur_dim.append(cur_line) dims.append(cur_dim) cur_dim = None if len(dims) == 12: print_row("DIM Slots Count", 12, 12, GREEN + "PASSED" + RESET) else: print_row("DIM Slots Count", 12, len(dims), RED + "FAILED" + RESET) for dim in dims: if "32 GB" not in dim[1]: print_row("DIM Memory", "32 GB", dim[1].replace("Size:", "").strip(), RED + 'FAILED' + RESET) # print_row("DIM Memory", "32 GB", dim[0].split(",")[0]+" "+dim[1], RED + 'FAILED' + RESET) else: print_row("DIM Memory", "32 GB", dim[1].replace("Size:", "").strip(), GREEN + 'PASSED' + RESET) raid_slot = get_console_out(["racadm", "storage", "get", "controllers"], shell=False) if "RAID.Slot.6-1" in raid_slot[0]: print_row("RAID Slot", "RAID.Slot.6-1", "RAID.Slot.6-1", GREEN + "PASSED" + RESET) else: print_row("RAID Slot", "RAID.Slot.6-1", re.findall("RAID\.Slot\.[0-9]+\-[0-9]+", raid_slot[0])[0], RED + "FAILED" + RESET) hw_disks = get_console_out("/bin/racadm storage get pdisks|grep Disk|wc -l", shell=True) if hw_disks[0].strip() in ["4", "16"]: print_row("HW Disk Count", "4 or 16", hw_disks[0].strip(), GREEN + "PASSED" + RESET) else: print_row("HW Disk Count", "4 or 16", hw_disks[0].strip(), RED + "FAILED" + RESET) counter = 0 print('\n') print('-'*120) print("Software Validations") print_headers() BIOS_firmware = get_console_out("/bin/racadm getversion -f bios|sed -n 1p|awk -F= '{print $2}'|/bin/sed 's/^ *//g'", shell=True) BIOS_firmware = [i for i in BIOS_firmware if i and i.strip()!=""] if BIOS_firmware and BIOS_firmware[0].strip() == BIOS_VERSION_REQD: print_row("BIOS firmware", BIOS_VERSION_REQD, BIOS_VERSION_REQD, GREEN + "PASSED" + RESET) else: print_row("BIOS firmware", BIOS_VERSION_REQD, BIOS_firmware[0], RED + "FAILED" + RESET) iDRAC_firmware = get_console_out(["/bin/racadm getversion -f lc |sed -n 1p|awk -F= '{print $2}'|/bin/sed 's/^ *//g'"], shell=True) if iDRAC_firmware and iDRAC_firmware[0].strip() == IDRAC_VERSION_REQD: print_row("iDRAC firmware", IDRAC_VERSION_REQD, IDRAC_VERSION_REQD, GREEN + "PASSED" + RESET) else: print_row("iDRAC firmware", IDRAC_VERSION_REQD, iDRAC_firmware[0], RED + "FAILED" + RESET) RAID_controller_firmware = get_console_out(["/bin/racadm storage get controllers:RAID.Slot.6-1|/bin/grep -i Firmware|/usr/bin/awk -F= '{print $2}'|/bin/sed 's/^[ \t]*//;s/[ \t]*$//'"], shell=True) if RAID_controller_firmware and RAID_controller_firmware[0].strip() == RAID_VERSION_REQD: print_row("RAID controller firmware", RAID_VERSION_REQD, RAID_VERSION_REQD, GREEN + "PASSED" + RESET) else: print_row("RAID controller firmware", RAID_VERSION_REQD, RAID_controller_firmware[0], RED + "FAILED" + RESET) CPLD_firmware = get_console_out(["/bin/racadm getversion -c|sed -n 1p|awk -F= '{print $2}'|/bin/sed 's/^ *//g'"], shell=True) if CPLD_firmware and CPLD_firmware[0].strip() == CPLD_VERSION_REQD: print_row("CPLD firmware", CPLD_VERSION_REQD, CPLD_VERSION_REQD, GREEN + "PASSED" + RESET) else: print_row("CPLD firmware", CPLD_VERSION_REQD, CPLD_firmware[0], RED + "FAILED" + RESET) nics_order = ['eno1', 'eno2', 'eno3', 'eno4', 'enp94s0f0', 'enp94s0f1', 'enp216s0f0', 'enp216s0f1'] for i in range(1,9): cmd_out = get_console_out("/bin/racadm get NIC.FrmwImgMenu."+str(i)+"|sed -n 3p|awk -F= '{print $2}'", shell=True) if cmd_out and i<9: if i<5: if cmd_out[0].strip() == INTEL_X710_VERSION_REQD: print_row(nics_order[i-1], INTEL_X710_VERSION_REQD, INTEL_X710_VERSION_REQD, GREEN + "PASSED" + RESET) else: print_row(nics_order[i-1], INTEL_X710_VERSION_REQD, cmd_out[0], RED + "FAILED" + RESET) else: nic_type = get_console_out("/usr/bin/lshw -businfo | /bin/grep 5e:00.0 |/usr/bin/awk '{print $4}'|/bin/sed 's/^ *//g'", shell=True)[0].strip() if cmd_out[0].strip() == NIC_VERSIONS[nic_type]: print_row(nics_order[i-1], NIC_VERSIONS[nic_type], NIC_VERSIONS[nic_type], GREEN + "PASSED" + RESET) else: print_row(nics_order[i-1], NIC_VERSIONS[nic_type], cmd_out[0], RED + "FAILED" + RESET) counter = 0 print('\n') print('-'*120) print('Connectivity Validations') print_headers() def skip_interface(port_desc_line): for port in GF_PORT_NOS: if port in port_desc_line: return False return True def compare_server(sources): interfaces = sources compared = {} lines = get_console_out(['lldpcli', 'show', 'neighbors']) cur_if = None eno3_count = 0 line_count = 0 eno3_flag = False for line in lines: line_count += 1 line1 = line.strip() if "Interface" in line1: cur_if = line1.replace("Interface:", "").split(",")[0].strip() compared[cur_if] = [] if cur_if == "eno3": eno3_count += 1 if "SysName:" in line1: if cur_if == "eno3" and eno3_flag: continue cur_dest = re.findall(".*40[1-2]+to[1-2]", line1.replace("SysName:", "").strip().lower()) if cur_dest: interfaces[cur_if]['cur_dest'] = cur_dest[0] else: interfaces[cur_if]['cur_dest'] = line1.replace("SysName:", "").strip() if "PortID" in line1 and 'PASSED' not in interfaces[cur_if].get('result', ""): if cur_if == "eno3" and 'PXE' not in lines[line_count]: continue if cur_if == "eno3": eno3_flag = True if dcu_type == "GF" and skip_interface(lines[line_count]): continue cur_itf_lst = re.findall(f'{interfaces[cur_if]["dest_itf"][0:-2]}[0-9]+', line1) cur_itf = cur_itf_lst[0] if cur_itf_lst else line1.replace("PortID:", "").replace("mac", "").replace("ifname", "").strip() interfaces[cur_if]['cur_itf'] = cur_itf if interfaces[cur_if]["dest_itf"] in cur_itf and interfaces[cur_if]['cur_dest'] == interfaces[cur_if]['dest'].lower(): interfaces[cur_if]['result'] = GREEN + "PASSED" + RESET else: interfaces[cur_if]['result'] = RED + "FAILED" + RESET if cur_if in ["idrac", "eno3"] and not interfaces[cur_if]["enabled"]: interfaces[cur_if]['cur_dest'] = "" interfaces[cur_if]["cur_itf"] = "" interfaces[cur_if]['result'] = YELLOW + "NA" + RESET return compared compare_server(sources) for intf, rec in sources.items(): cur_val = f"{rec.get('cur_dest', '')}:{rec.get('cur_itf', '')}" if cur_val.strip() == ":": cur_val = "" if intf in ["idrac", "eno3"] and not sources[intf]["enabled"]: print_row(f"Conn: {intf}", f"{rec['dest']}:{rec['dest_itf']}", cur_val, YELLOW + "NA" + RESET) else: print_row(f"Conn: {intf}", f"{rec['dest']}:{rec['dest_itf']}", cur_val, rec.get('result', RED +'FAILED'+ RESET)) print('\n')

Posted By