From da5f1f2d0c6dd507fa573ac1c107cf680ce9a92c Mon Sep 17 00:00:00 2001 From: mindesbunister Date: Fri, 10 Oct 2025 15:39:59 +0200 Subject: [PATCH] Reorganize project structure: move code to src/, docs to docs/, config to config/, scripts to scripts/, results to results/, tests to tests/. Keep only main script and latest scan results in root. --- complete_workflow.sh | 12 +- ...g-gw-nue01.egonetix.lan-20251010104729.xml | 0 ...ig-gw-st01.egonetix.lan-20251010104656.xml | 0 requirements.txt => config/requirements.txt | 0 GETTING_STARTED.md => docs/GETTING_STARTED.md | 0 PROJECT_SUMMARY.md => docs/PROJECT_SUMMARY.md | 0 README.md => docs/README.md | 0 network_report.md => docs/network_report.md | 0 network_summary.md => docs/network_summary.md | 0 results/network_summary_20251010_113352.md | 53 +++++++ results/network_summary_20251010_113528.md | 53 +++++++ results/network_summary_20251010_150701.md | 53 +++++++ results/network_summary_20251010_151015.md | 53 +++++++ results/network_summary_20251010_151139.md | 53 +++++++ results/network_summary_20251010_153418.md | 53 +++++++ EXAMPLES.sh => scripts/EXAMPLES.sh | 0 quickstart.sh => scripts/quickstart.sh | 0 .../run_network_mapping.sh | 0 .../comprehensive_mapper.py | 0 .../integrated_scanner.py | 137 +++++++++++++++++- network_scanner.py => src/network_scanner.py | 111 +++++++++----- .../pfsense_integrator.py | 38 ++++- pfsense_scanner.py => src/pfsense_scanner.py | 0 .../pfsense_xml_parser.py | 0 svg_generator.py => src/svg_generator.py | 0 test_system.py => src/test_system.py | 18 +-- 26 files changed, 581 insertions(+), 53 deletions(-) rename config-gw-nue01.egonetix.lan-20251010104729.xml => config/config-gw-nue01.egonetix.lan-20251010104729.xml (100%) rename config-gw-st01.egonetix.lan-20251010104656.xml => config/config-gw-st01.egonetix.lan-20251010104656.xml (100%) rename requirements.txt => config/requirements.txt (100%) rename GETTING_STARTED.md => docs/GETTING_STARTED.md (100%) rename PROJECT_SUMMARY.md => docs/PROJECT_SUMMARY.md (100%) rename README.md => docs/README.md (100%) rename network_report.md => docs/network_report.md (100%) rename network_summary.md => docs/network_summary.md (100%) create mode 100644 results/network_summary_20251010_113352.md create mode 100644 results/network_summary_20251010_113528.md create mode 100644 results/network_summary_20251010_150701.md create mode 100644 results/network_summary_20251010_151015.md create mode 100644 results/network_summary_20251010_151139.md create mode 100644 results/network_summary_20251010_153418.md rename EXAMPLES.sh => scripts/EXAMPLES.sh (100%) rename quickstart.sh => scripts/quickstart.sh (100%) rename run_network_mapping.sh => scripts/run_network_mapping.sh (100%) rename comprehensive_mapper.py => src/comprehensive_mapper.py (100%) rename integrated_scanner.py => src/integrated_scanner.py (65%) rename network_scanner.py => src/network_scanner.py (82%) rename pfsense_integrator.py => src/pfsense_integrator.py (90%) rename pfsense_scanner.py => src/pfsense_scanner.py (100%) rename pfsense_xml_parser.py => src/pfsense_xml_parser.py (100%) rename svg_generator.py => src/svg_generator.py (100%) rename test_system.py => src/test_system.py (95%) diff --git a/complete_workflow.sh b/complete_workflow.sh index 2af9815..93b409f 100755 --- a/complete_workflow.sh +++ b/complete_workflow.sh @@ -33,8 +33,8 @@ log_error() { } # Check if we're in the right directory -if [ ! -f "integrated_scanner.py" ]; then - log_error "integrated_scanner.py not found. Please run this script from the network scanner directory." +if [ ! -f "src/integrated_scanner.py" ]; then + log_error "src/integrated_scanner.py not found. Please run this script from the network scanner directory." exit 1 fi @@ -48,7 +48,7 @@ fi # Step 1: Run system verification log_info "Step 1: Verifying system requirements..." -if ./test_system.py >/dev/null 2>&1; then +if python3 src/test_system.py >/dev/null 2>&1; then log_success "System verification passed" else log_error "System verification failed. Please check the output above." @@ -58,7 +58,7 @@ fi # Step 2: Run integrated network scan log_info "Step 2: Running integrated network scan..." SCAN_OUTPUT="network_scan_$(date +%Y%m%d_%H%M%S).json" -if ./integrated_scanner.py -o "$SCAN_OUTPUT" -v; then +if python3 src/integrated_scanner.py -o "$SCAN_OUTPUT" -v; then log_success "Network scan completed: $SCAN_OUTPUT" else log_error "Network scan failed" @@ -68,7 +68,7 @@ fi # Step 3: Generate SVG diagram log_info "Step 3: Generating network diagram..." SVG_OUTPUT="${SCAN_OUTPUT%.json}.svg" -if ./svg_generator.py "$SCAN_OUTPUT" -o "$SVG_OUTPUT"; then +if python3 src/svg_generator.py "$SCAN_OUTPUT" -o "$SVG_OUTPUT"; then log_success "SVG diagram generated: $SVG_OUTPUT" else log_error "SVG generation failed" @@ -79,7 +79,7 @@ fi if [ "$XML_FILES" -gt 0 ]; then log_info "Step 4: Generating pfSense network summary..." SUMMARY_OUTPUT="network_summary_$(date +%Y%m%d_%H%M%S).md" - if ./pfsense_integrator.py *.xml --summary "$SUMMARY_OUTPUT"; then + if python3 src/pfsense_integrator.py *.xml --summary "$SUMMARY_OUTPUT"; then log_success "Network summary generated: $SUMMARY_OUTPUT" else log_warning "Network summary generation failed" diff --git a/config-gw-nue01.egonetix.lan-20251010104729.xml b/config/config-gw-nue01.egonetix.lan-20251010104729.xml similarity index 100% rename from config-gw-nue01.egonetix.lan-20251010104729.xml rename to config/config-gw-nue01.egonetix.lan-20251010104729.xml diff --git a/config-gw-st01.egonetix.lan-20251010104656.xml b/config/config-gw-st01.egonetix.lan-20251010104656.xml similarity index 100% rename from config-gw-st01.egonetix.lan-20251010104656.xml rename to config/config-gw-st01.egonetix.lan-20251010104656.xml diff --git a/requirements.txt b/config/requirements.txt similarity index 100% rename from requirements.txt rename to config/requirements.txt diff --git a/GETTING_STARTED.md b/docs/GETTING_STARTED.md similarity index 100% rename from GETTING_STARTED.md rename to docs/GETTING_STARTED.md diff --git a/PROJECT_SUMMARY.md b/docs/PROJECT_SUMMARY.md similarity index 100% rename from PROJECT_SUMMARY.md rename to docs/PROJECT_SUMMARY.md diff --git a/README.md b/docs/README.md similarity index 100% rename from README.md rename to docs/README.md diff --git a/network_report.md b/docs/network_report.md similarity index 100% rename from network_report.md rename to docs/network_report.md diff --git a/network_summary.md b/docs/network_summary.md similarity index 100% rename from network_summary.md rename to docs/network_summary.md diff --git a/results/network_summary_20251010_113352.md b/results/network_summary_20251010_113352.md new file mode 100644 index 0000000..29b3e92 --- /dev/null +++ b/results/network_summary_20251010_113352.md @@ -0,0 +1,53 @@ +# Network Topology Summary +Generated from pfSense XML configurations + +## pfSense Firewall: gw-nue01 +**Version:** unknown +**Domain:** egonetix.lan + +### Network Interfaces +- **WAN** (wan): dhcp +- **LAN** (lan): 10.0.0.1 +- **wireguardnachhause** (opt1): 10.69.69.1 + - Gateway: WirusguardusGW + +### Static Routes +- 172.20.0.0/16 via WirusguardusGW + *heyme* + +### WireGuard VPN +- **Tunnel tun_wg0** (Port 51820) + *heyme* + - Peer: wireguardheyme - Networks: 172.20.0.0/16, 10.69.69.2/32 + +### DHCP Configuration + +## pfSense Firewall: gw-st01 +**Version:** unknown +**Domain:** egonetix.lan + +### Network Interfaces +- **WAN** (wan): 192.168.178.3 + - Gateway: WANGW +- **LAN** (lan): 172.20.20.1 +- **wireguardnnbesch** (opt1): 10.69.69.2 + - Gateway: wirenuenbesch +- **HomeAssistant** (opt2): 172.20.70.1 +- **WireguardOpenvpn** (opt3): 10.5.0.2 + - Gateway: WireguardOpenvpnGW + +### Static Routes +- 10.0.0.0/24 via wirenuenbesch + *wireguardnünbesch* +- 12.1.0.0/24 via wirenuenbesch + *openvpn nutzer* + +### WireGuard VPN +- **Tunnel tun_wg0** (Port 51820) + *de1099.nordvpn.com* +- **Tunnel tun_wg1** (Port 51821) + *wireguardnünbesch* + - Peer: de1099.nordvpn.com - Networks: 0.0.0.0/0 + - Peer: wireguardnünbesch - Networks: 10.0.0.0/24, 10.69.69.1/32, 12.1.0.0/24 + +### DHCP Configuration diff --git a/results/network_summary_20251010_113528.md b/results/network_summary_20251010_113528.md new file mode 100644 index 0000000..29b3e92 --- /dev/null +++ b/results/network_summary_20251010_113528.md @@ -0,0 +1,53 @@ +# Network Topology Summary +Generated from pfSense XML configurations + +## pfSense Firewall: gw-nue01 +**Version:** unknown +**Domain:** egonetix.lan + +### Network Interfaces +- **WAN** (wan): dhcp +- **LAN** (lan): 10.0.0.1 +- **wireguardnachhause** (opt1): 10.69.69.1 + - Gateway: WirusguardusGW + +### Static Routes +- 172.20.0.0/16 via WirusguardusGW + *heyme* + +### WireGuard VPN +- **Tunnel tun_wg0** (Port 51820) + *heyme* + - Peer: wireguardheyme - Networks: 172.20.0.0/16, 10.69.69.2/32 + +### DHCP Configuration + +## pfSense Firewall: gw-st01 +**Version:** unknown +**Domain:** egonetix.lan + +### Network Interfaces +- **WAN** (wan): 192.168.178.3 + - Gateway: WANGW +- **LAN** (lan): 172.20.20.1 +- **wireguardnnbesch** (opt1): 10.69.69.2 + - Gateway: wirenuenbesch +- **HomeAssistant** (opt2): 172.20.70.1 +- **WireguardOpenvpn** (opt3): 10.5.0.2 + - Gateway: WireguardOpenvpnGW + +### Static Routes +- 10.0.0.0/24 via wirenuenbesch + *wireguardnünbesch* +- 12.1.0.0/24 via wirenuenbesch + *openvpn nutzer* + +### WireGuard VPN +- **Tunnel tun_wg0** (Port 51820) + *de1099.nordvpn.com* +- **Tunnel tun_wg1** (Port 51821) + *wireguardnünbesch* + - Peer: de1099.nordvpn.com - Networks: 0.0.0.0/0 + - Peer: wireguardnünbesch - Networks: 10.0.0.0/24, 10.69.69.1/32, 12.1.0.0/24 + +### DHCP Configuration diff --git a/results/network_summary_20251010_150701.md b/results/network_summary_20251010_150701.md new file mode 100644 index 0000000..29b3e92 --- /dev/null +++ b/results/network_summary_20251010_150701.md @@ -0,0 +1,53 @@ +# Network Topology Summary +Generated from pfSense XML configurations + +## pfSense Firewall: gw-nue01 +**Version:** unknown +**Domain:** egonetix.lan + +### Network Interfaces +- **WAN** (wan): dhcp +- **LAN** (lan): 10.0.0.1 +- **wireguardnachhause** (opt1): 10.69.69.1 + - Gateway: WirusguardusGW + +### Static Routes +- 172.20.0.0/16 via WirusguardusGW + *heyme* + +### WireGuard VPN +- **Tunnel tun_wg0** (Port 51820) + *heyme* + - Peer: wireguardheyme - Networks: 172.20.0.0/16, 10.69.69.2/32 + +### DHCP Configuration + +## pfSense Firewall: gw-st01 +**Version:** unknown +**Domain:** egonetix.lan + +### Network Interfaces +- **WAN** (wan): 192.168.178.3 + - Gateway: WANGW +- **LAN** (lan): 172.20.20.1 +- **wireguardnnbesch** (opt1): 10.69.69.2 + - Gateway: wirenuenbesch +- **HomeAssistant** (opt2): 172.20.70.1 +- **WireguardOpenvpn** (opt3): 10.5.0.2 + - Gateway: WireguardOpenvpnGW + +### Static Routes +- 10.0.0.0/24 via wirenuenbesch + *wireguardnünbesch* +- 12.1.0.0/24 via wirenuenbesch + *openvpn nutzer* + +### WireGuard VPN +- **Tunnel tun_wg0** (Port 51820) + *de1099.nordvpn.com* +- **Tunnel tun_wg1** (Port 51821) + *wireguardnünbesch* + - Peer: de1099.nordvpn.com - Networks: 0.0.0.0/0 + - Peer: wireguardnünbesch - Networks: 10.0.0.0/24, 10.69.69.1/32, 12.1.0.0/24 + +### DHCP Configuration diff --git a/results/network_summary_20251010_151015.md b/results/network_summary_20251010_151015.md new file mode 100644 index 0000000..29b3e92 --- /dev/null +++ b/results/network_summary_20251010_151015.md @@ -0,0 +1,53 @@ +# Network Topology Summary +Generated from pfSense XML configurations + +## pfSense Firewall: gw-nue01 +**Version:** unknown +**Domain:** egonetix.lan + +### Network Interfaces +- **WAN** (wan): dhcp +- **LAN** (lan): 10.0.0.1 +- **wireguardnachhause** (opt1): 10.69.69.1 + - Gateway: WirusguardusGW + +### Static Routes +- 172.20.0.0/16 via WirusguardusGW + *heyme* + +### WireGuard VPN +- **Tunnel tun_wg0** (Port 51820) + *heyme* + - Peer: wireguardheyme - Networks: 172.20.0.0/16, 10.69.69.2/32 + +### DHCP Configuration + +## pfSense Firewall: gw-st01 +**Version:** unknown +**Domain:** egonetix.lan + +### Network Interfaces +- **WAN** (wan): 192.168.178.3 + - Gateway: WANGW +- **LAN** (lan): 172.20.20.1 +- **wireguardnnbesch** (opt1): 10.69.69.2 + - Gateway: wirenuenbesch +- **HomeAssistant** (opt2): 172.20.70.1 +- **WireguardOpenvpn** (opt3): 10.5.0.2 + - Gateway: WireguardOpenvpnGW + +### Static Routes +- 10.0.0.0/24 via wirenuenbesch + *wireguardnünbesch* +- 12.1.0.0/24 via wirenuenbesch + *openvpn nutzer* + +### WireGuard VPN +- **Tunnel tun_wg0** (Port 51820) + *de1099.nordvpn.com* +- **Tunnel tun_wg1** (Port 51821) + *wireguardnünbesch* + - Peer: de1099.nordvpn.com - Networks: 0.0.0.0/0 + - Peer: wireguardnünbesch - Networks: 10.0.0.0/24, 10.69.69.1/32, 12.1.0.0/24 + +### DHCP Configuration diff --git a/results/network_summary_20251010_151139.md b/results/network_summary_20251010_151139.md new file mode 100644 index 0000000..29b3e92 --- /dev/null +++ b/results/network_summary_20251010_151139.md @@ -0,0 +1,53 @@ +# Network Topology Summary +Generated from pfSense XML configurations + +## pfSense Firewall: gw-nue01 +**Version:** unknown +**Domain:** egonetix.lan + +### Network Interfaces +- **WAN** (wan): dhcp +- **LAN** (lan): 10.0.0.1 +- **wireguardnachhause** (opt1): 10.69.69.1 + - Gateway: WirusguardusGW + +### Static Routes +- 172.20.0.0/16 via WirusguardusGW + *heyme* + +### WireGuard VPN +- **Tunnel tun_wg0** (Port 51820) + *heyme* + - Peer: wireguardheyme - Networks: 172.20.0.0/16, 10.69.69.2/32 + +### DHCP Configuration + +## pfSense Firewall: gw-st01 +**Version:** unknown +**Domain:** egonetix.lan + +### Network Interfaces +- **WAN** (wan): 192.168.178.3 + - Gateway: WANGW +- **LAN** (lan): 172.20.20.1 +- **wireguardnnbesch** (opt1): 10.69.69.2 + - Gateway: wirenuenbesch +- **HomeAssistant** (opt2): 172.20.70.1 +- **WireguardOpenvpn** (opt3): 10.5.0.2 + - Gateway: WireguardOpenvpnGW + +### Static Routes +- 10.0.0.0/24 via wirenuenbesch + *wireguardnünbesch* +- 12.1.0.0/24 via wirenuenbesch + *openvpn nutzer* + +### WireGuard VPN +- **Tunnel tun_wg0** (Port 51820) + *de1099.nordvpn.com* +- **Tunnel tun_wg1** (Port 51821) + *wireguardnünbesch* + - Peer: de1099.nordvpn.com - Networks: 0.0.0.0/0 + - Peer: wireguardnünbesch - Networks: 10.0.0.0/24, 10.69.69.1/32, 12.1.0.0/24 + +### DHCP Configuration diff --git a/results/network_summary_20251010_153418.md b/results/network_summary_20251010_153418.md new file mode 100644 index 0000000..29b3e92 --- /dev/null +++ b/results/network_summary_20251010_153418.md @@ -0,0 +1,53 @@ +# Network Topology Summary +Generated from pfSense XML configurations + +## pfSense Firewall: gw-nue01 +**Version:** unknown +**Domain:** egonetix.lan + +### Network Interfaces +- **WAN** (wan): dhcp +- **LAN** (lan): 10.0.0.1 +- **wireguardnachhause** (opt1): 10.69.69.1 + - Gateway: WirusguardusGW + +### Static Routes +- 172.20.0.0/16 via WirusguardusGW + *heyme* + +### WireGuard VPN +- **Tunnel tun_wg0** (Port 51820) + *heyme* + - Peer: wireguardheyme - Networks: 172.20.0.0/16, 10.69.69.2/32 + +### DHCP Configuration + +## pfSense Firewall: gw-st01 +**Version:** unknown +**Domain:** egonetix.lan + +### Network Interfaces +- **WAN** (wan): 192.168.178.3 + - Gateway: WANGW +- **LAN** (lan): 172.20.20.1 +- **wireguardnnbesch** (opt1): 10.69.69.2 + - Gateway: wirenuenbesch +- **HomeAssistant** (opt2): 172.20.70.1 +- **WireguardOpenvpn** (opt3): 10.5.0.2 + - Gateway: WireguardOpenvpnGW + +### Static Routes +- 10.0.0.0/24 via wirenuenbesch + *wireguardnünbesch* +- 12.1.0.0/24 via wirenuenbesch + *openvpn nutzer* + +### WireGuard VPN +- **Tunnel tun_wg0** (Port 51820) + *de1099.nordvpn.com* +- **Tunnel tun_wg1** (Port 51821) + *wireguardnünbesch* + - Peer: de1099.nordvpn.com - Networks: 0.0.0.0/0 + - Peer: wireguardnünbesch - Networks: 10.0.0.0/24, 10.69.69.1/32, 12.1.0.0/24 + +### DHCP Configuration diff --git a/EXAMPLES.sh b/scripts/EXAMPLES.sh similarity index 100% rename from EXAMPLES.sh rename to scripts/EXAMPLES.sh diff --git a/quickstart.sh b/scripts/quickstart.sh similarity index 100% rename from quickstart.sh rename to scripts/quickstart.sh diff --git a/run_network_mapping.sh b/scripts/run_network_mapping.sh similarity index 100% rename from run_network_mapping.sh rename to scripts/run_network_mapping.sh diff --git a/comprehensive_mapper.py b/src/comprehensive_mapper.py similarity index 100% rename from comprehensive_mapper.py rename to src/comprehensive_mapper.py diff --git a/integrated_scanner.py b/src/integrated_scanner.py similarity index 65% rename from integrated_scanner.py rename to src/integrated_scanner.py index 454cf97..c58e116 100755 --- a/integrated_scanner.py +++ b/src/integrated_scanner.py @@ -7,7 +7,10 @@ Combines network scanning with pfSense-specific features import json import logging import argparse +import subprocess +import re from datetime import datetime +from typing import Dict from network_scanner import NetworkScanner, NetworkSegment from pfsense_scanner import PfSenseScanner from dataclasses import asdict @@ -38,11 +41,71 @@ class IntegratedNetworkScanner: # Check for pfSense XML files and integrate them self._integrate_pfsense_xml() + # Scan pfSense LAN networks + self._scan_pfsense_lan_networks() + # Identify and enhance pfSense devices self._scan_pfsense_devices() logger.info("Integrated scan complete") + def _scan_pfsense_lan_networks(self): + """Scan LAN networks served by pfSense devices""" + logger.info("Scanning pfSense LAN networks...") + + # Get pfSense configurations from the integrator + try: + from pfsense_integrator import PfSenseIntegrator + import glob + + xml_files = glob.glob("*.xml") + if xml_files: + integrator = PfSenseIntegrator(xml_files) + integrator.load_pfsense_configs() + + for pfsense_name, pfsense_config in integrator.pfsense_configs.items(): + logger.info(f"Checking LAN networks for pfSense: {pfsense_name}") + + interfaces = pfsense_config.get('interfaces', {}) + for iface_name, iface_config in interfaces.items(): + # Skip WAN interfaces and VPN interfaces + if iface_name.lower() in ['wan', 'wireguard', 'openvpn', 'ipsec']: + continue + + # Get the network for this interface + ipaddr = iface_config.get('ipaddr') + subnet = iface_config.get('subnet') + + if ipaddr and subnet: + try: + # Calculate the network from IP and subnet + import ipaddress + ip = ipaddress.IPv4Address(ipaddr) + net = ipaddress.IPv4Network(f"{ip}/{subnet}", strict=False) + network_cidr = str(net) + + logger.info(f"Scanning pfSense LAN network: {network_cidr} (interface: {iface_name})") + + # Check if this network is already scanned + existing_networks = [seg.cidr for seg in self.base_scanner.segments] + if network_cidr not in existing_networks: + # Scan this network + lan_segment = self.base_scanner.scan_network( + network_cidr, + f"{pfsense_name}_{iface_name.upper()}", + is_vpn=False + ) + self.base_scanner.segments.append(lan_segment) + logger.info(f"Added LAN network {network_cidr} with {len(lan_segment.devices)} devices") + else: + logger.info(f"Network {network_cidr} already scanned") + + except Exception as e: + logger.error(f"Error scanning pfSense LAN network {ipaddr}/{subnet}: {e}") + + except Exception as e: + logger.error(f"Error scanning pfSense LAN networks: {e}") + def _integrate_pfsense_xml(self): """Automatically integrate pfSense XML files if present""" import glob @@ -91,7 +154,9 @@ class IntegratedNetworkScanner: enhanced_data = json.load(f) # Update segments + vpn_interfaces = self._check_vpn_interfaces() self.base_scanner.segments = [] + for seg_data in enhanced_data.get('segments', []): segment = NetworkSegment( name=seg_data['name'], @@ -101,9 +166,48 @@ class IntegratedNetworkScanner: devices=[] ) - for dev_data in seg_data['devices']: - device = Device(**dev_data) - segment.devices.append(device) + # Only scan VPN networks if VPN interfaces are active + if seg_data['is_vpn']: + if vpn_interfaces: + logger.info(f"Scanning VPN network {seg_data['cidr']} (VPN interfaces active)") + # Scan the VPN network for devices + vpn_segment = self.base_scanner.scan_network( + seg_data['cidr'], + seg_data['name'], + is_vpn=True + ) + segment.devices = vpn_segment.devices + else: + logger.info(f"VPN network {seg_data['cidr']} (no active VPN interfaces)") + # Try to scan anyway if network might be reachable + try: + logger.info(f"Attempting to scan VPN network {seg_data['cidr']} anyway...") + vpn_segment = self.base_scanner.scan_network( + seg_data['cidr'], + seg_data['name'], + is_vpn=True + ) + segment.devices = vpn_segment.devices + logger.info(f"Successfully scanned VPN network {seg_data['cidr']} with {len(vpn_segment.devices)} devices") + except Exception as e: + logger.warning(f"Could not scan VPN network {seg_data['cidr']}: {e}") + # Keep any devices that were already in the segment data + for dev_data in seg_data['devices']: + device_kwargs = {k: v for k, v in dev_data.items() + if k in ['ip', 'hostname', 'mac', 'manufacturer', 'os_type', + 'os_version', 'device_type', 'open_ports', 'ssh_accessible', + 'services', 'routes', 'interfaces']} + device = Device(**device_kwargs) + segment.devices.append(device) + else: + # For non-VPN networks, use the devices from the segment data + for dev_data in seg_data['devices']: + device_kwargs = {k: v for k, v in dev_data.items() + if k in ['ip', 'hostname', 'mac', 'manufacturer', 'os_type', + 'os_version', 'device_type', 'open_ports', 'ssh_accessible', + 'services', 'routes', 'interfaces']} + device = Device(**device_kwargs) + segment.devices.append(device) self.base_scanner.segments.append(segment) @@ -304,6 +408,33 @@ class IntegratedNetworkScanner: 'client': '💻' } return icons.get(device_type, '❓') + + def _check_vpn_interfaces(self) -> dict: + """Check which VPN interfaces are active""" + vpn_interfaces = {} + + try: + # Check for WireGuard interfaces + result = subprocess.run( + ['ip', 'link', 'show'], + capture_output=True, + text=True, + timeout=5 + ) + + for line in result.stdout.splitlines(): + if 'wg' in line.lower() or 'tun' in line.lower() or 'tap' in line.lower(): + # Extract interface name (format: 123: wg0: ...) + match = re.search(r'\d+:\s+(\w+):', line) + if match: + iface = match.group(1) + vpn_interfaces[iface] = True + logger.info(f"Found active VPN interface: {iface}") + + except Exception as e: + logger.warning(f"Error checking VPN interfaces: {e}") + + return vpn_interfaces def main(): diff --git a/network_scanner.py b/src/network_scanner.py similarity index 82% rename from network_scanner.py rename to src/network_scanner.py index e2c59a8..6466339 100755 --- a/network_scanner.py +++ b/src/network_scanner.py @@ -11,8 +11,9 @@ import json import re import socket import argparse +import multiprocessing from typing import Dict, List, Optional, Tuple -from dataclasses import dataclass, asdict +from dataclasses import dataclass, asdict, field from concurrent.futures import ThreadPoolExecutor, as_completed import logging @@ -34,21 +35,11 @@ class Device: os_type: Optional[str] = None os_version: Optional[str] = None device_type: Optional[str] = None # router, switch, server, client, etc. - open_ports: List[int] = None + open_ports: List[int] = field(default_factory=list) ssh_accessible: bool = False - services: List[str] = None - routes: List[Dict] = None - interfaces: List[Dict] = None - - def __post_init__(self): - if self.open_ports is None: - self.open_ports = [] - if self.services is None: - self.services = [] - if self.routes is None: - self.routes = [] - if self.interfaces is None: - self.interfaces = [] + services: List[str] = field(default_factory=list) + routes: List[Dict] = field(default_factory=list) + interfaces: List[Dict] = field(default_factory=list) @dataclass @@ -59,11 +50,7 @@ class NetworkSegment: gateway: Optional[str] = None vlan: Optional[int] = None is_vpn: bool = False - devices: List[Device] = None - - def __post_init__(self): - if self.devices is None: - self.devices = [] + devices: List[Device] = field(default_factory=list) class NetworkScanner: @@ -114,6 +101,33 @@ class NetworkScanner: return networks + def _check_vpn_interfaces(self) -> Dict[str, bool]: + """Check which VPN interfaces are active""" + vpn_interfaces = {} + + try: + # Check for WireGuard interfaces + result = subprocess.run( + ['ip', 'link', 'show'], + capture_output=True, + text=True, + timeout=5 + ) + + for line in result.stdout.splitlines(): + if 'wg' in line.lower() or 'tun' in line.lower() or 'tap' in line.lower(): + # Extract interface name (format: 123: wg0: ...) + match = re.search(r'\d+:\s+(\w+):', line) + if match: + iface = match.group(1) + vpn_interfaces[iface] = True + logger.info(f"Found active VPN interface: {iface}") + + except Exception as e: + logger.warning(f"Error checking VPN interfaces: {e}") + + return vpn_interfaces + def ping_sweep(self, network: str) -> List[str]: """Perform ping sweep to find live hosts""" logger.info(f"Performing ping sweep on {network}") @@ -128,7 +142,10 @@ class NetworkScanner: logger.warning(f"Large network {network}, limiting scan") hosts = hosts[:254] - with ThreadPoolExecutor(max_workers=50) as executor: + max_ping_workers = min(len(hosts), multiprocessing.cpu_count() * 8) + logger.info(f"Pinging {len(hosts)} hosts using {max_ping_workers} concurrent workers") + + with ThreadPoolExecutor(max_workers=max_ping_workers) as executor: future_to_ip = { executor.submit(self._ping_host, str(ip)): str(ip) for ip in hosts @@ -222,10 +239,22 @@ class NetworkScanner: common_ports = [22, 80, 443, 8080, 8443, 3389, 445, 139, 21, 23, 25, 53, 3306, 5432] open_ports = [] - for port in common_ports: - if self._check_port(ip, port): - open_ports.append(port) - logger.debug(f"{ip}:{port} - OPEN") + max_port_workers = min(len(common_ports), multiprocessing.cpu_count() * 2) + + with ThreadPoolExecutor(max_workers=max_port_workers) as executor: + future_to_port = { + executor.submit(self._check_port, ip, port): port + for port in common_ports + } + + for future in as_completed(future_to_port): + port = future_to_port[future] + try: + if future.result(): + open_ports.append(port) + logger.debug(f"{ip}:{port} - OPEN") + except Exception as e: + logger.debug(f"Error checking {ip}:{port}: {e}") return open_ports @@ -395,7 +424,7 @@ class NetworkScanner: else: return 'client' - def scan_network(self, network: str, name: str = None, is_vpn: bool = False) -> NetworkSegment: + def scan_network(self, network: str, name: Optional[str] = None, is_vpn: bool = False) -> NetworkSegment: """Scan a complete network segment""" logger.info(f"Scanning network segment: {network}") @@ -409,7 +438,10 @@ class NetworkScanner: live_hosts = self.ping_sweep(network) # Gather device info - with ThreadPoolExecutor(max_workers=10) as executor: + max_device_workers = min(len(live_hosts), multiprocessing.cpu_count() * 2) + logger.info(f"Gathering device info for {len(live_hosts)} hosts using {max_device_workers} concurrent workers") + + with ThreadPoolExecutor(max_workers=max_device_workers) as executor: future_to_ip = { executor.submit(self.get_device_info, ip): ip for ip in live_hosts @@ -431,13 +463,24 @@ class NetworkScanner: # Discover networks networks = self.discover_networks() - # Scan each network - for network in networks: - try: - segment = self.scan_network(network) - self.segments.append(segment) - except Exception as e: - logger.error(f"Error scanning {network}: {e}") + # Scan networks concurrently + max_concurrent_networks = min(len(networks), multiprocessing.cpu_count()) + logger.info(f"Scanning {len(networks)} networks using {max_concurrent_networks} concurrent processes") + + with ThreadPoolExecutor(max_workers=max_concurrent_networks) as executor: + future_to_network = { + executor.submit(self.scan_network, network): network + for network in networks + } + + for future in as_completed(future_to_network): + network = future_to_network[future] + try: + segment = future.result() + self.segments.append(segment) + logger.info(f"Completed scanning network: {network}") + except Exception as e: + logger.error(f"Error scanning {network}: {e}") logger.info(f"Scan complete. Found {len(self.segments)} segments") diff --git a/pfsense_integrator.py b/src/pfsense_integrator.py similarity index 90% rename from pfsense_integrator.py rename to src/pfsense_integrator.py index 6cf9be7..3d6e28a 100755 --- a/pfsense_integrator.py +++ b/src/pfsense_integrator.py @@ -82,7 +82,12 @@ class PfSenseIntegrator: # Look for LAN interface lan_interface = interfaces.get('lan') if lan_interface and lan_interface.get('ipaddr') and lan_interface.get('subnet'): - lan_network = f"{lan_interface['ipaddr']}/{lan_interface['subnet']}" + # Calculate the proper network address + import ipaddress + ip = ipaddress.IPv4Address(lan_interface['ipaddr']) + subnet_bits = int(lan_interface['subnet']) + network = ipaddress.IPv4Network(f"{ip}/{subnet_bits}", strict=False) + lan_network = str(network) # Find existing segment for segment in segments: @@ -213,6 +218,11 @@ class PfSenseIntegrator: network_cidr = iface_config.get('network_cidr') if network_cidr and network_cidr != 'unknown': + # Filter out invalid networks + if self._is_invalid_network(network_cidr): + logger.warning(f"Skipping invalid interface network: {network_cidr}") + continue + # Check if segment already exists segment_exists = any(seg.get('cidr') == network_cidr for seg in segments) @@ -233,6 +243,12 @@ class PfSenseIntegrator: for peer in wireguard.get('peers', []): for allowed_ip in peer.get('allowed_ips', []): network = f"{allowed_ip['address']}/{allowed_ip['mask']}" + + # Filter out invalid networks + if self._is_invalid_network(network): + logger.warning(f"Skipping invalid WireGuard network: {network}") + continue + segment_exists = any(seg.get('cidr') == network for seg in segments) if not segment_exists: @@ -246,6 +262,26 @@ class PfSenseIntegrator: segments.append(new_segment) logger.info(f"Added WireGuard network: {network}") + def _is_invalid_network(self, network: str) -> bool: + """Check if a network should not be scanned""" + try: + import ipaddress + net = ipaddress.ip_network(network, strict=False) + + # Skip networks that are too large or invalid + if net.prefixlen == 0: # 0.0.0.0/0 - route all traffic + return True + if net.prefixlen < 8: # Very large networks + return True + if net.network_address.is_private and net.prefixlen < 16: # Large private networks + return True + if str(net.network_address) == '0.0.0.0': # Invalid network + return True + + return False + except ValueError: + return True # Invalid network format + def generate_network_summary(self, output_file: str): """Generate a human-readable network summary""" summary = [] diff --git a/pfsense_scanner.py b/src/pfsense_scanner.py similarity index 100% rename from pfsense_scanner.py rename to src/pfsense_scanner.py diff --git a/pfsense_xml_parser.py b/src/pfsense_xml_parser.py similarity index 100% rename from pfsense_xml_parser.py rename to src/pfsense_xml_parser.py diff --git a/svg_generator.py b/src/svg_generator.py similarity index 100% rename from svg_generator.py rename to src/svg_generator.py diff --git a/test_system.py b/src/test_system.py similarity index 95% rename from test_system.py rename to src/test_system.py index 63b5e2f..eb211bd 100755 --- a/test_system.py +++ b/src/test_system.py @@ -56,11 +56,11 @@ def test_commands(): def test_scripts_exist(): """Check if all scripts exist""" scripts = [ - 'network_scanner.py', - 'pfsense_scanner.py', - 'svg_generator.py', - 'integrated_scanner.py', - 'quickstart.sh' + 'src/network_scanner.py', + 'src/pfsense_scanner.py', + 'src/svg_generator.py', + 'src/integrated_scanner.py', + 'scripts/quickstart.sh' ] all_ok = True @@ -77,10 +77,10 @@ def test_scripts_exist(): def test_script_syntax(): """Test Python script syntax""" scripts = [ - 'network_scanner.py', - 'pfsense_scanner.py', - 'svg_generator.py', - 'integrated_scanner.py' + 'src/network_scanner.py', + 'src/pfsense_scanner.py', + 'src/svg_generator.py', + 'src/integrated_scanner.py' ] all_ok = True