diff --git a/README.md b/README.md index 5fb2a5c..18cda19 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ A comprehensive network topology discovery tool that scans local and VPN-connect - šŸ“Š **SVG Diagram Generation**: Creates visual network topology diagrams - šŸ”„ **Routing Analysis**: Extracts and analyzes routing tables from routers - šŸ“ **JSON Export**: Structured data output for further processing +- šŸ“„ **pfSense XML Parsing**: Automatically parses pfSense backup XML files for complete configuration analysis +- šŸ”— **Automatic Integration**: Seamlessly integrates pfSense XML data into network scans ## Requirements @@ -205,6 +207,23 @@ The scanner produces JSON with the following structure: firefox network_topology.svg ``` +### Complete Automated Workflow + +For the ultimate network discovery experience, use the automated workflow script: + +```bash +./complete_workflow.sh +``` + +This script will: +1. āœ… Verify system requirements +2. šŸ” Run integrated network scan (including pfSense XML if present) +3. šŸŽØ Generate SVG network diagram +4. šŸ“‹ Create network summary (if pfSense XML files exist) +5. šŸ“Š Display statistics and next steps + +**One-command network discovery!** + ## SSH Access Setup For automated scanning, SSH key-based authentication is recommended: @@ -362,3 +381,27 @@ Created for comprehensive network topology discovery and visualization. --- **Note**: Always ensure you have proper authorization before scanning networks. This tool performs active network reconnaissance. + +### pfSense XML Integration + +The scanner can automatically parse pfSense backup XML files to extract comprehensive configuration data: + +- **Network Interfaces**: All interface configurations, IP addresses, VLANs +- **Routing Tables**: Static routes, dynamic routing, gateway configurations +- **VPN Configurations**: WireGuard tunnels, OpenVPN servers/clients, IPsec +- **Firewall Rules**: NAT rules, port forwarding, access control lists +- **DHCP Services**: Server configurations, static mappings, lease pools +- **DNS Settings**: Resolvers, domain configurations +- **System Information**: Hostname, domain, version, services + +**Automatic Integration**: Place pfSense XML backup files in the scanner directory, and they will be automatically parsed and integrated into network scans. + +**Manual Parsing**: Use `pfsense_integrator.py` to work with XML files independently: + +```bash +# Parse XML files and generate summary +./pfsense_integrator.py *.xml --summary network_summary.md + +# Integrate with existing scan +./pfsense_integrator.py *.xml -s scan.json -o enhanced_scan.json +``` diff --git a/complete_workflow.sh b/complete_workflow.sh new file mode 100755 index 0000000..2af9815 --- /dev/null +++ b/complete_workflow.sh @@ -0,0 +1,124 @@ +#!/bin/bash +# Complete Network Discovery Workflow +# Automatically scans network, integrates pfSense XML, and generates diagrams + +set -e + +echo "==========================================" +echo "Complete Network Discovery Workflow" +echo "==========================================" +echo "" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Check if we're in the right directory +if [ ! -f "integrated_scanner.py" ]; then + log_error "integrated_scanner.py not found. Please run this script from the network scanner directory." + exit 1 +fi + +# Check for pfSense XML files +XML_FILES=$(ls *.xml 2>/dev/null | wc -l) +if [ "$XML_FILES" -gt 0 ]; then + log_info "Found $XML_FILES pfSense XML configuration file(s)" +else + log_warning "No pfSense XML files found. Network scan will proceed without pfSense integration." +fi + +# Step 1: Run system verification +log_info "Step 1: Verifying system requirements..." +if ./test_system.py >/dev/null 2>&1; then + log_success "System verification passed" +else + log_error "System verification failed. Please check the output above." + exit 1 +fi + +# Step 2: Run integrated network scan +log_info "Step 2: Running integrated network scan..." +SCAN_OUTPUT="network_scan_$(date +%Y%m%d_%H%M%S).json" +if ./integrated_scanner.py -o "$SCAN_OUTPUT" -v; then + log_success "Network scan completed: $SCAN_OUTPUT" +else + log_error "Network scan failed" + exit 1 +fi + +# Step 3: Generate SVG diagram +log_info "Step 3: Generating network diagram..." +SVG_OUTPUT="${SCAN_OUTPUT%.json}.svg" +if ./svg_generator.py "$SCAN_OUTPUT" -o "$SVG_OUTPUT"; then + log_success "SVG diagram generated: $SVG_OUTPUT" +else + log_error "SVG generation failed" + exit 1 +fi + +# Step 4: Generate pfSense summary if XML files exist +if [ "$XML_FILES" -gt 0 ]; then + log_info "Step 4: Generating pfSense network summary..." + SUMMARY_OUTPUT="network_summary_$(date +%Y%m%d_%H%M%S).md" + if ./pfsense_integrator.py *.xml --summary "$SUMMARY_OUTPUT"; then + log_success "Network summary generated: $SUMMARY_OUTPUT" + else + log_warning "Network summary generation failed" + fi +fi + +# Step 5: Show results summary +echo "" +echo "==========================================" +log_success "Network Discovery Complete!" +echo "==========================================" +echo "" +echo "Generated files:" +echo " šŸ“Š Network Scan: $SCAN_OUTPUT" +echo " šŸŽØ Network Diagram: $SVG_OUTPUT" +if [ "$XML_FILES" -gt 0 ]; then + echo " šŸ“‹ Network Summary: $SUMMARY_OUTPUT" +fi +echo "" + +# Show network statistics +if command -v jq >/dev/null 2>&1; then + echo "Network Statistics:" + TOTAL_SEGMENTS=$(jq '.segments | length' "$SCAN_OUTPUT") + TOTAL_DEVICES=$(jq '[.segments[].devices[]] | length' "$SCAN_OUTPUT") + PFSENSE_DEVICES=$(jq '[.segments[].devices[] | select(.device_type=="firewall")] | length' "$SCAN_OUTPUT") + + echo " šŸ“” Network Segments: $TOTAL_SEGMENTS" + echo " šŸ–„ļø Total Devices: $TOTAL_DEVICES" + echo " šŸ›”ļø pfSense Firewalls: $PFSENSE_DEVICES" + echo "" +fi + +echo "Next steps:" +echo " 1. Open $SVG_OUTPUT in your web browser to view the network diagram" +if [ "$XML_FILES" -gt 0 ]; then + echo " 2. Review $SUMMARY_OUTPUT for detailed pfSense configuration" +fi +echo " 3. Examine $SCAN_OUTPUT for complete network data (use jq for querying)" +echo "" + +log_success "Workflow completed successfully! šŸŽ‰" \ No newline at end of file diff --git a/integrated_scanner.py b/integrated_scanner.py index fdb94b2..454cf97 100755 --- a/integrated_scanner.py +++ b/integrated_scanner.py @@ -10,6 +10,8 @@ import argparse from datetime import datetime from network_scanner import NetworkScanner, NetworkSegment from pfsense_scanner import PfSenseScanner +from dataclasses import asdict +from network_scanner import Device logging.basicConfig( level=logging.INFO, @@ -33,11 +35,91 @@ class IntegratedNetworkScanner: # Run base network scan self.base_scanner.scan_all() + # Check for pfSense XML files and integrate them + self._integrate_pfsense_xml() + # Identify and enhance pfSense devices self._scan_pfsense_devices() logger.info("Integrated scan complete") + def _integrate_pfsense_xml(self): + """Automatically integrate pfSense XML files if present""" + import glob + from pfsense_integrator import PfSenseIntegrator + + # Look for XML files in current directory + xml_files = glob.glob("*.xml") + if not xml_files: + logger.info("No pfSense XML files found, skipping XML integration") + return + + logger.info(f"Found {len(xml_files)} pfSense XML files, integrating...") + + try: + integrator = PfSenseIntegrator(xml_files) + integrator.load_pfsense_configs() + + # Create temporary scan file for integration + import tempfile + import os + + temp_scan = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + try: + # Export current scan data + temp_data = { + 'scan_timestamp': None, + 'segments': [ + { + 'name': seg.name, + 'cidr': seg.cidr, + 'gateway': seg.gateway, + 'is_vpn': seg.is_vpn, + 'devices': [asdict(dev) for dev in seg.devices] + } + for seg in self.base_scanner.segments + ] + } + json.dump(temp_data, temp_scan) + temp_scan.close() + + # Integrate pfSense data + integrator.integrate_with_scan(temp_scan.name, temp_scan.name + '_enhanced') + + # Load enhanced data back + with open(temp_scan.name + '_enhanced', 'r') as f: + enhanced_data = json.load(f) + + # Update segments + self.base_scanner.segments = [] + for seg_data in enhanced_data.get('segments', []): + segment = NetworkSegment( + name=seg_data['name'], + cidr=seg_data['cidr'], + gateway=seg_data['gateway'], + is_vpn=seg_data['is_vpn'], + devices=[] + ) + + for dev_data in seg_data['devices']: + device = Device(**dev_data) + segment.devices.append(device) + + self.base_scanner.segments.append(segment) + + logger.info(f"Integrated {len(integrator.pfsense_configs)} pfSense configurations") + + finally: + # Clean up temp files + try: + os.unlink(temp_scan.name) + os.unlink(temp_scan.name + '_enhanced') + except: + pass + + except Exception as e: + logger.error(f"Error integrating pfSense XML: {e}") + def _scan_pfsense_devices(self): """Find and deeply scan pfSense devices""" logger.info("Looking for pfSense devices...") diff --git a/network_summary.md b/network_summary.md new file mode 100644 index 0000000..29b3e92 --- /dev/null +++ b/network_summary.md @@ -0,0 +1,53 @@ +# Network Topology Summary +Generated from pfSense XML configurations + +## pfSense Firewall: gw-nue01 +**Version:** unknown +**Domain:** egonetix.lan + +### Network Interfaces +- **WAN** (wan): dhcp +- **LAN** (lan): 10.0.0.1 +- **wireguardnachhause** (opt1): 10.69.69.1 + - Gateway: WirusguardusGW + +### Static Routes +- 172.20.0.0/16 via WirusguardusGW + *heyme* + +### WireGuard VPN +- **Tunnel tun_wg0** (Port 51820) + *heyme* + - Peer: wireguardheyme - Networks: 172.20.0.0/16, 10.69.69.2/32 + +### DHCP Configuration + +## pfSense Firewall: gw-st01 +**Version:** unknown +**Domain:** egonetix.lan + +### Network Interfaces +- **WAN** (wan): 192.168.178.3 + - Gateway: WANGW +- **LAN** (lan): 172.20.20.1 +- **wireguardnnbesch** (opt1): 10.69.69.2 + - Gateway: wirenuenbesch +- **HomeAssistant** (opt2): 172.20.70.1 +- **WireguardOpenvpn** (opt3): 10.5.0.2 + - Gateway: WireguardOpenvpnGW + +### Static Routes +- 10.0.0.0/24 via wirenuenbesch + *wireguardnünbesch* +- 12.1.0.0/24 via wirenuenbesch + *openvpn nutzer* + +### WireGuard VPN +- **Tunnel tun_wg0** (Port 51820) + *de1099.nordvpn.com* +- **Tunnel tun_wg1** (Port 51821) + *wireguardnünbesch* + - Peer: de1099.nordvpn.com - Networks: 0.0.0.0/0 + - Peer: wireguardnünbesch - Networks: 10.0.0.0/24, 10.69.69.1/32, 12.1.0.0/24 + +### DHCP Configuration diff --git a/pfsense_integrator.py b/pfsense_integrator.py new file mode 100755 index 0000000..6cf9be7 --- /dev/null +++ b/pfsense_integrator.py @@ -0,0 +1,380 @@ +#!/usr/bin/env python3 +""" +pfSense XML Integration Script +Automatically processes pfSense XML backup files and integrates them into network scan results +""" + +import json +import argparse +from pathlib import Path +import logging +from typing import Dict, List, Any +from pfsense_xml_parser import PfSenseXMLParser + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class PfSenseIntegrator: + """Integrates pfSense XML data into network scan results""" + + def __init__(self, xml_files: List[str]): + self.xml_files = xml_files + self.pfsense_configs = {} + + def load_pfsense_configs(self): + """Load and parse all pfSense XML files""" + logger.info(f"Loading {len(self.xml_files)} pfSense configuration files...") + + for xml_file in self.xml_files: + try: + parser = PfSenseXMLParser(xml_file) + if parser.load_xml(): + config = parser.parse_all() + hostname = config.get('system', {}).get('hostname', 'unknown') + self.pfsense_configs[hostname] = config + logger.info(f"Loaded pfSense config: {hostname}") + else: + logger.error(f"Failed to load: {xml_file}") + except Exception as e: + logger.error(f"Error parsing {xml_file}: {e}") + + def integrate_with_scan(self, scan_file: str, output_file: str): + """Integrate pfSense data into network scan results""" + logger.info(f"Integrating pfSense data into {scan_file}") + + # Load network scan + with open(scan_file, 'r') as f: + scan_data = json.load(f) + + # Add pfSense configurations + scan_data['pfsense_configs'] = self.pfsense_configs + + # Enhance network segments with pfSense data + self._enhance_segments_with_pfsense(scan_data) + + # Save enhanced scan + with open(output_file, 'w') as f: + json.dump(scan_data, f, indent=2) + + logger.info(f"Enhanced scan saved to {output_file}") + + def _enhance_segments_with_pfsense(self, scan_data: Dict): + """Enhance network segments with pfSense-specific information""" + segments = scan_data.get('segments', []) + + for pfsense_name, pfsense_config in self.pfsense_configs.items(): + logger.info(f"Processing pfSense: {pfsense_name}") + + # Find or create segment for this pfSense + pfsense_segment = self._find_or_create_pfsense_segment(segments, pfsense_config) + + # Add pfSense device if not already present + pfsense_device = self._add_pfsense_device(pfsense_segment, pfsense_config) + + # Add networks discovered from pfSense config + self._add_pfsense_networks(segments, pfsense_config) + + def _find_or_create_pfsense_segment(self, segments: List[Dict], pfsense_config: Dict) -> Dict: + """Find existing segment or create new one for pfSense""" + interfaces = pfsense_config.get('interfaces', {}) + + # Look for LAN interface + lan_interface = interfaces.get('lan') + if lan_interface and lan_interface.get('ipaddr') and lan_interface.get('subnet'): + lan_network = f"{lan_interface['ipaddr']}/{lan_interface['subnet']}" + + # Find existing segment + for segment in segments: + if segment.get('cidr') == lan_network: + return segment + + # Create new segment + new_segment = { + 'name': f"{pfsense_config['system']['hostname']}_LAN", + 'cidr': lan_network, + 'gateway': lan_interface.get('ipaddr'), + 'is_vpn': False, + 'devices': [] + } + segments.append(new_segment) + return new_segment + + # Fallback: create segment with hostname + hostname = pfsense_config['system']['hostname'] + for segment in segments: + if segment.get('name', '').startswith(hostname): + return segment + + new_segment = { + 'name': f"{hostname}_networks", + 'cidr': 'unknown', + 'gateway': None, + 'is_vpn': False, + 'devices': [] + } + segments.append(new_segment) + return new_segment + + def _add_pfsense_device(self, segment: Dict, pfsense_config: Dict) -> Dict: + """Add pfSense device to segment""" + hostname = pfsense_config['system']['hostname'] + domain = pfsense_config['system'].get('domain', '') + + # Find LAN interface for IP + interfaces = pfsense_config.get('interfaces', {}) + lan_interface = interfaces.get('lan') + pfsense_ip = lan_interface.get('ipaddr') if lan_interface else 'unknown' + + # Create pfSense device + device = { + 'ip': pfsense_ip, + 'hostname': f"{hostname}.{domain}" if domain else hostname, + 'mac': None, + 'manufacturer': None, + 'os_type': 'pfSense (FreeBSD)', + 'os_version': pfsense_config.get('version', 'unknown'), + 'device_type': 'firewall', + 'open_ports': [22, 80, 443], # Common pfSense ports + 'ssh_accessible': False, # Assume not accessible for security + 'services': ['pfSense Firewall', 'Web GUI', 'SSH'], + 'routes': self._extract_routes(pfsense_config), + 'interfaces': self._extract_interfaces(pfsense_config), + 'pfsense_config': pfsense_config + } + + # Check if device already exists + for existing_device in segment['devices']: + if existing_device.get('ip') == pfsense_ip: + # Update existing device with pfSense info + existing_device.update(device) + return existing_device + + # Add new device + segment['devices'].append(device) + return device + + def _extract_routes(self, pfsense_config: Dict) -> List[Dict]: + """Extract routing information from pfSense config""" + routes = [] + + # Static routes + static_routes = pfsense_config.get('static_routes', []) + for route in static_routes: + routes.append({ + 'destination': route.get('network', ''), + 'gateway': route.get('gateway', ''), + 'interface': 'static', + 'description': route.get('description', ''), + 'type': 'static' + }) + + # Default route from gateways + gateways = pfsense_config.get('gateways', {}) + for gw_name, gw_config in gateways.items(): + if gw_config.get('defaultgw'): + routes.append({ + 'destination': 'default', + 'gateway': gw_config.get('gateway', ''), + 'interface': gw_config.get('interface', ''), + 'description': f"Default gateway: {gw_name}", + 'type': 'default' + }) + + return routes + + def _extract_interfaces(self, pfsense_config: Dict) -> List[Dict]: + """Extract interface information""" + interfaces = [] + + for iface_name, iface_config in pfsense_config.get('interfaces', {}).items(): + interface = { + 'name': iface_name, + 'description': iface_config.get('description', ''), + 'physical_interface': iface_config.get('interface', ''), + 'ip_address': iface_config.get('ipaddr', ''), + 'subnet': iface_config.get('subnet', ''), + 'network_cidr': iface_config.get('network_cidr', ''), + 'gateway': iface_config.get('gateway', ''), + 'mtu': iface_config.get('mtu', ''), + 'type': iface_config.get('type', 'physical') + } + interfaces.append(interface) + + return interfaces + + def _add_pfsense_networks(self, segments: List[Dict], pfsense_config: Dict): + """Add networks discovered from pfSense configuration""" + interfaces = pfsense_config.get('interfaces', {}) + + for iface_name, iface_config in interfaces.items(): + if iface_name == 'wan': + continue # Skip WAN for now + + network_cidr = iface_config.get('network_cidr') + if network_cidr and network_cidr != 'unknown': + # Check if segment already exists + segment_exists = any(seg.get('cidr') == network_cidr for seg in segments) + + if not segment_exists: + segment_name = f"{pfsense_config['system']['hostname']}_{iface_name.upper()}" + new_segment = { + 'name': segment_name, + 'cidr': network_cidr, + 'gateway': iface_config.get('ipaddr'), + 'is_vpn': iface_name.startswith('opt') and 'wireguard' in iface_config.get('description', '').lower(), + 'devices': [] + } + segments.append(new_segment) + logger.info(f"Added network segment: {network_cidr}") + + # Add WireGuard networks + wireguard = pfsense_config.get('wireguard', {}) + for peer in wireguard.get('peers', []): + for allowed_ip in peer.get('allowed_ips', []): + network = f"{allowed_ip['address']}/{allowed_ip['mask']}" + segment_exists = any(seg.get('cidr') == network for seg in segments) + + if not segment_exists: + new_segment = { + 'name': f"WireGuard_{peer.get('description', 'unknown').replace(' ', '_')}", + 'cidr': network, + 'gateway': None, + 'is_vpn': True, + 'devices': [] + } + segments.append(new_segment) + logger.info(f"Added WireGuard network: {network}") + + def generate_network_summary(self, output_file: str): + """Generate a human-readable network summary""" + summary = [] + summary.append("# Network Topology Summary") + summary.append("Generated from pfSense XML configurations\n") + + for pfsense_name, config in self.pfsense_configs.items(): + summary.append(f"## pfSense Firewall: {pfsense_name}") + summary.append(f"**Version:** {config.get('version', 'unknown')}") + summary.append(f"**Domain:** {config.get('system', {}).get('domain', 'unknown')}\n") + + # Interfaces + interfaces = config.get('interfaces', {}) + if interfaces: + summary.append("### Network Interfaces") + for iface_name, iface in interfaces.items(): + ip = iface.get('ipaddr', 'DHCP') + subnet = iface.get('subnet', '') + network = iface.get('network_cidr', '') + desc = iface.get('description', iface_name.upper()) + + summary.append(f"- **{desc}** ({iface_name}): {ip}") + if network: + summary.append(f" - Network: {network}") + if iface.get('gateway'): + summary.append(f" - Gateway: {iface.get('gateway')}") + summary.append("") + + # Static Routes + routes = config.get('static_routes', []) + if routes: + summary.append("### Static Routes") + for route in routes: + network = route.get('network', '') + gateway = route.get('gateway', '') + desc = route.get('description', '') + summary.append(f"- {network} via {gateway}") + if desc: + summary.append(f" *{desc}*") + summary.append("") + + # WireGuard + wg = config.get('wireguard', {}) + if wg.get('enabled') and wg.get('tunnels'): + summary.append("### WireGuard VPN") + for tunnel in wg.get('tunnels', []): + name = tunnel.get('name', 'unknown') + port = tunnel.get('listenport', 'unknown') + desc = tunnel.get('description', '') + summary.append(f"- **Tunnel {name}** (Port {port})") + if desc: + summary.append(f" *{desc}*") + + for peer in wg.get('peers', []): + desc = peer.get('description', 'unknown') + allowed_ips = peer.get('allowed_ips', []) + if allowed_ips: + networks = [f"{ip['address']}/{ip['mask']}" for ip in allowed_ips] + summary.append(f" - Peer: {desc} - Networks: {', '.join(networks)}") + summary.append("") + + # DHCP + dhcp = config.get('dhcp', {}) + if dhcp: + summary.append("### DHCP Configuration") + for iface_name, dhcp_config in dhcp.items(): + if dhcp_config.get('enabled', True): + range_from = dhcp_config.get('range_from', '') + range_to = dhcp_config.get('range_to', '') + if range_from and range_to: + summary.append(f"- **{iface_name.upper()}**: {range_from} - {range_to}") + + static_maps = dhcp_config.get('static_maps', []) + if static_maps: + summary.append(" **Static Mappings:**") + for static in static_maps: + ip = static.get('ipaddr', '') + mac = static.get('mac', '') + hostname = static.get('hostname', '') + summary.append(f" - {ip} ({mac}) - {hostname}") + summary.append("") + + # Write summary + with open(output_file, 'w') as f: + f.write('\n'.join(summary)) + + logger.info(f"Network summary saved to {output_file}") + + +def main(): + """Command line interface""" + parser = argparse.ArgumentParser(description='Integrate pfSense XML configs with network scan') + parser.add_argument('xml_files', nargs='+', help='pfSense XML configuration files') + parser.add_argument('-s', '--scan', help='Network scan JSON file to enhance') + parser.add_argument('-o', '--output', help='Output enhanced scan file') + parser.add_argument('--summary', help='Generate network summary markdown file') + parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output') + + args = parser.parse_args() + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + # Initialize integrator + integrator = PfSenseIntegrator(args.xml_files) + integrator.load_pfsense_configs() + + if not integrator.pfsense_configs: + logger.error("No pfSense configurations loaded!") + return 1 + + # Generate summary if requested + if args.summary: + integrator.generate_network_summary(args.summary) + print(f"āœ… Network summary generated: {args.summary}") + + # Integrate with scan if provided + if args.scan: + output_file = args.output or args.scan.replace('.json', '_enhanced.json') + integrator.integrate_with_scan(args.scan, output_file) + print(f"āœ… Enhanced scan saved: {output_file}") + + # Show summary + print(f"\nšŸ“Š Loaded {len(integrator.pfsense_configs)} pfSense configurations:") + for name in integrator.pfsense_configs.keys(): + print(f" - {name}") + + return 0 + + +if __name__ == '__main__': + exit(main())