From afe8903454ea02ac866f20666ac3aed83a8f92cd Mon Sep 17 00:00:00 2001 From: mindesbunister Date: Fri, 10 Oct 2025 11:14:37 +0200 Subject: [PATCH] Add comprehensive network mapper and workflow script - comprehensive_mapper.py: Combines network scanning with pfSense XML parsing - run_network_mapping.sh: Complete workflow script for network discovery - Successfully tested with both pfSense XML files and live network scan - Generates comprehensive JSON data and SVG network diagrams - Includes WireGuard VPN topology, static routes, and DHCP mappings --- comprehensive_mapper.py | 355 ++++++++++++++++++++++++++++++++++++++++ network_report.md | 14 ++ run_network_mapping.sh | 214 ++++++++++++++++++++++++ 3 files changed, 583 insertions(+) create mode 100755 comprehensive_mapper.py create mode 100644 network_report.md create mode 100755 run_network_mapping.sh diff --git a/comprehensive_mapper.py b/comprehensive_mapper.py new file mode 100755 index 0000000..0e4917f --- /dev/null +++ b/comprehensive_mapper.py @@ -0,0 +1,355 @@ +#!/usr/bin/env python3 +""" +Comprehensive Network Diagram Generator +Combines network scanning with pfSense XML parsing for complete network topology +""" + +import json +import argparse +import logging +from pathlib import Path +from typing import Dict, List, Any +from network_scanner import NetworkScanner +from pfsense_xml_parser import PfSenseXMLParser +from svg_generator import NetworkDiagramGenerator + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + + +class ComprehensiveNetworkMapper: + """Complete network mapping solution combining multiple data sources""" + + def __init__(self, config: Dict): + self.config = config + self.network_data = {} + self.pfsense_data = {} + self.combined_data = { + 'scan_timestamp': None, + 'data_sources': [], + 'segments': [], + 'pfsense_firewalls': [], + 'wireguard_networks': [], + 'openvpn_networks': [], + 'routing_table': [], + 'dhcp_leases': [], + 'static_mappings': [] + } + + def load_network_scan(self, scan_file: str): + """Load network scan data""" + logger.info(f"Loading network scan data from {scan_file}") + try: + with open(scan_file, 'r') as f: + self.network_data = json.load(f) + self.combined_data['data_sources'].append('network_scan') + logger.info("Network scan data loaded successfully") + except Exception as e: + logger.error(f"Error loading network scan: {e}") + + def load_pfsense_configs(self, xml_files: List[str]): + """Load and parse pfSense XML configuration files""" + for xml_file in xml_files: + if Path(xml_file).exists(): + logger.info(f"Parsing pfSense config: {xml_file}") + parser = PfSenseXMLParser(xml_file) + data = parser.parse_all() + + if data: + hostname = data.get('hostname', f'pfsense_{len(self.pfsense_data)}') + self.pfsense_data[hostname] = data + self.combined_data['data_sources'].append(f'pfsense_{hostname}') + self.combined_data['pfsense_firewalls'].append(data) + logger.info(f"pfSense config {hostname} loaded successfully") + else: + logger.error(f"Failed to parse pfSense config: {xml_file}") + else: + logger.warning(f"pfSense config file not found: {xml_file}") + + def merge_network_data(self): + """Merge all data sources into comprehensive network map""" + logger.info("Merging network data sources...") + + # Start with network scan data if available + if self.network_data: + self.combined_data.update({ + 'scan_timestamp': self.network_data.get('scan_timestamp'), + 'segments': self.network_data.get('segments', []) + }) + + # Process pfSense data + self._process_pfsense_data() + + # Extract additional network information + self._extract_network_topology() + + logger.info("Network data merging complete") + + def _process_pfsense_data(self): + """Process and integrate pfSense configuration data""" + for hostname, pfsense in self.pfsense_data.items(): + logger.info(f"Processing pfSense data for {hostname}") + + # Add interfaces as network segments + interfaces = pfsense.get('interfaces', {}) + for iface_name, iface_data in interfaces.items(): + if iface_data.get('ipaddr') and iface_data.get('ipaddr') != 'dhcp': + try: + # Create network segment from interface + ip = iface_data['ipaddr'] + subnet = iface_data.get('subnet', '24') + network_cidr = f"{ip.rsplit('.', 1)[0]}.0/{subnet}" + + segment = { + 'name': f"{hostname}_{iface_name}", + 'cidr': network_cidr, + 'gateway': iface_data.get('gateway', ip), + 'interface': iface_data.get('interface'), + 'pfsense_host': hostname, + 'is_vpn': iface_data.get('type') in ['wireguard', 'openvpn'], + 'devices': [] + } + + # Add gateway device + gateway_device = { + 'ip': ip, + 'hostname': hostname, + 'device_type': 'firewall', + 'os_type': 'pfSense', + 'interface': iface_name, + 'pfsense_config': True + } + segment['devices'].append(gateway_device) + + self.combined_data['segments'].append(segment) + + # Track VPN networks + if iface_data.get('type') == 'wireguard': + self.combined_data['wireguard_networks'].append({ + 'network': network_cidr, + 'interface': iface_data.get('interface'), + 'pfsense': hostname, + 'peers': pfsense.get('wireguard', {}).get('peers', []) + }) + elif iface_data.get('type') == 'openvpn': + self.combined_data['openvpn_networks'].append({ + 'network': network_cidr, + 'interface': iface_data.get('interface'), + 'pfsense': hostname + }) + + except Exception as e: + logger.debug(f"Error processing interface {iface_name}: {e}") + + # Add static routes + static_routes = pfsense.get('static_routes', []) + for route in static_routes: + route_info = { + 'network': route.get('network'), + 'gateway': route.get('gateway'), + 'description': route.get('description'), + 'pfsense': hostname + } + self.combined_data['routing_table'].append(route_info) + + # Add DHCP information + dhcp_config = pfsense.get('dhcp', {}) + for iface_name, dhcp_data in dhcp_config.items(): + if dhcp_data.get('static_mappings'): + for mapping in dhcp_data['static_mappings']: + mapping_info = { + 'ip': mapping.get('ipaddr'), + 'mac': mapping.get('mac'), + 'hostname': mapping.get('hostname'), + 'description': mapping.get('description'), + 'interface': iface_name, + 'pfsense': hostname + } + self.combined_data['static_mappings'].append(mapping_info) + + def _extract_network_topology(self): + """Extract network topology information from all sources""" + logger.info("Extracting network topology...") + + # Process WireGuard peer information + for wg_net in self.combined_data['wireguard_networks']: + for peer in wg_net.get('peers', []): + for allowed_ip in peer.get('allowed_ips', []): + # Create segment for remote networks + if allowed_ip.get('mask') and allowed_ip['mask'] != '32': + remote_segment = { + 'name': f"WG_Remote_{allowed_ip['address']}_{allowed_ip['mask']}", + 'cidr': f"{allowed_ip['address']}/{allowed_ip['mask']}", + 'gateway': wg_net.get('network', '').split('/')[0].rsplit('.', 1)[0] + '.1', + 'is_vpn': True, + 'vpn_type': 'wireguard', + 'pfsense_host': wg_net.get('pfsense'), + 'devices': [] + } + self.combined_data['segments'].append(remote_segment) + + def generate_svg_diagram(self, output_file: str): + """Generate SVG network diagram""" + logger.info(f"Generating SVG diagram: {output_file}") + + # Use the existing SVG generator + generator = NetworkDiagramGenerator(self.combined_data) + generator.generate_svg(output_file) + + logger.info(f"SVG diagram generated: {output_file}") + + def export_comprehensive_json(self, output_file: str): + """Export comprehensive network data""" + logger.info(f"Exporting comprehensive data to {output_file}") + + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(self.combined_data, f, indent=2, ensure_ascii=False) + + logger.info(f"Comprehensive network data exported to {output_file}") + + def print_summary(self): + """Print comprehensive network summary""" + print("\n" + "="*80) + print("COMPREHENSIVE NETWORK MAPPING SUMMARY") + print("="*80) + + print(f"\nData Sources: {', '.join(self.combined_data['data_sources'])}") + print(f"Network Segments: {len(self.combined_data['segments'])}") + print(f"pfSense Firewalls: {len(self.combined_data['pfsense_firewalls'])}") + print(f"WireGuard Networks: {len(self.combined_data['wireguard_networks'])}") + print(f"OpenVPN Networks: {len(self.combined_data['openvpn_networks'])}") + print(f"Static Routes: {len(self.combined_data['routing_table'])}") + print(f"DHCP Static Mappings: {len(self.combined_data['static_mappings'])}") + + # Network segments summary + print(f"\n{'='*80}") + print("NETWORK SEGMENTS") + print(f"{'='*80}") + + for segment in self.combined_data['segments']: + vpn_indicator = " (VPN)" if segment.get('is_vpn') else "" + pfsense_indicator = f" [{segment.get('pfsense_host')}]" if segment.get('pfsense_host') else "" + print(f"\nšŸ“” {segment['name']}{vpn_indicator}{pfsense_indicator}") + print(f" CIDR: {segment['cidr']}") + print(f" Gateway: {segment.get('gateway', 'N/A')}") + print(f" Devices: {len(segment.get('devices', []))}") + + # Show devices + for device in segment.get('devices', [])[:5]: # Show first 5 + print(f" ā€¢ {device.get('ip', 'N/A')} - {device.get('hostname', 'N/A')} ({device.get('device_type', 'N/A')})") + + if len(segment.get('devices', [])) > 5: + print(f" ... and {len(segment.get('devices', [])) - 5} more devices") + + # pfSense summary + if self.combined_data['pfsense_firewalls']: + print(f"\n{'='*80}") + print("PFSENSE FIREWALLS") + print(f"{'='*80}") + + for pfsense in self.combined_data['pfsense_firewalls']: + hostname = pfsense.get('hostname', 'Unknown') + print(f"\nšŸ›”ļø {hostname}") + print(f" Interfaces: {len(pfsense.get('interfaces', {}))}") + print(f" Static Routes: {len(pfsense.get('static_routes', []))}") + + wg_config = pfsense.get('wireguard', {}) + if wg_config.get('enabled'): + print(f" WireGuard: {len(wg_config.get('tunnels', []))} tunnels, {len(wg_config.get('peers', []))} peers") + + dhcp_config = pfsense.get('dhcp', {}) + total_mappings = sum(len(dhcp.get('static_mappings', [])) for dhcp in dhcp_config.values()) + print(f" DHCP Static Mappings: {total_mappings}") + + # VPN Networks + if self.combined_data['wireguard_networks'] or self.combined_data['openvpn_networks']: + print(f"\n{'='*80}") + print("VPN NETWORKS") + print(f"{'='*80}") + + for wg_net in self.combined_data['wireguard_networks']: + print(f"\nšŸ” WireGuard: {wg_net['network']} via {wg_net['pfsense']}") + print(f" Interface: {wg_net['interface']}") + print(f" Peers: {len(wg_net.get('peers', []))}") + + for ovpn_net in self.combined_data['openvpn_networks']: + print(f"\nšŸ” OpenVPN: {ovpn_net['network']} via {ovpn_net['pfsense']}") + print(f" Interface: {ovpn_net['interface']}") + + print(f"\n{'='*80}") + print("NETWORK TOPOLOGY ANALYSIS COMPLETE") + print(f"{'='*80}") + + +def main(): + """Command line interface""" + parser = argparse.ArgumentParser( + description='Comprehensive Network Diagram Generator with pfSense Integration' + ) + parser.add_argument('-c', '--config', default='config.json', + help='Network scanner configuration file') + parser.add_argument('-s', '--scan-data', help='Existing network scan JSON file') + parser.add_argument('-p', '--pfsense-xml', nargs='+', + help='pfSense XML configuration files') + parser.add_argument('-o', '--output', default='comprehensive_network.json', + help='Output comprehensive JSON file') + parser.add_argument('--svg', help='Generate SVG diagram') + parser.add_argument('-v', '--verbose', action='store_true', + help='Verbose output') + + args = parser.parse_args() + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + # Load configuration + config = {} + if Path(args.config).exists(): + try: + with open(args.config, 'r') as f: + config = json.load(f) + except Exception as e: + logger.warning(f"Could not load config {args.config}: {e}") + + # Initialize comprehensive mapper + mapper = ComprehensiveNetworkMapper(config) + + # Load network scan data if provided + if args.scan_data and Path(args.scan_data).exists(): + mapper.load_network_scan(args.scan_data) + else: + logger.info("No network scan data provided - will use pfSense data only") + + # Load pfSense configurations + pfsense_files = args.pfsense_xml or [] + # Auto-detect pfSense XML files if none specified + if not pfsense_files: + xml_files = list(Path('.').glob('config-*.xml')) + if xml_files: + pfsense_files = [str(f) for f in xml_files] + logger.info(f"Auto-detected pfSense XML files: {pfsense_files}") + + if pfsense_files: + mapper.load_pfsense_configs(pfsense_files) + else: + logger.warning("No pfSense XML files found") + + # Merge all data + mapper.merge_network_data() + + # Print summary + mapper.print_summary() + + # Export comprehensive data + mapper.export_comprehensive_json(args.output) + print(f"\nāœ… Comprehensive network data saved to: {args.output}") + + # Generate SVG if requested + if args.svg: + svg_file = args.svg + mapper.generate_svg_diagram(svg_file) + print(f"āœ… SVG network diagram generated: {svg_file}") + + +if __name__ == '__main__': + main() diff --git a/network_report.md b/network_report.md new file mode 100644 index 0000000..790d2c7 --- /dev/null +++ b/network_report.md @@ -0,0 +1,14 @@ +# Network Mapping Report +Generated on: Fr 10. Okt 11:14:30 CEST 2025 + +## Network Statistics +- Network Segments: 15 +- pfSense Firewalls: 2 +- WireGuard Networks: 3 +- Static Routes: 3 +- DHCP Static Mappings: 54 + +## Generated Files +- comprehensive_network.json - Complete network data +- comprehensive_network.svg - Network topology diagram +- network_report.md - This summary report diff --git a/run_network_mapping.sh b/run_network_mapping.sh new file mode 100755 index 0000000..fd39037 --- /dev/null +++ b/run_network_mapping.sh @@ -0,0 +1,214 @@ +#!/bin/bash +# Complete Network Mapping Workflow +# This script runs the full network discovery and diagram generation process + +set -e # Exit on any error + +echo "==========================================" +echo "COMPREHENSIVE NETWORK MAPPING WORKFLOW" +echo "==========================================" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Function to print colored output +print_status() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +print_step() { + echo -e "${BLUE}[STEP]${NC} $1" +} + +# Check if required files exist +check_requirements() { + print_step "Checking requirements..." + + if [ ! -f "network_scanner.py" ]; then + print_error "network_scanner.py not found!" + exit 1 + fi + + if [ ! -f "pfsense_xml_parser.py" ]; then + print_error "pfsense_xml_parser.py not found!" + exit 1 + fi + + if [ ! -f "comprehensive_mapper.py" ]; then + print_error "comprehensive_mapper.py not found!" + exit 1 + fi + + if [ ! -f "svg_generator.py" ]; then + print_error "svg_generator.py not found!" + exit 1 + fi + + print_status "All required scripts found" +} + +# Find pfSense XML files +find_pfsense_files() { + print_step "Looking for pfSense XML configuration files..." + + PFSENSE_FILES=$(ls config-*.xml 2>/dev/null || true) + + if [ -z "$PFSENSE_FILES" ]; then + print_warning "No pfSense XML files found in current directory" + print_warning "Please place your pfSense backup XML files here" + echo "Expected format: config-hostname-timestamp.xml" + return 1 + fi + + print_status "Found pfSense XML files:" + echo "$PFSENSE_FILES" | while read -r file; do + echo " - $file" + done + + # Export for use in other functions + export PFSENSE_FILES + return 0 +} + +# Run network scan (optional) +run_network_scan() { + print_step "Running network scan..." + + if [ -f "config.json" ]; then + print_status "Using existing config.json for network scan" + python3 network_scanner.py -c config.json -o network_scan.json + else + print_warning "No config.json found - skipping network scan" + print_warning "Create config.json to enable live network scanning" + return 1 + fi +} + +# Run comprehensive mapping +run_comprehensive_mapping() { + print_step "Running comprehensive network mapping..." + + # Build command with pfSense files + CMD="./comprehensive_mapper.py -o comprehensive_network.json --svg comprehensive_network.svg -v" + + if [ -n "$PFSENSE_FILES" ]; then + CMD="$CMD -p $PFSENSE_FILES" + fi + + if [ -f "network_scan.json" ]; then + CMD="$CMD -s network_scan.json" + fi + + print_status "Executing: $CMD" + eval $CMD +} + +# Generate summary report +generate_report() { + print_step "Generating summary report..." + + if [ -f "comprehensive_network.json" ]; then + echo "# Network Mapping Report" > network_report.md + echo "Generated on: $(date)" >> network_report.md + echo "" >> network_report.md + + # Extract key statistics + SEGMENTS=$(jq '.segments | length' comprehensive_network.json) + PFSENSE_COUNT=$(jq '.pfsense_firewalls | length' comprehensive_network.json) + WG_NETWORKS=$(jq '.wireguard_networks | length' comprehensive_network.json) + STATIC_ROUTES=$(jq '.routing_table | length' comprehensive_network.json) + DHCP_MAPPINGS=$(jq '.static_mappings | length' comprehensive_network.json) + + echo "## Network Statistics" >> network_report.md + echo "- Network Segments: $SEGMENTS" >> network_report.md + echo "- pfSense Firewalls: $PFSENSE_COUNT" >> network_report.md + echo "- WireGuard Networks: $WG_NETWORKS" >> network_report.md + echo "- Static Routes: $STATIC_ROUTES" >> network_report.md + echo "- DHCP Static Mappings: $DHCP_MAPPINGS" >> network_report.md + echo "" >> network_report.md + + echo "## Generated Files" >> network_report.md + echo "- comprehensive_network.json - Complete network data" >> network_report.md + echo "- comprehensive_network.svg - Network topology diagram" >> network_report.md + echo "- network_report.md - This summary report" >> network_report.md + + print_status "Report generated: network_report.md" + else + print_error "No comprehensive network data found" + fi +} + +# Main workflow +main() { + echo "Starting comprehensive network mapping workflow..." + echo "" + + check_requirements + + if ! find_pfsense_files; then + print_error "Cannot proceed without pfSense XML files" + exit 1 + fi + + # Optional network scan + if [ -f "config.json" ]; then + run_network_scan + fi + + # Comprehensive mapping (required) + run_comprehensive_mapping + + # Generate report + generate_report + + echo "" + print_status "Workflow completed successfully!" + echo "" + echo "Generated files:" + echo " šŸ“Š comprehensive_network.json - Complete network data" + echo " šŸ–¼ļø comprehensive_network.svg - Network topology diagram" + echo " šŸ“‹ network_report.md - Summary report" + echo "" + echo "Open comprehensive_network.svg in your browser to view the network diagram" +} + +# Handle command line arguments +case "${1:-}" in + "scan-only") + check_requirements + run_network_scan + ;; + "map-only") + check_requirements + find_pfsense_files + run_comprehensive_mapping + ;; + "report-only") + generate_report + ;; + "help"|"-h"|"--help") + echo "Usage: $0 [command]" + echo "" + echo "Commands:" + echo " (no command) - Run full workflow" + echo " scan-only - Run only network scan" + echo " map-only - Run only comprehensive mapping" + echo " report-only - Generate only summary report" + echo " help - Show this help" + ;; + *) + main + ;; +esac \ No newline at end of file