Add comprehensive network mapper and workflow script

- comprehensive_mapper.py: Combines network scanning with pfSense XML parsing
- run_network_mapping.sh: Complete workflow script for network discovery
- Successfully tested with both pfSense XML files and live network scan
- Generates comprehensive JSON data and SVG network diagrams
- Includes WireGuard VPN topology, static routes, and DHCP mappings
This commit is contained in:
mindesbunister
2025-10-10 11:14:37 +02:00
parent 7621e1829d
commit afe8903454
3 changed files with 583 additions and 0 deletions

355
comprehensive_mapper.py Executable file
View File

@@ -0,0 +1,355 @@
#!/usr/bin/env python3
"""
Comprehensive Network Diagram Generator
Combines network scanning with pfSense XML parsing for complete network topology
"""
import json
import argparse
import logging
from pathlib import Path
from typing import Dict, List, Any
from network_scanner import NetworkScanner
from pfsense_xml_parser import PfSenseXMLParser
from svg_generator import NetworkDiagramGenerator
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
class ComprehensiveNetworkMapper:
"""Complete network mapping solution combining multiple data sources"""
def __init__(self, config: Dict):
self.config = config
self.network_data = {}
self.pfsense_data = {}
self.combined_data = {
'scan_timestamp': None,
'data_sources': [],
'segments': [],
'pfsense_firewalls': [],
'wireguard_networks': [],
'openvpn_networks': [],
'routing_table': [],
'dhcp_leases': [],
'static_mappings': []
}
def load_network_scan(self, scan_file: str):
"""Load network scan data"""
logger.info(f"Loading network scan data from {scan_file}")
try:
with open(scan_file, 'r') as f:
self.network_data = json.load(f)
self.combined_data['data_sources'].append('network_scan')
logger.info("Network scan data loaded successfully")
except Exception as e:
logger.error(f"Error loading network scan: {e}")
def load_pfsense_configs(self, xml_files: List[str]):
"""Load and parse pfSense XML configuration files"""
for xml_file in xml_files:
if Path(xml_file).exists():
logger.info(f"Parsing pfSense config: {xml_file}")
parser = PfSenseXMLParser(xml_file)
data = parser.parse_all()
if data:
hostname = data.get('hostname', f'pfsense_{len(self.pfsense_data)}')
self.pfsense_data[hostname] = data
self.combined_data['data_sources'].append(f'pfsense_{hostname}')
self.combined_data['pfsense_firewalls'].append(data)
logger.info(f"pfSense config {hostname} loaded successfully")
else:
logger.error(f"Failed to parse pfSense config: {xml_file}")
else:
logger.warning(f"pfSense config file not found: {xml_file}")
def merge_network_data(self):
"""Merge all data sources into comprehensive network map"""
logger.info("Merging network data sources...")
# Start with network scan data if available
if self.network_data:
self.combined_data.update({
'scan_timestamp': self.network_data.get('scan_timestamp'),
'segments': self.network_data.get('segments', [])
})
# Process pfSense data
self._process_pfsense_data()
# Extract additional network information
self._extract_network_topology()
logger.info("Network data merging complete")
def _process_pfsense_data(self):
"""Process and integrate pfSense configuration data"""
for hostname, pfsense in self.pfsense_data.items():
logger.info(f"Processing pfSense data for {hostname}")
# Add interfaces as network segments
interfaces = pfsense.get('interfaces', {})
for iface_name, iface_data in interfaces.items():
if iface_data.get('ipaddr') and iface_data.get('ipaddr') != 'dhcp':
try:
# Create network segment from interface
ip = iface_data['ipaddr']
subnet = iface_data.get('subnet', '24')
network_cidr = f"{ip.rsplit('.', 1)[0]}.0/{subnet}"
segment = {
'name': f"{hostname}_{iface_name}",
'cidr': network_cidr,
'gateway': iface_data.get('gateway', ip),
'interface': iface_data.get('interface'),
'pfsense_host': hostname,
'is_vpn': iface_data.get('type') in ['wireguard', 'openvpn'],
'devices': []
}
# Add gateway device
gateway_device = {
'ip': ip,
'hostname': hostname,
'device_type': 'firewall',
'os_type': 'pfSense',
'interface': iface_name,
'pfsense_config': True
}
segment['devices'].append(gateway_device)
self.combined_data['segments'].append(segment)
# Track VPN networks
if iface_data.get('type') == 'wireguard':
self.combined_data['wireguard_networks'].append({
'network': network_cidr,
'interface': iface_data.get('interface'),
'pfsense': hostname,
'peers': pfsense.get('wireguard', {}).get('peers', [])
})
elif iface_data.get('type') == 'openvpn':
self.combined_data['openvpn_networks'].append({
'network': network_cidr,
'interface': iface_data.get('interface'),
'pfsense': hostname
})
except Exception as e:
logger.debug(f"Error processing interface {iface_name}: {e}")
# Add static routes
static_routes = pfsense.get('static_routes', [])
for route in static_routes:
route_info = {
'network': route.get('network'),
'gateway': route.get('gateway'),
'description': route.get('description'),
'pfsense': hostname
}
self.combined_data['routing_table'].append(route_info)
# Add DHCP information
dhcp_config = pfsense.get('dhcp', {})
for iface_name, dhcp_data in dhcp_config.items():
if dhcp_data.get('static_mappings'):
for mapping in dhcp_data['static_mappings']:
mapping_info = {
'ip': mapping.get('ipaddr'),
'mac': mapping.get('mac'),
'hostname': mapping.get('hostname'),
'description': mapping.get('description'),
'interface': iface_name,
'pfsense': hostname
}
self.combined_data['static_mappings'].append(mapping_info)
def _extract_network_topology(self):
"""Extract network topology information from all sources"""
logger.info("Extracting network topology...")
# Process WireGuard peer information
for wg_net in self.combined_data['wireguard_networks']:
for peer in wg_net.get('peers', []):
for allowed_ip in peer.get('allowed_ips', []):
# Create segment for remote networks
if allowed_ip.get('mask') and allowed_ip['mask'] != '32':
remote_segment = {
'name': f"WG_Remote_{allowed_ip['address']}_{allowed_ip['mask']}",
'cidr': f"{allowed_ip['address']}/{allowed_ip['mask']}",
'gateway': wg_net.get('network', '').split('/')[0].rsplit('.', 1)[0] + '.1',
'is_vpn': True,
'vpn_type': 'wireguard',
'pfsense_host': wg_net.get('pfsense'),
'devices': []
}
self.combined_data['segments'].append(remote_segment)
def generate_svg_diagram(self, output_file: str):
"""Generate SVG network diagram"""
logger.info(f"Generating SVG diagram: {output_file}")
# Use the existing SVG generator
generator = NetworkDiagramGenerator(self.combined_data)
generator.generate_svg(output_file)
logger.info(f"SVG diagram generated: {output_file}")
def export_comprehensive_json(self, output_file: str):
"""Export comprehensive network data"""
logger.info(f"Exporting comprehensive data to {output_file}")
with open(output_file, 'w', encoding='utf-8') as f:
json.dump(self.combined_data, f, indent=2, ensure_ascii=False)
logger.info(f"Comprehensive network data exported to {output_file}")
def print_summary(self):
"""Print comprehensive network summary"""
print("\n" + "="*80)
print("COMPREHENSIVE NETWORK MAPPING SUMMARY")
print("="*80)
print(f"\nData Sources: {', '.join(self.combined_data['data_sources'])}")
print(f"Network Segments: {len(self.combined_data['segments'])}")
print(f"pfSense Firewalls: {len(self.combined_data['pfsense_firewalls'])}")
print(f"WireGuard Networks: {len(self.combined_data['wireguard_networks'])}")
print(f"OpenVPN Networks: {len(self.combined_data['openvpn_networks'])}")
print(f"Static Routes: {len(self.combined_data['routing_table'])}")
print(f"DHCP Static Mappings: {len(self.combined_data['static_mappings'])}")
# Network segments summary
print(f"\n{'='*80}")
print("NETWORK SEGMENTS")
print(f"{'='*80}")
for segment in self.combined_data['segments']:
vpn_indicator = " (VPN)" if segment.get('is_vpn') else ""
pfsense_indicator = f" [{segment.get('pfsense_host')}]" if segment.get('pfsense_host') else ""
print(f"\n📡 {segment['name']}{vpn_indicator}{pfsense_indicator}")
print(f" CIDR: {segment['cidr']}")
print(f" Gateway: {segment.get('gateway', 'N/A')}")
print(f" Devices: {len(segment.get('devices', []))}")
# Show devices
for device in segment.get('devices', [])[:5]: # Show first 5
print(f"{device.get('ip', 'N/A')} - {device.get('hostname', 'N/A')} ({device.get('device_type', 'N/A')})")
if len(segment.get('devices', [])) > 5:
print(f" ... and {len(segment.get('devices', [])) - 5} more devices")
# pfSense summary
if self.combined_data['pfsense_firewalls']:
print(f"\n{'='*80}")
print("PFSENSE FIREWALLS")
print(f"{'='*80}")
for pfsense in self.combined_data['pfsense_firewalls']:
hostname = pfsense.get('hostname', 'Unknown')
print(f"\n🛡️ {hostname}")
print(f" Interfaces: {len(pfsense.get('interfaces', {}))}")
print(f" Static Routes: {len(pfsense.get('static_routes', []))}")
wg_config = pfsense.get('wireguard', {})
if wg_config.get('enabled'):
print(f" WireGuard: {len(wg_config.get('tunnels', []))} tunnels, {len(wg_config.get('peers', []))} peers")
dhcp_config = pfsense.get('dhcp', {})
total_mappings = sum(len(dhcp.get('static_mappings', [])) for dhcp in dhcp_config.values())
print(f" DHCP Static Mappings: {total_mappings}")
# VPN Networks
if self.combined_data['wireguard_networks'] or self.combined_data['openvpn_networks']:
print(f"\n{'='*80}")
print("VPN NETWORKS")
print(f"{'='*80}")
for wg_net in self.combined_data['wireguard_networks']:
print(f"\n🔐 WireGuard: {wg_net['network']} via {wg_net['pfsense']}")
print(f" Interface: {wg_net['interface']}")
print(f" Peers: {len(wg_net.get('peers', []))}")
for ovpn_net in self.combined_data['openvpn_networks']:
print(f"\n🔐 OpenVPN: {ovpn_net['network']} via {ovpn_net['pfsense']}")
print(f" Interface: {ovpn_net['interface']}")
print(f"\n{'='*80}")
print("NETWORK TOPOLOGY ANALYSIS COMPLETE")
print(f"{'='*80}")
def main():
"""Command line interface"""
parser = argparse.ArgumentParser(
description='Comprehensive Network Diagram Generator with pfSense Integration'
)
parser.add_argument('-c', '--config', default='config.json',
help='Network scanner configuration file')
parser.add_argument('-s', '--scan-data', help='Existing network scan JSON file')
parser.add_argument('-p', '--pfsense-xml', nargs='+',
help='pfSense XML configuration files')
parser.add_argument('-o', '--output', default='comprehensive_network.json',
help='Output comprehensive JSON file')
parser.add_argument('--svg', help='Generate SVG diagram')
parser.add_argument('-v', '--verbose', action='store_true',
help='Verbose output')
args = parser.parse_args()
if args.verbose:
logging.getLogger().setLevel(logging.DEBUG)
# Load configuration
config = {}
if Path(args.config).exists():
try:
with open(args.config, 'r') as f:
config = json.load(f)
except Exception as e:
logger.warning(f"Could not load config {args.config}: {e}")
# Initialize comprehensive mapper
mapper = ComprehensiveNetworkMapper(config)
# Load network scan data if provided
if args.scan_data and Path(args.scan_data).exists():
mapper.load_network_scan(args.scan_data)
else:
logger.info("No network scan data provided - will use pfSense data only")
# Load pfSense configurations
pfsense_files = args.pfsense_xml or []
# Auto-detect pfSense XML files if none specified
if not pfsense_files:
xml_files = list(Path('.').glob('config-*.xml'))
if xml_files:
pfsense_files = [str(f) for f in xml_files]
logger.info(f"Auto-detected pfSense XML files: {pfsense_files}")
if pfsense_files:
mapper.load_pfsense_configs(pfsense_files)
else:
logger.warning("No pfSense XML files found")
# Merge all data
mapper.merge_network_data()
# Print summary
mapper.print_summary()
# Export comprehensive data
mapper.export_comprehensive_json(args.output)
print(f"\n✅ Comprehensive network data saved to: {args.output}")
# Generate SVG if requested
if args.svg:
svg_file = args.svg
mapper.generate_svg_diagram(svg_file)
print(f"✅ SVG network diagram generated: {svg_file}")
if __name__ == '__main__':
main()

14
network_report.md Normal file
View File

@@ -0,0 +1,14 @@
# Network Mapping Report
Generated on: Fr 10. Okt 11:14:30 CEST 2025
## Network Statistics
- Network Segments: 15
- pfSense Firewalls: 2
- WireGuard Networks: 3
- Static Routes: 3
- DHCP Static Mappings: 54
## Generated Files
- comprehensive_network.json - Complete network data
- comprehensive_network.svg - Network topology diagram
- network_report.md - This summary report

214
run_network_mapping.sh Executable file
View File

@@ -0,0 +1,214 @@
#!/bin/bash
# Complete Network Mapping Workflow
# This script runs the full network discovery and diagram generation process
set -e # Exit on any error
echo "=========================================="
echo "COMPREHENSIVE NETWORK MAPPING WORKFLOW"
echo "=========================================="
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Function to print colored output
print_status() {
echo -e "${GREEN}[INFO]${NC} $1"
}
print_warning() {
echo -e "${YELLOW}[WARN]${NC} $1"
}
print_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
print_step() {
echo -e "${BLUE}[STEP]${NC} $1"
}
# Check if required files exist
check_requirements() {
print_step "Checking requirements..."
if [ ! -f "network_scanner.py" ]; then
print_error "network_scanner.py not found!"
exit 1
fi
if [ ! -f "pfsense_xml_parser.py" ]; then
print_error "pfsense_xml_parser.py not found!"
exit 1
fi
if [ ! -f "comprehensive_mapper.py" ]; then
print_error "comprehensive_mapper.py not found!"
exit 1
fi
if [ ! -f "svg_generator.py" ]; then
print_error "svg_generator.py not found!"
exit 1
fi
print_status "All required scripts found"
}
# Find pfSense XML files
find_pfsense_files() {
print_step "Looking for pfSense XML configuration files..."
PFSENSE_FILES=$(ls config-*.xml 2>/dev/null || true)
if [ -z "$PFSENSE_FILES" ]; then
print_warning "No pfSense XML files found in current directory"
print_warning "Please place your pfSense backup XML files here"
echo "Expected format: config-hostname-timestamp.xml"
return 1
fi
print_status "Found pfSense XML files:"
echo "$PFSENSE_FILES" | while read -r file; do
echo " - $file"
done
# Export for use in other functions
export PFSENSE_FILES
return 0
}
# Run network scan (optional)
run_network_scan() {
print_step "Running network scan..."
if [ -f "config.json" ]; then
print_status "Using existing config.json for network scan"
python3 network_scanner.py -c config.json -o network_scan.json
else
print_warning "No config.json found - skipping network scan"
print_warning "Create config.json to enable live network scanning"
return 1
fi
}
# Run comprehensive mapping
run_comprehensive_mapping() {
print_step "Running comprehensive network mapping..."
# Build command with pfSense files
CMD="./comprehensive_mapper.py -o comprehensive_network.json --svg comprehensive_network.svg -v"
if [ -n "$PFSENSE_FILES" ]; then
CMD="$CMD -p $PFSENSE_FILES"
fi
if [ -f "network_scan.json" ]; then
CMD="$CMD -s network_scan.json"
fi
print_status "Executing: $CMD"
eval $CMD
}
# Generate summary report
generate_report() {
print_step "Generating summary report..."
if [ -f "comprehensive_network.json" ]; then
echo "# Network Mapping Report" > network_report.md
echo "Generated on: $(date)" >> network_report.md
echo "" >> network_report.md
# Extract key statistics
SEGMENTS=$(jq '.segments | length' comprehensive_network.json)
PFSENSE_COUNT=$(jq '.pfsense_firewalls | length' comprehensive_network.json)
WG_NETWORKS=$(jq '.wireguard_networks | length' comprehensive_network.json)
STATIC_ROUTES=$(jq '.routing_table | length' comprehensive_network.json)
DHCP_MAPPINGS=$(jq '.static_mappings | length' comprehensive_network.json)
echo "## Network Statistics" >> network_report.md
echo "- Network Segments: $SEGMENTS" >> network_report.md
echo "- pfSense Firewalls: $PFSENSE_COUNT" >> network_report.md
echo "- WireGuard Networks: $WG_NETWORKS" >> network_report.md
echo "- Static Routes: $STATIC_ROUTES" >> network_report.md
echo "- DHCP Static Mappings: $DHCP_MAPPINGS" >> network_report.md
echo "" >> network_report.md
echo "## Generated Files" >> network_report.md
echo "- comprehensive_network.json - Complete network data" >> network_report.md
echo "- comprehensive_network.svg - Network topology diagram" >> network_report.md
echo "- network_report.md - This summary report" >> network_report.md
print_status "Report generated: network_report.md"
else
print_error "No comprehensive network data found"
fi
}
# Main workflow
main() {
echo "Starting comprehensive network mapping workflow..."
echo ""
check_requirements
if ! find_pfsense_files; then
print_error "Cannot proceed without pfSense XML files"
exit 1
fi
# Optional network scan
if [ -f "config.json" ]; then
run_network_scan
fi
# Comprehensive mapping (required)
run_comprehensive_mapping
# Generate report
generate_report
echo ""
print_status "Workflow completed successfully!"
echo ""
echo "Generated files:"
echo " 📊 comprehensive_network.json - Complete network data"
echo " 🖼️ comprehensive_network.svg - Network topology diagram"
echo " 📋 network_report.md - Summary report"
echo ""
echo "Open comprehensive_network.svg in your browser to view the network diagram"
}
# Handle command line arguments
case "${1:-}" in
"scan-only")
check_requirements
run_network_scan
;;
"map-only")
check_requirements
find_pfsense_files
run_comprehensive_mapping
;;
"report-only")
generate_report
;;
"help"|"-h"|"--help")
echo "Usage: $0 [command]"
echo ""
echo "Commands:"
echo " (no command) - Run full workflow"
echo " scan-only - Run only network scan"
echo " map-only - Run only comprehensive mapping"
echo " report-only - Generate only summary report"
echo " help - Show this help"
;;
*)
main
;;
esac