feat: Add periodic Drift position validation to prevent ghost positions
- Added 5-minute validation interval to Position Manager - Validates tracked positions against actual Drift state - Auto-cleanup ghost positions (DB shows open but Drift shows closed) - Prevents rate limit storms from accumulated ghost positions - Logs detailed ghost detection: DB state vs Drift state - Self-healing system requires no manual intervention Implementation: - scheduleValidation(): Sets 5-minute timer after monitoring starts - validatePositions(): Queries each tracked position on Drift - handleExternalClosure(): Reusable method for ghost cleanup - Clears interval when monitoring stops Benefits: - Prevents ghost position accumulation - Eliminates need for manual container restarts - Minimal RPC overhead (1 check per 5 min per position) - Addresses root cause (state management) not symptom (rate limits) Fixes: - Ghost positions from failed DB updates during external closures - Container restart state sync issues - Rate limit exhaustion from managing non-existent positions
This commit is contained in:
mindesbunister
@@ -75,6 +75,7 @@ export class PositionManager {
|
||||
private config: TradingConfig
|
||||
private isMonitoring: boolean = false
|
||||
private initialized: boolean = false
|
||||
private validationInterval: NodeJS.Timeout | null = null
|
||||
|
||||
constructor(config?: Partial<TradingConfig>) {
|
||||
this.config = getMergedConfig(config)
|
||||
@@ -197,6 +198,142 @@ export class PositionManager {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule periodic validation to detect ghost positions
|
||||
*/
|
||||
private scheduleValidation(): void {
|
||||
// Clear any existing interval
|
||||
if (this.validationInterval) {
|
||||
clearInterval(this.validationInterval)
|
||||
}
|
||||
|
||||
// Run validation every 5 minutes
|
||||
const validationIntervalMs = 5 * 60 * 1000
|
||||
this.validationInterval = setInterval(async () => {
|
||||
await this.validatePositions()
|
||||
}, validationIntervalMs)
|
||||
|
||||
console.log('🔍 Scheduled position validation every 5 minutes')
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate tracked positions against Drift to detect ghosts
|
||||
*
|
||||
* Ghost positions occur when:
|
||||
* - Database has exitReason IS NULL (we think it's open)
|
||||
* - But Drift shows position closed or missing
|
||||
*
|
||||
* This happens due to:
|
||||
* - Failed database updates during external closures
|
||||
* - Container restarts before cleanup completed
|
||||
* - On-chain orders filled without Position Manager knowing
|
||||
*/
|
||||
private async validatePositions(): Promise<void> {
|
||||
if (this.activeTrades.size === 0) {
|
||||
return // Nothing to validate
|
||||
}
|
||||
|
||||
console.log('🔍 Validating positions against Drift...')
|
||||
|
||||
try {
|
||||
const driftService = getDriftService()
|
||||
|
||||
// Skip if Drift service not initialized
|
||||
if (!driftService || !(driftService as any).isInitialized) {
|
||||
console.log('⏳ Drift service not ready, skipping validation')
|
||||
return
|
||||
}
|
||||
|
||||
// Check each tracked trade individually
|
||||
for (const [tradeId, trade] of this.activeTrades) {
|
||||
const marketConfig = getMarketConfig(trade.symbol)
|
||||
|
||||
try {
|
||||
const position = await driftService.getPosition(marketConfig.driftMarketIndex)
|
||||
|
||||
// Ghost detected: we're tracking it but Drift shows closed/missing
|
||||
if (!position || Math.abs(position.size) < 0.01) {
|
||||
console.log(`🔴 Ghost position detected: ${trade.symbol} (${tradeId})`)
|
||||
console.log(` Database: exitReason IS NULL (thinks it's open)`)
|
||||
console.log(` Drift: Position ${position ? 'closed (size=' + position.size + ')' : 'missing'}`)
|
||||
console.log(` Cause: Likely failed DB update during external closure`)
|
||||
|
||||
// Auto-cleanup: Handle as external closure
|
||||
await this.handleExternalClosure(trade, 'Ghost position cleanup')
|
||||
console.log(`✅ Ghost position cleaned up: ${trade.symbol}`)
|
||||
}
|
||||
} catch (posError) {
|
||||
console.error(`⚠️ Could not check ${trade.symbol} on Drift:`, posError)
|
||||
// Continue checking other positions
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`✅ Validation complete: ${this.activeTrades.size} positions healthy`)
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Position validation failed:', error)
|
||||
// Don't throw - validation errors shouldn't break monitoring
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle external closure for ghost position cleanup
|
||||
*
|
||||
* Called when:
|
||||
* - Periodic validation detects position closed on Drift but tracked in DB
|
||||
* - Manual cleanup needed after failed database updates
|
||||
*/
|
||||
private async handleExternalClosure(trade: ActiveTrade, reason: string): Promise<void> {
|
||||
console.log(`🧹 Handling external closure: ${trade.symbol} (${reason})`)
|
||||
|
||||
// Calculate approximate P&L using last known price
|
||||
const profitPercent = this.calculateProfitPercent(
|
||||
trade.entryPrice,
|
||||
trade.lastPrice,
|
||||
trade.direction
|
||||
)
|
||||
const estimatedPnL = (trade.currentSize * profitPercent) / 100
|
||||
|
||||
console.log(`💰 Estimated P&L: ${profitPercent.toFixed(2)}% → $${estimatedPnL.toFixed(2)}`)
|
||||
|
||||
// Remove from monitoring FIRST to prevent race conditions
|
||||
const tradeId = trade.id
|
||||
if (!this.activeTrades.has(tradeId)) {
|
||||
console.log(`⚠️ Trade ${tradeId} already removed, skipping cleanup`)
|
||||
return
|
||||
}
|
||||
|
||||
this.activeTrades.delete(tradeId)
|
||||
console.log(`🗑️ Removed ${trade.symbol} from monitoring`)
|
||||
|
||||
// Update database
|
||||
try {
|
||||
const holdTimeSeconds = Math.floor((Date.now() - trade.entryTime) / 1000)
|
||||
await updateTradeExit({
|
||||
positionId: trade.positionId,
|
||||
exitPrice: trade.lastPrice,
|
||||
exitReason: 'manual', // Ghost closures treated as manual
|
||||
realizedPnL: estimatedPnL,
|
||||
exitOrderTx: reason, // Store cleanup reason
|
||||
holdTimeSeconds,
|
||||
maxDrawdown: Math.abs(Math.min(0, trade.maxAdverseExcursion)),
|
||||
maxGain: Math.max(0, trade.maxFavorableExcursion),
|
||||
maxFavorableExcursion: trade.maxFavorableExcursion,
|
||||
maxAdverseExcursion: trade.maxAdverseExcursion,
|
||||
maxFavorablePrice: trade.maxFavorablePrice,
|
||||
maxAdversePrice: trade.maxAdversePrice,
|
||||
})
|
||||
console.log(`💾 Ghost closure saved to database`)
|
||||
} catch (dbError) {
|
||||
console.error('❌ Failed to save ghost closure:', dbError)
|
||||
}
|
||||
|
||||
// Stop monitoring if no more trades
|
||||
if (this.activeTrades.size === 0 && this.isMonitoring) {
|
||||
this.stopMonitoring()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all active trades
|
||||
*/
|
||||
@@ -244,6 +381,9 @@ export class PositionManager {
|
||||
|
||||
this.isMonitoring = true
|
||||
console.log('✅ Position monitoring active')
|
||||
|
||||
// Schedule periodic validation to detect and cleanup ghost positions
|
||||
this.scheduleValidation()
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -258,6 +398,12 @@ export class PositionManager {
|
||||
|
||||
const priceMonitor = getPythPriceMonitor()
|
||||
await priceMonitor.stop()
|
||||
|
||||
// Clear validation interval
|
||||
if (this.validationInterval) {
|
||||
clearInterval(this.validationInterval)
|
||||
this.validationInterval = null
|
||||
}
|
||||
|
||||
this.isMonitoring = false
|
||||
console.log('✅ Position monitoring stopped')
|
||||
|
||||
Reference in New Issue
Block a user