feat: Add periodic Drift position validation to prevent ghost positions
- Added 5-minute validation interval to Position Manager - Validates tracked positions against actual Drift state - Auto-cleanup ghost positions (DB shows open but Drift shows closed) - Prevents rate limit storms from accumulated ghost positions - Logs detailed ghost detection: DB state vs Drift state - Self-healing system requires no manual intervention Implementation: - scheduleValidation(): Sets 5-minute timer after monitoring starts - validatePositions(): Queries each tracked position on Drift - handleExternalClosure(): Reusable method for ghost cleanup - Clears interval when monitoring stops Benefits: - Prevents ghost position accumulation - Eliminates need for manual container restarts - Minimal RPC overhead (1 check per 5 min per position) - Addresses root cause (state management) not symptom (rate limits) Fixes: - Ghost positions from failed DB updates during external closures - Container restart state sync issues - Rate limit exhaustion from managing non-existent positions
This commit is contained in:
@@ -75,6 +75,7 @@ export class PositionManager {
|
|||||||
private config: TradingConfig
|
private config: TradingConfig
|
||||||
private isMonitoring: boolean = false
|
private isMonitoring: boolean = false
|
||||||
private initialized: boolean = false
|
private initialized: boolean = false
|
||||||
|
private validationInterval: NodeJS.Timeout | null = null
|
||||||
|
|
||||||
constructor(config?: Partial<TradingConfig>) {
|
constructor(config?: Partial<TradingConfig>) {
|
||||||
this.config = getMergedConfig(config)
|
this.config = getMergedConfig(config)
|
||||||
@@ -197,6 +198,142 @@ export class PositionManager {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Schedule periodic validation to detect ghost positions
|
||||||
|
*/
|
||||||
|
private scheduleValidation(): void {
|
||||||
|
// Clear any existing interval
|
||||||
|
if (this.validationInterval) {
|
||||||
|
clearInterval(this.validationInterval)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run validation every 5 minutes
|
||||||
|
const validationIntervalMs = 5 * 60 * 1000
|
||||||
|
this.validationInterval = setInterval(async () => {
|
||||||
|
await this.validatePositions()
|
||||||
|
}, validationIntervalMs)
|
||||||
|
|
||||||
|
console.log('🔍 Scheduled position validation every 5 minutes')
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate tracked positions against Drift to detect ghosts
|
||||||
|
*
|
||||||
|
* Ghost positions occur when:
|
||||||
|
* - Database has exitReason IS NULL (we think it's open)
|
||||||
|
* - But Drift shows position closed or missing
|
||||||
|
*
|
||||||
|
* This happens due to:
|
||||||
|
* - Failed database updates during external closures
|
||||||
|
* - Container restarts before cleanup completed
|
||||||
|
* - On-chain orders filled without Position Manager knowing
|
||||||
|
*/
|
||||||
|
private async validatePositions(): Promise<void> {
|
||||||
|
if (this.activeTrades.size === 0) {
|
||||||
|
return // Nothing to validate
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('🔍 Validating positions against Drift...')
|
||||||
|
|
||||||
|
try {
|
||||||
|
const driftService = getDriftService()
|
||||||
|
|
||||||
|
// Skip if Drift service not initialized
|
||||||
|
if (!driftService || !(driftService as any).isInitialized) {
|
||||||
|
console.log('⏳ Drift service not ready, skipping validation')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check each tracked trade individually
|
||||||
|
for (const [tradeId, trade] of this.activeTrades) {
|
||||||
|
const marketConfig = getMarketConfig(trade.symbol)
|
||||||
|
|
||||||
|
try {
|
||||||
|
const position = await driftService.getPosition(marketConfig.driftMarketIndex)
|
||||||
|
|
||||||
|
// Ghost detected: we're tracking it but Drift shows closed/missing
|
||||||
|
if (!position || Math.abs(position.size) < 0.01) {
|
||||||
|
console.log(`🔴 Ghost position detected: ${trade.symbol} (${tradeId})`)
|
||||||
|
console.log(` Database: exitReason IS NULL (thinks it's open)`)
|
||||||
|
console.log(` Drift: Position ${position ? 'closed (size=' + position.size + ')' : 'missing'}`)
|
||||||
|
console.log(` Cause: Likely failed DB update during external closure`)
|
||||||
|
|
||||||
|
// Auto-cleanup: Handle as external closure
|
||||||
|
await this.handleExternalClosure(trade, 'Ghost position cleanup')
|
||||||
|
console.log(`✅ Ghost position cleaned up: ${trade.symbol}`)
|
||||||
|
}
|
||||||
|
} catch (posError) {
|
||||||
|
console.error(`⚠️ Could not check ${trade.symbol} on Drift:`, posError)
|
||||||
|
// Continue checking other positions
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`✅ Validation complete: ${this.activeTrades.size} positions healthy`)
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('❌ Position validation failed:', error)
|
||||||
|
// Don't throw - validation errors shouldn't break monitoring
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle external closure for ghost position cleanup
|
||||||
|
*
|
||||||
|
* Called when:
|
||||||
|
* - Periodic validation detects position closed on Drift but tracked in DB
|
||||||
|
* - Manual cleanup needed after failed database updates
|
||||||
|
*/
|
||||||
|
private async handleExternalClosure(trade: ActiveTrade, reason: string): Promise<void> {
|
||||||
|
console.log(`🧹 Handling external closure: ${trade.symbol} (${reason})`)
|
||||||
|
|
||||||
|
// Calculate approximate P&L using last known price
|
||||||
|
const profitPercent = this.calculateProfitPercent(
|
||||||
|
trade.entryPrice,
|
||||||
|
trade.lastPrice,
|
||||||
|
trade.direction
|
||||||
|
)
|
||||||
|
const estimatedPnL = (trade.currentSize * profitPercent) / 100
|
||||||
|
|
||||||
|
console.log(`💰 Estimated P&L: ${profitPercent.toFixed(2)}% → $${estimatedPnL.toFixed(2)}`)
|
||||||
|
|
||||||
|
// Remove from monitoring FIRST to prevent race conditions
|
||||||
|
const tradeId = trade.id
|
||||||
|
if (!this.activeTrades.has(tradeId)) {
|
||||||
|
console.log(`⚠️ Trade ${tradeId} already removed, skipping cleanup`)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
this.activeTrades.delete(tradeId)
|
||||||
|
console.log(`🗑️ Removed ${trade.symbol} from monitoring`)
|
||||||
|
|
||||||
|
// Update database
|
||||||
|
try {
|
||||||
|
const holdTimeSeconds = Math.floor((Date.now() - trade.entryTime) / 1000)
|
||||||
|
await updateTradeExit({
|
||||||
|
positionId: trade.positionId,
|
||||||
|
exitPrice: trade.lastPrice,
|
||||||
|
exitReason: 'manual', // Ghost closures treated as manual
|
||||||
|
realizedPnL: estimatedPnL,
|
||||||
|
exitOrderTx: reason, // Store cleanup reason
|
||||||
|
holdTimeSeconds,
|
||||||
|
maxDrawdown: Math.abs(Math.min(0, trade.maxAdverseExcursion)),
|
||||||
|
maxGain: Math.max(0, trade.maxFavorableExcursion),
|
||||||
|
maxFavorableExcursion: trade.maxFavorableExcursion,
|
||||||
|
maxAdverseExcursion: trade.maxAdverseExcursion,
|
||||||
|
maxFavorablePrice: trade.maxFavorablePrice,
|
||||||
|
maxAdversePrice: trade.maxAdversePrice,
|
||||||
|
})
|
||||||
|
console.log(`💾 Ghost closure saved to database`)
|
||||||
|
} catch (dbError) {
|
||||||
|
console.error('❌ Failed to save ghost closure:', dbError)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop monitoring if no more trades
|
||||||
|
if (this.activeTrades.size === 0 && this.isMonitoring) {
|
||||||
|
this.stopMonitoring()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get all active trades
|
* Get all active trades
|
||||||
*/
|
*/
|
||||||
@@ -244,6 +381,9 @@ export class PositionManager {
|
|||||||
|
|
||||||
this.isMonitoring = true
|
this.isMonitoring = true
|
||||||
console.log('✅ Position monitoring active')
|
console.log('✅ Position monitoring active')
|
||||||
|
|
||||||
|
// Schedule periodic validation to detect and cleanup ghost positions
|
||||||
|
this.scheduleValidation()
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -259,6 +399,12 @@ export class PositionManager {
|
|||||||
const priceMonitor = getPythPriceMonitor()
|
const priceMonitor = getPythPriceMonitor()
|
||||||
await priceMonitor.stop()
|
await priceMonitor.stop()
|
||||||
|
|
||||||
|
// Clear validation interval
|
||||||
|
if (this.validationInterval) {
|
||||||
|
clearInterval(this.validationInterval)
|
||||||
|
this.validationInterval = null
|
||||||
|
}
|
||||||
|
|
||||||
this.isMonitoring = false
|
this.isMonitoring = false
|
||||||
console.log('✅ Position monitoring stopped')
|
console.log('✅ Position monitoring stopped')
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user