Files
trading_bot_v4/lib/monitoring/drift-state-verifier.ts
mindesbunister 9e78761648 critical: Bug #82 LONG-TERM FIX - Comprehensive position verification
REPLACES emergency disable with intelligent verification:

1. Position Identity Verification:
   - Compares DB exitTime vs active trade timestamps
   - Verifies size matches within 15% tolerance
   - Verifies direction matches (long/short)
   - Checks entry price matches within 2%

2. Grace Period Enforcement:
   - 10-minute wait after DB exit before attempting close
   - Allows Drift state propagation

3. Safety Checks:
   - Cooldown (5 min) prevents retry loops
   - Protection logging when position skipped
   - Fail-open bias: when uncertain, do nothing

4. Test Coverage:
   - 8 test scenarios covering active position protection
   - Verified ghost closure tests
   - Edge case handling tests
   - Fail-open bias validation

Files:
- lib/monitoring/drift-state-verifier.ts (276 lines added)
- tests/integration/drift-state-verifier/position-verification.test.ts (420 lines)

User can now rely on automatic orphan cleanup without risk of
accidentally closing active positions. System protects newer trades
when old database records exist for same symbol.

Deployed: Dec 10, 2025 ~11:25 CET
2025-12-10 11:58:27 +01:00

612 lines
20 KiB
TypeScript
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* Drift State Verifier Service
*
* Double-checks that positions marked as closed in our database
* are actually closed on Drift Protocol. If mismatches found,
* attempts to close the position again.
*
* Background: Drift occasionally confirms close transactions but
* doesn't actually close the position (state propagation delay or
* partial fill issues). This service detects and fixes those cases.
*
* Created: Dec 7, 2025
*/
import { getDriftService } from '../drift/client'
import { getPrismaClient } from '../database/trades'
import { closePosition } from '../drift/orders'
import { sendTelegramMessage } from '../notifications/telegram'
import { Prisma } from '@prisma/client'
export interface DriftStateMismatch {
tradeId: string
symbol: string
expectedState: 'closed' | 'open'
actualState: 'closed' | 'open'
driftSize: number
dbExitReason: string | null
timeSinceExit: number // milliseconds
}
class DriftStateVerifier {
private isRunning: boolean = false
private checkIntervalMs: number = 10 * 60 * 1000 // 10 minutes
private intervalId: NodeJS.Timeout | null = null
// BUG #80 FIX: Track close attempts per symbol to enforce cooldown
private recentCloseAttempts: Map<string, number> = new Map()
private readonly COOLDOWN_MS = 5 * 60 * 1000 // 5 minutes
/**
* Start the periodic verification service
*/
start(): void {
if (this.isRunning) {
console.log('🔍 Drift state verifier already running')
return
}
console.log('🔍 Starting Drift state verifier (checks every 10 minutes)')
this.isRunning = true
// Run first check after 2 minutes (allow time for initial startup)
setTimeout(() => {
this.runVerification().catch(err => {
console.error('❌ Error in initial Drift state verification:', err)
})
}, 2 * 60 * 1000)
// Then run every 10 minutes
this.intervalId = setInterval(() => {
this.runVerification().catch(err => {
console.error('❌ Error in Drift state verification:', err)
})
}, this.checkIntervalMs)
}
/**
* Stop the periodic verification service
*/
stop(): void {
if (this.intervalId) {
clearInterval(this.intervalId)
this.intervalId = null
}
this.isRunning = false
console.log('🔍 Drift state verifier stopped')
}
/**
* Run verification check once (can be called manually)
*/
async runVerification(): Promise<DriftStateMismatch[]> {
console.log('🔍 Running Drift state verification...')
const mismatches: DriftStateMismatch[] = []
try {
const driftService = await getDriftService()
const prisma = getPrismaClient()
// Check 1: Find trades marked as closed in last 24 hours
// These should definitely not exist on Drift anymore
const recentlyClosedTrades = await prisma.trade.findMany({
where: {
exitReason: { not: null },
exitTime: {
gte: new Date(Date.now() - 24 * 60 * 60 * 1000)
}
},
select: {
id: true,
positionId: true,
symbol: true,
exitReason: true,
exitTime: true,
},
})
console.log(` Checking ${recentlyClosedTrades.length} recently closed trades...`)
for (const trade of recentlyClosedTrades) {
try {
// Extract market index from symbol (SOL-PERP → 0, ETH-PERP → 1, etc.)
const marketIndex = this.getMarketIndex(trade.symbol)
if (marketIndex === null) continue
// Query Drift for position
const driftPosition = await driftService.getPosition(marketIndex)
if (driftPosition && Math.abs(driftPosition.size) >= 0.01) {
// MISMATCH: DB says closed, Drift says open
const timeSinceExit = Date.now() - new Date(trade.exitTime!).getTime()
mismatches.push({
tradeId: trade.id,
symbol: trade.symbol,
expectedState: 'closed',
actualState: 'open',
driftSize: Math.abs(driftPosition.size),
dbExitReason: trade.exitReason,
timeSinceExit,
})
console.error(`🚨 MISMATCH DETECTED: ${trade.symbol}`)
console.error(` DB: Closed ${(timeSinceExit / 60000).toFixed(1)}min ago (${trade.exitReason})`)
console.error(` Drift: Still open with size ${driftPosition.size}`)
}
} catch (err) {
console.error(` Error checking ${trade.symbol}:`, err)
}
}
// Check 2: Find trades marked as open but actually closed on Drift
// (Less critical but worth detecting)
const openTrades = await prisma.trade.findMany({
where: {
exitReason: null,
},
select: {
id: true,
positionId: true,
symbol: true,
createdAt: true,
},
})
console.log(` Checking ${openTrades.length} open trades...`)
for (const trade of openTrades) {
try {
const marketIndex = this.getMarketIndex(trade.symbol)
if (marketIndex === null) continue
const driftPosition = await driftService.getPosition(marketIndex)
if (!driftPosition || Math.abs(driftPosition.size) < 0.01) {
// MISMATCH: DB says open, Drift says closed
const timeSinceExit = Date.now() - new Date(trade.createdAt).getTime()
mismatches.push({
tradeId: trade.id,
symbol: trade.symbol,
expectedState: 'open',
actualState: 'closed',
driftSize: 0,
dbExitReason: null,
timeSinceExit,
})
console.error(`🚨 MISMATCH DETECTED: ${trade.symbol}`)
console.error(` DB: Open since ${(timeSinceExit / 60000).toFixed(1)}min ago`)
console.error(` Drift: Position closed (size 0)`)
}
} catch (err) {
console.error(` Error checking ${trade.symbol}:`, err)
}
}
if (mismatches.length === 0) {
console.log(' ✅ No mismatches found - DB and Drift states match')
} else {
console.error(` ❌ Found ${mismatches.length} mismatches!`)
await this.handleMismatches(mismatches)
}
} catch (error) {
console.error('❌ Error running Drift state verification:', error)
}
return mismatches
}
/**
* Handle detected mismatches
*/
private async handleMismatches(mismatches: DriftStateMismatch[]): Promise<void> {
for (const mismatch of mismatches) {
if (mismatch.expectedState === 'closed' && mismatch.actualState === 'open') {
// CRITICAL: Position should be closed but is still open on Drift
await this.retryClose(mismatch)
} else if (mismatch.expectedState === 'open' && mismatch.actualState === 'closed') {
// Position closed externally - this is handled by Position Manager's ghost detection
console.log(` ${mismatch.symbol}: Ghost position (will be cleaned by Position Manager)`)
}
}
// Send Telegram alert
await this.sendMismatchAlert(mismatches)
}
/**
* Retry closing a position that should be closed but isn't
* BUG #82 LONG-TERM FIX (Dec 10, 2025): Comprehensive position verification
*
* CRITICAL SAFETY CHECKS:
* 1. Verify Drift position exists and matches DB record
* 2. Check position freshness: is it NEWER than DB exit time?
* 3. Verify size/direction alignment within tolerance
* 4. Grace period: wait 10+ minutes after DB exit before acting
* 5. Fail-open bias: when in doubt, do nothing and alert
*/
private async retryClose(mismatch: DriftStateMismatch): Promise<void> {
console.log(`🔄 Analyzing close candidate for ${mismatch.symbol}...`)
try {
// STEP 1: Cooldown enforcement (prevents retry spam)
const cooldownCheck = await this.checkCooldown(mismatch.symbol)
if (!cooldownCheck.canProceed) {
console.log(` ⏸️ ${cooldownCheck.reason}`)
return
}
// STEP 2: Load full trade context from database
const prisma = getPrismaClient()
const dbTrade = await prisma.trade.findUnique({
where: { id: mismatch.tradeId },
select: {
id: true,
symbol: true,
direction: true,
entryTime: true,
exitTime: true,
exitReason: true,
positionSizeUSD: true,
entryPrice: true,
configSnapshot: true,
}
})
if (!dbTrade) {
console.warn(` ⚠️ SAFETY: Trade ${mismatch.tradeId} not found in DB - skipping`)
return
}
// STEP 3: Verify Drift position exists and get full details
const driftService = await getDriftService()
const marketIndex = this.getMarketIndex(dbTrade.symbol)
if (marketIndex === null) {
console.warn(` ⚠️ SAFETY: Unknown market ${dbTrade.symbol} - skipping`)
return
}
const driftPosition = await driftService.getPosition(marketIndex)
if (!driftPosition || Math.abs(driftPosition.size) < 0.01) {
console.log(` ✅ RESOLVED: Position already closed on Drift`)
return
}
// STEP 4: CRITICAL VERIFICATION - Check if this is a NEW position
const verificationResult = await this.verifyPositionIdentity({
dbTrade,
driftPosition,
mismatch,
})
console.log(`\n 📊 VERIFICATION DECISION:`, JSON.stringify(verificationResult, null, 2))
if (!verificationResult.isOldGhost) {
console.warn(` ⚠️ PROTECTION TRIGGERED: ${verificationResult.reason}`)
console.warn(` 🛡️ Skipping close to protect potentially active position`)
// Log detailed protection event
await this.logProtectedPosition({
tradeId: dbTrade.id,
symbol: dbTrade.symbol,
reason: verificationResult.reason,
details: verificationResult.details,
})
return
}
// STEP 5: All checks passed - proceed with close
console.log(` ✅ VERIFIED OLD GHOST: Safe to close`)
console.log(` 📋 Evidence:`, verificationResult.details)
const attemptTime = Date.now()
this.recentCloseAttempts.set(dbTrade.symbol, attemptTime)
const result = await closePosition({
symbol: dbTrade.symbol,
percentToClose: 100,
slippageTolerance: 0.05,
})
if (result.success) {
console.log(` ✅ Orphan closed: ${result.transactionSignature}`)
console.log(` 💰 P&L: $${result.realizedPnL?.toFixed(2) || 0}`)
// Record successful cleanup
await prisma.trade.update({
where: { id: dbTrade.id },
data: {
exitOrderTx: result.transactionSignature || 'ORPHAN_CLEANUP',
realizedPnL: result.realizedPnL || 0,
configSnapshot: {
...dbTrade.configSnapshot as any,
orphanCleanup: true,
orphanCleanupTime: new Date(attemptTime).toISOString(),
verificationPassed: verificationResult.details,
}
}
})
} else {
console.error(` ❌ Close failed: ${result.error}`)
}
} catch (error) {
console.error(` ❌ Error in close verification:`, error)
this.recentCloseAttempts.set(mismatch.symbol, Date.now())
}
}
/**
* Check cooldown status for a symbol
*/
private async checkCooldown(symbol: string): Promise<{ canProceed: boolean; reason?: string }> {
// Check in-memory cooldown first
const lastAttemptTime = this.recentCloseAttempts.get(symbol)
if (lastAttemptTime) {
const timeSinceAttempt = Date.now() - lastAttemptTime
if (timeSinceAttempt < this.COOLDOWN_MS) {
const remaining = Math.ceil((this.COOLDOWN_MS - timeSinceAttempt) / 1000)
return {
canProceed: false,
reason: `Cooldown active: ${remaining}s remaining (last attempt ${(timeSinceAttempt/1000).toFixed(0)}s ago)`
}
}
}
// Check database cooldown (survives restarts)
const prisma = getPrismaClient()
const recentAttempt = await prisma.trade.findFirst({
where: {
symbol,
configSnapshot: {
path: ['orphanCleanupTime'],
not: Prisma.JsonNull,
}
},
orderBy: { updatedAt: 'desc' },
select: { configSnapshot: true }
})
if (recentAttempt?.configSnapshot) {
const snapshot = recentAttempt.configSnapshot as any
const lastCleanup = snapshot.orphanCleanupTime ? new Date(snapshot.orphanCleanupTime) : null
if (lastCleanup) {
const timeSince = Date.now() - lastCleanup.getTime()
if (timeSince < this.COOLDOWN_MS) {
return {
canProceed: false,
reason: `Database cooldown: ${Math.ceil((this.COOLDOWN_MS - timeSince)/1000)}s remaining`
}
}
}
}
return { canProceed: true }
}
/**
* CRITICAL: Verify if Drift position is an old ghost or new active trade
*
* Uses multiple verification methods:
* 1. Time-based: Position age vs DB exit time
* 2. Size-based: Position size vs DB recorded size
* 3. Grace period: Wait 10+ minutes after DB exit
* 4. Direction check: Must match DB direction
*
* FAIL-OPEN BIAS: When verification is uncertain, assume position is active
*/
private async verifyPositionIdentity(params: {
dbTrade: any
driftPosition: any
mismatch: DriftStateMismatch
}): Promise<{
isOldGhost: boolean
reason: string
details: Record<string, any>
}> {
const { dbTrade, driftPosition, mismatch } = params
// Grace period check: Has enough time passed since DB exit?
const GRACE_PERIOD_MS = 10 * 60 * 1000 // 10 minutes
const timeSinceExit = Date.now() - new Date(dbTrade.exitTime).getTime()
if (timeSinceExit < GRACE_PERIOD_MS) {
return {
isOldGhost: false,
reason: 'GRACE_PERIOD_ACTIVE',
details: {
timeSinceExitMin: (timeSinceExit / 60000).toFixed(1),
gracePeriodMin: 10,
message: 'Too soon after exit - may still be propagating'
}
}
}
// Direction check: Must match
const driftDirection = driftPosition.side // 'long' | 'short' | 'none'
if (driftDirection !== dbTrade.direction) {
return {
isOldGhost: false,
reason: 'DIRECTION_MISMATCH',
details: {
dbDirection: dbTrade.direction,
driftDirection,
message: 'Different direction = definitely different position'
}
}
}
// Size check: Must be within 15% tolerance
// (Allows for partial fills, funding rate impacts, etc.)
const dbSizeTokens = dbTrade.positionSizeUSD / dbTrade.entryPrice
const driftSizeTokens = driftPosition.size
const sizeRatio = Math.abs(driftSizeTokens) / Math.abs(dbSizeTokens)
if (sizeRatio < 0.85 || sizeRatio > 1.15) {
return {
isOldGhost: false,
reason: 'SIZE_MISMATCH',
details: {
dbSizeTokens: dbSizeTokens.toFixed(2),
driftSizeTokens: driftSizeTokens.toFixed(2),
sizeRatio: sizeRatio.toFixed(3),
tolerance: '0.85-1.15',
message: 'Size difference too large = likely different position'
}
}
}
// Position age estimation (best effort - no direct timestamp from SDK)
// We use entry price comparison as a proxy:
// - If Drift entry price significantly different from DB → likely new position
const priceDiffPercent = Math.abs(driftPosition.entryPrice - dbTrade.entryPrice) / dbTrade.entryPrice * 100
if (priceDiffPercent > 2.0) {
return {
isOldGhost: false,
reason: 'ENTRY_PRICE_MISMATCH',
details: {
dbEntryPrice: dbTrade.entryPrice.toFixed(2),
driftEntryPrice: driftPosition.entryPrice.toFixed(2),
diffPercent: priceDiffPercent.toFixed(2),
message: 'Entry price difference >2% suggests different position'
}
}
}
// Check if there are any newer trades on this symbol in DB
const prisma = getPrismaClient()
const newerTrades = await prisma.trade.findMany({
where: {
symbol: dbTrade.symbol,
exitReason: null, // Open trades
createdAt: { gt: new Date(dbTrade.exitTime) }
},
select: { id: true, createdAt: true }
})
if (newerTrades.length > 0) {
return {
isOldGhost: false,
reason: 'NEWER_TRADE_EXISTS',
details: {
newerTradeCount: newerTrades.length,
newerTradeIds: newerTrades.map(t => t.id),
message: 'DB shows newer open position on this symbol - Drift position likely belongs to it'
}
}
}
// ALL CHECKS PASSED - This appears to be an old ghost
return {
isOldGhost: true,
reason: 'VERIFIED_OLD_GHOST',
details: {
timeSinceExitMin: (timeSinceExit / 60000).toFixed(1),
directionMatch: true,
sizeRatio: sizeRatio.toFixed(3),
entryPriceDiff: priceDiffPercent.toFixed(2) + '%',
noNewerTrades: true,
message: 'All verification checks passed - safe to close'
}
}
}
/**
* Log when we protect a position from accidental closure
*/
private async logProtectedPosition(params: {
tradeId: string
symbol: string
reason: string
details: Record<string, any>
}): Promise<void> {
try {
const prisma = getPrismaClient()
await prisma.trade.update({
where: { id: params.tradeId },
data: {
configSnapshot: {
path: ['protectionEvents'],
arrayAppend: {
timestamp: new Date().toISOString(),
reason: params.reason,
details: params.details,
message: 'Position protected from accidental closure by Bug #82 fix'
}
}
}
}).catch(() => {
// Ignore errors updating protection log - not critical
})
} catch (error) {
// Silent failure - protection logging is supplementary
}
}
/**
* Send Telegram alert about mismatches
*/
private async sendMismatchAlert(mismatches: DriftStateMismatch[]): Promise<void> {
const criticalMismatches = mismatches.filter(m =>
m.expectedState === 'closed' && m.actualState === 'open'
)
if (criticalMismatches.length === 0) return
const message = `
🚨 DRIFT STATE MISMATCH ALERT
Found ${criticalMismatches.length} position(s) that should be closed but are still open on Drift:
${criticalMismatches.map(m => `
📊 ${m.symbol}
DB Status: Closed (${m.dbExitReason})
Drift Status: Open (${m.driftSize.toFixed(2)} tokens)
Time since exit: ${(m.timeSinceExit / 60000).toFixed(1)} minutes
⚠️ Retry close attempted automatically
`).join('\n')}
This indicates Drift Protocol state propagation issues.
Check Drift UI to verify actual position status.
`.trim()
try {
await sendTelegramMessage(message)
} catch (error) {
console.error('Failed to send Telegram alert:', error)
}
}
/**
* Get Drift market index from symbol
*/
private getMarketIndex(symbol: string): number | null {
const marketMap: Record<string, number> = {
'SOL-PERP': 0,
'BTC-PERP': 1,
'ETH-PERP': 2,
}
return marketMap[symbol] ?? null
}
}
// Singleton instance
let verifierInstance: DriftStateVerifier | null = null
export function getDriftStateVerifier(): DriftStateVerifier {
if (!verifierInstance) {
verifierInstance = new DriftStateVerifier()
}
return verifierInstance
}
export function startDriftStateVerifier(): void {
const verifier = getDriftStateVerifier()
verifier.start()
}