Skip to content

Commit

Permalink
Fix spl token indexer parsing (#4460)
Browse files Browse the repository at this point in the history 
* Fix spl token indexer parsing

* remove log

* better logging

* PR Comments

* Don't redefine sender_idx

* logger.error instead of info
  • Loading branch information
@dharit-tan
dharit-tan committed Dec 14, 2022
1 parent ebcfb7f commit de09f20
Show file tree
Hide file tree
Showing 4 changed files with 302 additions and 60 deletions.
175 changes: 170 additions & 5 deletions discovery-provider/integration_tests/tasks/test_index_spl_token.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,25 @@
}
],
"preTokenBalances": [],
"logMessages": [
"Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL invoke [1]",
"Program log: Transfer 2039280 lamports to the associated token account",
"Program 11111111111111111111111111111111 invoke [2]",
"Program 11111111111111111111111111111111 success",
"Program log: Allocate space for the associated token account",
"Program 11111111111111111111111111111111 invoke [2]",
"Program 11111111111111111111111111111111 success",
"Program log: Assign the associated token account to the SPL Token program",
"Program 11111111111111111111111111111111 invoke [2]",
"Program 11111111111111111111111111111111 success",
"Program log: Initialize the associated token account",
"Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [2]",
"Program log: Instruction: InitializeAccount",
"Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3272 of 176939 compute units",
"Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success",
"Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL consumed 27014 of 200000 compute units",
"Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL success",
],
},
"transaction": {
"message": {
Expand Down Expand Up @@ -214,14 +233,14 @@ def test_parse_spl_token_transaction():
tx_info = parse_spl_token_transaction(
solana_client_manager_mock, mock_confirmed_signature_for_address
)
assert tx_info["user_bank"] == "9f79QvW5XQ1XCXGLeCCHjBZkPet51yAPxtU7fcaY6UxD"
assert tx_info["user_bank"] == "7CyoHxibpPrTVc2AsmoSq7gRoDwnwN7LRnHDcR4yWVf9"
assert tx_info["root_accounts"] == [
"5ZiE3vAkrdXBgyFL7KqG3RoEGBws4CjRcXVbABDLZTgx",
"iVsXfN4oZnARo6AYwm8FFXBnDQYnwhkPxJiuPHDzfJ4",
"5ZiE3vAkrdXBgyFL7KqG3RoEGBws4CjRcXVbABDLZTgx",
]
assert tx_info["token_accounts"] == [
"7CyoHxibpPrTVc2AsmoSq7gRoDwnwN7LRnHDcR4yWVf9",
"9f79QvW5XQ1XCXGLeCCHjBZkPet51yAPxtU7fcaY6UxD",
"7CyoHxibpPrTVc2AsmoSq7gRoDwnwN7LRnHDcR4yWVf9",
]


Expand Down Expand Up @@ -341,12 +360,128 @@ def test_parse_spl_token_transaction():
},
}

mock_purchase_tx_info = {
mock_purchase_meta_different_ordering = {
"blockTime": 1667410823,
"meta": {
"err": None,
"fee": 5000,
"innerInstructions": [],
"loadedAddresses": {"readonly": [], "writable": []},
"logMessages": [
"Program Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo invoke [1]",
"Program Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo consumed 588 of 400000 compute units",
"Program Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo success",
"Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [1]",
"Program log: Instruction: TransferChecked",
"Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 6172 of 399412 compute units",
"Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success",
],
"postBalances": [2930160, 2039280, 2039280, 260042654, 121159680, 934087680],
"postTokenBalances": [
{
"accountIndex": 1,
"mint": "9LzCMqDgTKYz9Drzqnpgee3SGa89up3a247ypMj2xrqM",
"owner": "AEty8wg5HqCJz7a8U8FS9sYXtCudYH93JLZgMfjCAR6u",
"programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA",
"uiTokenAmount": {
"amount": "0",
"decimals": 8,
"uiAmount": None,
"uiAmountString": "0",
},
},
{
"accountIndex": 2,
"mint": "9LzCMqDgTKYz9Drzqnpgee3SGa89up3a247ypMj2xrqM",
"owner": "5ZiE3vAkrdXBgyFL7KqG3RoEGBws4CjRcXVbABDLZTgx",
"programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA",
"uiTokenAmount": {
"amount": "12780604010",
"decimals": 8,
"uiAmount": 127.8060401,
"uiAmountString": "127.8060401",
},
},
],
"preBalances": [2935160, 2039280, 2039280, 260042654, 121159680, 934087680],
"preTokenBalances": [
{
"accountIndex": 1,
"mint": "9LzCMqDgTKYz9Drzqnpgee3SGa89up3a247ypMj2xrqM",
"owner": "AEty8wg5HqCJz7a8U8FS9sYXtCudYH93JLZgMfjCAR6u",
"programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA",
"uiTokenAmount": {
"amount": "1025896376",
"decimals": 8,
"uiAmount": 10.25896376,
"uiAmountString": "10.25896376",
},
},
{
"accountIndex": 2,
"mint": "9LzCMqDgTKYz9Drzqnpgee3SGa89up3a247ypMj2xrqM",
"owner": "5ZiE3vAkrdXBgyFL7KqG3RoEGBws4CjRcXVbABDLZTgx",
"programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA",
"uiTokenAmount": {
"amount": "11754707634",
"decimals": 8,
"uiAmount": 117.54707634,
"uiAmountString": "117.54707634",
},
},
],
"rewards": [],
"status": {"Ok": None},
},
"slot": 158882288,
"transaction": {
"message": {
"header": {
"numReadonlySignedAccounts": 0,
"numReadonlyUnsignedAccounts": 3,
"numRequiredSignatures": 1,
},
"accountKeys": [
"AEty8wg5HqCJz7a8U8FS9sYXtCudYH93JLZgMfjCAR6u",
"8aEJ32LCGaGbNbidd1Gg33yqvZ11AXfAzzEAXxAJEzA8",
"HTmKqU5T3uhzz6heG47awyVuzcbWu9o4rE1HtrXv5ACg",
"9LzCMqDgTKYz9Drzqnpgee3SGa89up3a247ypMj2xrqM",
"Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo",
"TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA",
],
"recentBlockhash": "23SZfDKbmSqtHaR9F2UMkRzTvvoTo8NT7oRH5VB3D5xh",
"instructions": [
{
"accounts": [0],
"data": "Bs2CZBUGWJZV5kqF3ecfJisidP9WQtCpeeWCzk6AUyYLQWgLdHPz",
"programIdIndex": 4,
},
{
"accounts": [1, 3, 2, 0],
"data": "iTUiLjKABVyv7",
"programIdIndex": 5,
},
],
"indexToProgramIds": {},
},
"signatures": [
"iBTtH8W1ViHdnAzQYNKovYsBszHvikwUAG4V8Qk5HNWQRCYc468ugKnoYSqL6f7LLmbKa5ZYFSLaEbZuH42fbBM"
],
},
}

mock_purchase_tx_info_1 = {
"jsonrpc": "2.0",
"result": mock_purchase_meta,
"id": 4,
}

mock_purchase_tx_info_2 = {
"jsonrpc": "2.0",
"result": mock_purchase_meta_different_ordering,
"id": 4,
}


def test_parse_memo_instruction():
memo = parse_memo_instruction(mock_transfer_checked_meta)
Expand All @@ -357,6 +492,36 @@ def test_parse_memo_instruction():
assert vendor == "Link by Stripe"


def test_parse_spl_token_purchase_transactions():
solana_client_manager_mock = create_autospec(SolanaClientManager)
solana_client_manager_mock.get_sol_tx_info.return_value = mock_purchase_tx_info_1
tx_info = parse_spl_token_transaction(
solana_client_manager_mock, mock_confirmed_signature_for_address
)
assert tx_info["user_bank"] == "7dw7W4Yv7F1uWb9dVH1CFPm39mePyypuCji2zxcFA556"
assert tx_info["root_accounts"] == [
"GNkt1SGkdzvfaCYVpSKs7yjLxeyLJFKZv32cVYJ3GyHX",
"5ZiE3vAkrdXBgyFL7KqG3RoEGBws4CjRcXVbABDLZTgx",
]
assert tx_info["token_accounts"] == [
"C4qrWm6kkLwUNExA2Ldt6uXLroRfcTGXJLx1zGvEt5DB",
"7dw7W4Yv7F1uWb9dVH1CFPm39mePyypuCji2zxcFA556",
]
solana_client_manager_mock.get_sol_tx_info.return_value = mock_purchase_tx_info_2
tx_info = parse_spl_token_transaction(
solana_client_manager_mock, mock_confirmed_signature_for_address
)
assert tx_info["user_bank"] == "HTmKqU5T3uhzz6heG47awyVuzcbWu9o4rE1HtrXv5ACg"
assert tx_info["root_accounts"] == [
"AEty8wg5HqCJz7a8U8FS9sYXtCudYH93JLZgMfjCAR6u",
"5ZiE3vAkrdXBgyFL7KqG3RoEGBws4CjRcXVbABDLZTgx",
]
assert tx_info["token_accounts"] == [
"8aEJ32LCGaGbNbidd1Gg33yqvZ11AXfAzzEAXxAJEzA8",
"HTmKqU5T3uhzz6heG47awyVuzcbWu9o4rE1HtrXv5ACg",
]


def test_fetch_and_parse_sol_rewards_transfer_instruction(app): # pylint: disable=W0621
with app.app_context():
db = get_db()
Expand Down Expand Up @@ -442,7 +607,7 @@ def test_fetch_and_parse_sol_rewards_transfer_instruction(app): # pylint: disab
"blockTime": 1665685554,
"confirmationStatus": "finalized",
}
solana_client_manager_mock.get_sol_tx_info.return_value = mock_purchase_tx_info
solana_client_manager_mock.get_sol_tx_info.return_value = mock_purchase_tx_info_1
parse_sol_tx_batch(
db,
solana_client_manager_mock,
Expand Down
57 changes: 43 additions & 14 deletions discovery-provider/src/tasks/index_spl_token.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
TX_SIGNATURES_RESIZE_LENGTH,
)
from src.solana.solana_client_manager import SolanaClientManager
from src.solana.solana_helpers import get_base_address
from src.solana.solana_helpers import SPL_TOKEN_ID, get_base_address
from src.solana.solana_transaction_types import (
ConfirmedSignatureForAddressResult,
ConfirmedTransaction,
Expand All @@ -38,7 +38,13 @@
fetch_and_cache_latest_program_tx_redis,
)
from src.utils.config import shared_config
from src.utils.helpers import get_solana_tx_owner, get_solana_tx_token_balances
from src.utils.helpers import (
get_account_index,
get_solana_tx_owner,
get_solana_tx_token_balances,
get_valid_instruction,
has_log,
)
from src.utils.prometheus_metric import save_duration_metric
from src.utils.redis_constants import (
latest_sol_spl_token_db_key,
Expand All @@ -52,6 +58,7 @@
USER_BANK_ADDRESS = shared_config["solana"]["user_bank_program_address"]
USER_BANK_PUBKEY = PublicKey(USER_BANK_ADDRESS) if USER_BANK_ADDRESS else None
PURCHASE_AUDIO_MEMO_PROGRAM = "Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo"
TRANSFER_CHECKED_INSTRUCTION = "Program log: Instruction: TransferChecked"

REDIS_TX_CACHE_QUEUE_PREFIX = "spl-token-tx-cache-queue"

Expand All @@ -65,10 +72,10 @@
# Index of receiver account in solana transaction pre/post balances
# Note: the receiver index is currently the same for purchase and transfer instructions
# but this assumption could change in the future.
RECEIVER_ACCOUNT_INDEX = 1
# Thought we don't index transfers from the sender's side in this task, we must still
RECEIVER_ACCOUNT_INDEX = 2
# Though we don't index transfers from the sender's side in this task, we must still
# enqueue the sender's accounts for balance refreshes if they are Audius accounts.
SENDER_ACCOUNT_INDEX = 2
SENDER_ACCOUNT_INDEX = 0

purchase_vendor_map = {
"Link by Stripe": TransactionType.purchase_stripe,
Expand Down Expand Up @@ -148,20 +155,42 @@ def parse_spl_token_transaction(
if error:
return None

has_transfer_checked_instruction = has_log(meta, TRANSFER_CHECKED_INSTRUCTION)
if not has_transfer_checked_instruction:
logger.error(
f"index_spl_token.py | {tx_sig} No transfer checked instruction found"
)
return None
tx_message = result["transaction"]["message"]
instruction = get_valid_instruction(tx_message, meta, SPL_TOKEN_ID)
if not instruction:
logger.error(
f"index_spl_token.py | {tx_sig} No valid instruction for spl token program found"
)
return None

memo_encoded = parse_memo_instruction(result)
vendor = decode_memo_and_extract_vendor(memo_encoded) if memo_encoded else None

sender_root_account = get_solana_tx_owner(meta, SENDER_ACCOUNT_INDEX)
receiver_root_account = get_solana_tx_owner(meta, RECEIVER_ACCOUNT_INDEX)
account_keys = result["transaction"]["message"]["accountKeys"]
receiver_token_account = account_keys[RECEIVER_ACCOUNT_INDEX]
sender_token_account = account_keys[SENDER_ACCOUNT_INDEX]
prebalance, postbalance = get_solana_tx_token_balances(
meta, RECEIVER_ACCOUNT_INDEX
)
# Skip if there is no balance change.
sender_idx = get_account_index(instruction, SENDER_ACCOUNT_INDEX)
receiver_idx = get_account_index(instruction, RECEIVER_ACCOUNT_INDEX)
account_keys = tx_message["accountKeys"]
sender_token_account = account_keys[sender_idx]
receiver_token_account = account_keys[receiver_idx]
sender_root_account = get_solana_tx_owner(meta, sender_idx)
receiver_root_account = get_solana_tx_owner(meta, receiver_idx)
prebalance, postbalance = get_solana_tx_token_balances(meta, receiver_idx)

# Balance is expected to change if there is a transfer instruction.
if postbalance == -1 or prebalance == -1:
logger.error(
f"index_spl_token.py | {tx_sig} error while parsing pre and post balances"
)
return None
if postbalance - prebalance == 0:
logger.error(f"index_spl_token.py | {tx_sig} no balance change found")
return None

receiver_spl_tx_info: SplTokenTransactionInfo = {
"user_bank": receiver_token_account,
"signature": tx_sig["signature"],
Expand Down

0 comments on commit de09f20

Please sign in to comment.