test.js

#!/usr/bin/env node


const handleError = require('./utils/handleError')
const INTERNAL_CACHE_FILE = 'tvl-adapter-repo/sdkInternalCache.json'

const deadChains = require('./projects/helper/deadChains')
process.on('unhandledRejection', handleError)
process.on('uncaughtException', handleError)

const path = require("path");
require("dotenv").config();
const { ENV_KEYS } = require("./projects/helper/env");
const { util: {
  blocks: { getBlocks },
  humanizeNumber: { humanizeNumber },
} } = require("@defillama/sdk");
const { util } = require("@defillama/sdk");
const sdk = require("@defillama/sdk");
const whitelistedExportKeys = require('./projects/helper/whitelistedExportKeys.json')
const chainList = require('./projects/helper/chains.json')
const { log, diplayUnknownTable, sliceIntoChunks, sleep } = require('./projects/helper/utils')
const { normalizeAddress } = require('./projects/helper/tokenMapping')
const { PromisePool } = require('@supercharge/promise-pool')
const { allProtocols } = require('./registries')

const currentCacheVersion = sdk.cache.currentVersion // load env for cache
// console.log(`Using cache version ${currentCacheVersion}`)

const whitelistedEnvKeys = new Set(['LLAMA_SANITIZE', 'TVL_LOCAL_CACHE_ROOT_FOLDER', 'LLAMA_DEBUG_MODE', 'INTERNAL_API_KEY', 'GRAPH_API_KEY', 'LLAMA_DEBUG_LEVEL2', 'LLAMA_INDEXER_V2_API_KEY', 'LLAMA_INDEXER_V2_ENDPOINT', 'LLAMA_RUN_LOCAL', ...ENV_KEYS])




if (process.env.LLAMA_SANITIZE)
  Object.keys(process.env).forEach((key) => {
    if (key.endsWith('_RPC')) return;
    if (whitelistedEnvKeys.has(key) || key.includes('SDK')) return;
    delete process.env[key]
  })
process.env.SKIP_RPC_CHECK = 'true'


async function getTvl(
  unixTimestamp,
  ethBlock,
  chainBlocks,
  usdTvls,
  tokensBalances,
  usdTokenBalances,
  tvlFunction,
  storedKey,
) {
  const chain = storedKey.split('-')[0]
  const api = new sdk.ChainApi({ chain, block: chainBlocks[chain], timestamp: unixTimestamp, storedKey, })
  api.api = api
  api.storedKey = storedKey

  let tvlBalances = await tvlFunction(api, ethBlock, chainBlocks, api);
  if (tvlBalances === undefined) tvlBalances = api.getBalances()
  const tvlResults = await computeTVL(tvlBalances, unixTimestamp);
  await diplayUnknownTable({ tvlResults, storedKey, tvlBalances, })
  usdTvls[storedKey] = tvlResults.usdTvl;
  tokensBalances[storedKey] = tvlResults.tokenBalances;
  usdTokenBalances[storedKey] = tvlResults.usdTokenBalances;

  if (
    typeof usdTvls[storedKey] !== "number" ||
    Number.isNaN(usdTvls[storedKey])
  ) {
    throw new Error(
      `TVL for key ${storedKey} is not a number, instead it is ${usdTvls[storedKey]}`
    );
  }
}

function mergeBalances(key, storedKeys, balancesObject) {
  if (balancesObject[key] === undefined) {
    balancesObject[key] = {};
    storedKeys.map((keyToMerge) => {
      Object.entries(balancesObject[keyToMerge]).forEach((balance) => {
        try {
          util.sumSingleBalance(balancesObject[key], balance[0], BigNumber(balance[1] || '0').toFixed(0));
        } catch (e) {
          console.log(e)
        }
      });
    });
  }
}

if (process.argv.length < 3) {
  console.error(`Missing argument, you need to provide the filename of the adapter to test.
    Eg: node test.js projects/myadapter.js`);
  process.exit(1);
}
let passedFile = path.resolve(process.cwd(), process.argv[2]);

const originalCall = sdk.api.abi.call
sdk.api.abi.call = async (...args) => {
  try {
    return await originalCall(...args)
  } catch (e) {
    console.log("sdk.api.abi.call errored with params:", args)
    throw e
  }
}

function validateHallmarks(hallmark) {
  if (!Array.isArray(hallmark)) {
    throw new Error("Hallmarks should be an array of [unixTimestamp, eventText] but got " + JSON.stringify(hallmark))
  }
  const [timestamp, text] = hallmark
  if ((typeof timestamp !== 'string' && !process.env.LLAMA_SANITIZE) || isNaN(+new Date(timestamp))) {
    throw new Error("Hallmark timestamp should be a dateString (YYYY-MM-DD)")
  }
  const year = new Date(timestamp * 1000).getFullYear()
  const currentYear = new Date().getFullYear()
  if (year < 2010 || year > currentYear) {
    throw new Error("Hallmark timestamp should be between 2010 and " + currentYear + " but got " + year)
  }

  if (typeof text !== 'string') {
    throw new Error("Hallmark text should be a string")
  }
}

(async () => {

  const moduleArg = process.argv[2].replace('/index.js', '').split('/').pop()
  // registry keys can be namespaced (e.g. `treasury/gmx`); preserve the full
  // key so a treasury entry isn't shadowed by a same-named protocol adapter.
  const registryKey = process.argv[2].replace(/\.js$/, '').replace(/\/index$/, '').replace(/^projects\//, '')

  // throw error if module doesnt start with lowercase letters
  if (!/^[a-z]/.test(moduleArg) && !process.env.LLAMA_RUN_LOCAL) {
    throw new Error("Module name should start with a lowercase letter: " + moduleArg);
  }

  let module = {};
  try {
    module = require(passedFile)
  } catch (e) {
    const registryName = allProtocols[registryKey] ? registryKey : moduleArg
    if (allProtocols[registryName]) {
      module = allProtocols[registryName]
      passedFile = `registry:${registryName}`
      console.log(`Loaded module ${registryName} from registry`)
    } else {
      console.error("Error loading module:", e)
      return handleError(e)
    }
  }
  deadChains.forEach(chain => {
    delete module[chain]
  })

  if (module.hallmarks) {
    if (!Array.isArray(module.hallmarks)) {
      throw new Error("Hallmarks should be an array of arrays")
    }
    if (module.hallmarks.length > 6) {
      console.error("WARNING: Hallmarks should only be set for events that led to a big change in TVL, please reduce hallmarks to only those that meet this condition")
    }

    module.hallmarks.forEach(validateHallmarks)
  }
  // await initCache()
  const chains = Object.keys(module).filter(item => typeof module[item] === 'object' && !Array.isArray(module[item]));
  checkExportKeys(module, passedFile, chains)

  let unixTimestamp = Math.round(Date.now() / 1000) - 60;
  let chainBlocks = {}

  function parseTimestampArg(arg) {
    if (!arg) return undefined;
    // Accept pure numeric input as unix seconds or milliseconds
    if (/^\d+$/.test(arg)) {
      const num = Number(arg);
      if (!Number.isFinite(num)) return undefined;
      // 13+ digits -> milliseconds, else assume seconds
      return num > 1e12 ? Math.floor(num / 1000) : num;
    }
    // Fallback to Date string parsing (e.g., YYYY-MM-DD)
    const d = new Date(arg);
    const ms = d.getTime();
    if (!Number.isFinite(ms)) return undefined;
    return Math.floor(ms / 1000);
  }

  const passedTimestamp = parseTimestampArg(process.argv[3]);
  if (passedTimestamp !== undefined) {
    unixTimestamp = passedTimestamp

    // other chains than evm will fail to get block at timestamp
    try {
      const res = await getBlocks(unixTimestamp, chains)
      chainBlocks = res.chainBlocks
    } catch (e) { /* ignore empty block statement */ }
  }
  const ethBlock = chainBlocks.ethereum;
  const usdTvls = {};
  const tokensBalances = {};
  const usdTokenBalances = {};
  const chainTvlsToAdd = {};
  const failedKeys = [];

  let tvlPromises = Object.entries(module).map(([chain, value]) => {
    if (typeof value !== "object" || value === null) {
      return;
    }

    return Object.entries(value).map(([tvlType, tvlFunction]) => {
      if (typeof tvlFunction !== "function") {
        return;
      }
      let storedKey = `${chain}-${tvlType}`;
      if (tvlType === "tvl") {
        storedKey = chain;
      }

      return async () => {
        try {
          await getTvl(unixTimestamp, ethBlock, chainBlocks, usdTvls, tokensBalances, usdTokenBalances, tvlFunction, storedKey,);
        } catch (e) {
          failedKeys.push({ storedKey, error: e && e.message ? e.message : String(e) })
          console.error(`Error pulling TVL for ${storedKey}:`, e)
          return;
        }
        let keyToAddChainBalances = tvlType;
        if (tvlType === "tvl")
          keyToAddChainBalances = "tvl";

        if (chainTvlsToAdd[keyToAddChainBalances] === undefined) {
          chainTvlsToAdd[keyToAddChainBalances] = [storedKey];
        } else {
          chainTvlsToAdd[keyToAddChainBalances].push(storedKey);
        }

      }
    })
  }).filter(p => p !== undefined && Array.isArray(p)).flat().filter(p => p !== undefined)

  await util.runInPromisePool({
    items: tvlPromises,
    concurrency: 5,
    processor: async (p) => p(),
    permitFailure: false,
  })
  Object.entries(chainTvlsToAdd).map(([tvlType, storedKeys]) => {
    if (usdTvls[tvlType] === undefined) {
      usdTvls[tvlType] = storedKeys.reduce(
        (total, key) => total + usdTvls[key],
        0
      );
      mergeBalances(tvlType, storedKeys, tokensBalances);
      mergeBalances(tvlType, storedKeys, usdTokenBalances);
    }
  });
  if (usdTvls.tvl === undefined) {
    if (failedKeys.length) {
      console.error(`\n------ FAILED (${failedKeys.length}) ------`)
      failedKeys.forEach(({ storedKey, error }) => console.error(storedKey.padEnd(25, " "), error))
      throw new Error(
        "Protocol doesn't have total tvl, all the tvl functions above failed to run"
      );
    }
    throw new Error(
      "Protocol doesn't have total tvl, make sure to export a tvl key either on the main object or in one of the chains"
    );
  }

  if (warningTable.length) {
    console.log(`WARNING: Found ${warningTable.length} tokens with a very high USD value, please check the following table for more details: `)
    console.table(warningTable)
  }

  Object.entries(usdTokenBalances).forEach(([chain, balances]) => {
    console.log(`--- ${chain} ---`);
    let entries = Object.entries(balances)
    entries.sort((a, b) => b[1] - a[1])

    if (entries.length > 30) {
      console.log("Showing top 30 tokens, total tokens:", entries.length)
      entries = entries.slice(0, 30)
    }
    entries.forEach(([symbol, balance]) => {
      console.log(symbol.padEnd(25, " "), humanizeNumber(balance));
    });
    console.log("Total:", humanizeNumber(usdTvls[chain]), "\n");
  });
  console.log(`------ TVL ------`);
  const usdVals = Object.entries(usdTvls)
  usdVals.sort((a, b) => b[1] - a[1])
  usdVals.forEach(([chain, usdTvl]) => {
    if (chain !== "tvl") {
      console.log(chain.padEnd(25, " "), humanizeNumber(Math.round(usdTvl)));
    }
  });
  console.log("\ntotal".padEnd(25, " "), humanizeNumber(usdTvls.tvl), "\n");

  if (failedKeys.length) {
    console.error(`------ FAILED (${failedKeys.length}) ------`)
    failedKeys.forEach(({ storedKey, error }) => console.error(storedKey.padEnd(25, " "), error))
    console.error("\nNote: total TVL above excludes the failed chains/components listed here.\n")
  }

  await preExit()
  process.exit(0);
})().catch(handleError);


function checkExportKeys(module, filePath, chains) {
  let _filePath = filePath
  filePath = filePath.split(path.sep)
  filePath = filePath.slice(filePath.lastIndexOf('projects') + 1)

  if (!_filePath.startsWith('registry:') &&
    (filePath.length > 2
      || (filePath.length === 1 && !['.js', ''].includes(path.extname(filePath[0]))) // matches .../projects/projectXYZ.js or .../projects/projectXYZ
      || (filePath.length === 2 &&
        !(['api.js', 'index.js', 'apiCache.js',].includes(filePath[1])  // matches .../projects/projectXYZ/index.js
          || ['treasury', 'entities'].includes(filePath[0])  // matches .../projects/treasury/project.js
          || /v\d+\.js$/.test(filePath[1]) // matches .../projects/projectXYZ/v1.js
        )))

  )
    process.exit(0)

  const blacklistedRootExportKeys = ['tvl', 'staking', 'pool2', 'borrowed', 'treasury', 'offers', 'vesting'];
  const rootexportKeys = Object.keys(module).filter(item => typeof module[item] !== 'object');

  const badChainNames = chains.filter(chain => !/^[a-z0-9_]+$/.test(chain));
  if (badChainNames.length) {
    throw new Error(`
    Invalid chain names: ${badChainNames.join(', ')}
    Chain names should only contain lowercase letters, numbers and underscores
    `)
  }

  const unknownChains = chains.filter(chain => !chainList.includes(chain));
  const blacklistedKeysFound = rootexportKeys.filter(key => blacklistedRootExportKeys.includes(key));
  let exportKeys = chains.map(chain => Object.keys(module[chain])).flat()
  exportKeys.push(...rootexportKeys)
  exportKeys = Object.keys(exportKeys.reduce((agg, key) => ({ ...agg, [key]: 1 }), {})) // get unique keys
  const unknownKeys = exportKeys.filter(key => !whitelistedExportKeys.includes(key))

  const hallmarks = module.hallmarks || [];

  if (hallmarks.length) {
    const TIMESTAMP_LENGTH = 10;
    hallmarks.forEach(([timestamp, text]) => {
      if (Array.isArray(timestamp)) timestamp.map(validateDateString)  // it is a range timestamp [start, end]
      else validateDateString(timestamp);

      function validateDateString(timestamp) {
        const strTimestamp = String(timestamp)
        if (strTimestamp.length !== TIMESTAMP_LENGTH) {
          throw new Error(`
        Incorrect time format for the hallmark: [${strTimestamp}, ${text}] ,please use unix timestamp
        `)
        }
      }
    })
  }


  if (unknownChains.length) {
    throw new Error(`
    Unknown chain(s): ${unknownChains.join(', ')}
    Note: if you think that the chain is correct but missing from our list, please add it to 'projects/helper/chains.json' file
    `)
  }

  if (blacklistedKeysFound.length) {
    throw new Error(`
    Please move the following keys into the chain: ${blacklistedKeysFound.join(', ')}

    We have a new adapter export specification now where tvl and other chain specific information are moved inside chain export.
    For example if your protocol is on ethereum and has tvl and pool2, the export file would look like:
    
        module.exports = {
          methodlogy: '...',
          ethereum: {
            tvl: 
            pool2:
          }
        }

    `)
  }

  if (unknownKeys.length) {
    throw new Error(`
    Found export keys that were not part of specification: ${unknownKeys.join(', ')}

    List of valid keys: ${['', '', ...whitelistedExportKeys].join('\n\t\t\t\t')}
    `)
  }
}


process.on('unhandledRejection', handleError)
process.on('uncaughtException', handleError)


const BigNumber = require("bignumber.js");
const axios = require("axios");

const ethereumAddress = "0x0000000000000000000000000000000000000000";
const weth = "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2";
function fixBalances(balances) {

  if (balances.usd) {
    // usd is mapped to USD+ token on coingecko
    throw new Error("Balance key 'usd' is not allowed, please use 'api.addUSDValue()' instead")
  }

  Object.entries(balances).forEach(([token, value]) => {
    let newKey
    if (token.startsWith("0x")) newKey = `ethereum:${token}`
    else if (!token.includes(':')) newKey = `coingecko:${token}`
    if (newKey) {
      delete balances[token]
      sdk.util.sumSingleBalance(balances, newKey, value)
    }
  })
}

const warningTable = []

const confidenceThreshold = 0.5
async function computeTVL(balances, timestamp) {
  fixBalances(balances)

  Object.keys(balances).map(k => {
    const balance = balances[k]
    delete balances[k]
    if (+balance === 0)
      return;
    const normalizedAddress = normalizeAddress(k, undefined, true)
    sdk.util.sumSingleBalance(balances, normalizedAddress, balance)
  })

  const eth = balances[ethereumAddress];
  if (eth !== undefined) {
    balances[weth] = new BigNumber(balances[weth] ?? 0).plus(eth).toFixed(0);
    delete balances[ethereumAddress];
  }

  const PKsToTokens = {};
  const readKeys = Object.keys(balances)
    .map((address) => {
      const PK = address;
      if (PKsToTokens[PK] === undefined) {
        PKsToTokens[PK] = [address];
        return PK;
      } else {
        PKsToTokens[PK].push(address);
        return undefined;
      }
    })
    .filter((item) => item !== undefined);

  const unknownTokens = {}
  let tokenData = []
  readKeys.forEach(i => unknownTokens[i] = true)

  const queries = buildPricesGetQueries(readKeys, timestamp)
  let queryCount = queries.length;
  if (queryCount > 7)
    sdk.log('price query count:', queryCount, 'readKeys:', readKeys.length)
  let i = 0
  let foundError = false
  const { errors } = await PromisePool.withConcurrency(3)
    .for(queries).process(async (query) => {
      if (foundError) return;
      try {

        tokenData.push((await axios.get(query)).data.coins)
        i++;
        if (queries.length > 6) {
          sdk.log(`Processed ${i}/${queryCount} queries, ${tokenData.length} responses received`)
          if (i < queryCount - 2) {
            await sleep(5000) // avoid rate limiting
          }
        }
      } catch (e) {
        foundError = true;
        throw e
      }
    })

  if (errors && errors.length)
    throw errors[0]

  let usdTvl = 0;
  const tokenBalances = {};
  const usdTokenBalances = {};

  tokenData.forEach(response => {
    Object.keys(response).forEach(address => {
      delete unknownTokens[address]
      const data = response[address];
      const balance = balances[address];

      if (data == undefined) tokenBalances[`UNKNOWN (${address})`] = balance
      if (data.symbol === '') {
        console.log('\nIgnored invalid coin data, please fix it!', address, data, '\n');
        return;
      }
      if ('confidence' in data && data.confidence < confidenceThreshold || !data.price) return
      if (Math.abs(data.timestamp - (timestamp ?? Date.now() / 1e3)) > (24 * 3600)) {
        console.log(`Price for ${address} is stale, ignoring...`)
        return
      }

      let amount, usdAmount;
      if (address.includes(":") && !address.startsWith("coingecko:")) {
        amount = Number(balance) / 10 ** data.decimals
        usdAmount = amount * data.price;
      } else {
        amount = Number(balance);
        usdAmount = amount * data.price;
      }

      if (Math.abs(usdAmount) > 1e8) {
        const chain = address.split(':')[0]
        warningTable.push({ chain, symbol: data.symbol, usdAmount: humanizeNumber(usdAmount), amount: humanizeNumber(amount), price: data.price, address, confidence: data.confidence, decimals: data.decimals, })
      }


      tokenBalances[data.symbol] = (tokenBalances[data.symbol] ?? 0) + amount;
      usdTokenBalances[data.symbol] = (usdTokenBalances[data.symbol] ?? 0) + usdAmount;
      usdTvl += usdAmount;
      if (isNaN(usdTvl)) {
        throw new Error(`NaN usdTvl for ${address} with balance ${balance} and price ${data.price}`)
      }
    })
  });

  Object.keys(unknownTokens).forEach(address => tokenBalances[`UNKNOWN (${address})`] = balances[address])


  // console.log('--------token balances-------')
  // console.table(tokenBalances)

  return {
    usdTvl,
    tokenBalances,
    usdTokenBalances,
  };
}

setTimeout(() => {
  console.log("Timeout reached, exiting...");
  if (!process.env.NO_EXIT_ON_LONG_RUN_RPC)
    process.exit(1);
}, 10 * 60 * 1000) // 10 minutes

function buildPricesGetQueries(readKeys, timestamp) {
  if (!readKeys.length) return []
  console.log(`Building prices get queries for ${readKeys.length} tokens`)
  const burl = (process.env.INTERNAL_API_KEY ? `https://pro-api.llama.fi/${process.env.INTERNAL_API_KEY}/coins/` : 'https://coins.llama.fi/')
    + (timestamp && timestamp < (Date.now() / 1000 - 30 * 60) ? `prices/historical/${timestamp}/` : 'prices/current/')
  const queries = []
  let query = burl

  for (const key of readKeys) {
    if (query.length + key.length > 2500) {
      queries.push(query.slice(0, -1))
      query = burl
    }
    query += `${key},`
  }

  queries.push(query.slice(0, -1))
  return queries
}

async function initCache() {
  let currentCache = await sdk.cache.readCache(INTERNAL_CACHE_FILE)
  // if (process.env.NO_EXIT_ON_LONG_RUN_RPC)
  //   sdk.log('cache size:', JSON.stringify(currentCache).length, 'chains:', Object.keys(currentCache).length)
  const ONE_WEEK = 60 * 60 * 24 * 31
  if (!currentCache || !currentCache.startTime || (Date.now() / 1000 - currentCache.startTime > ONE_WEEK)) {
    currentCache = {
      startTime: Math.round(Date.now() / 1000),
    }
    await sdk.cache.writeCache(INTERNAL_CACHE_FILE, currentCache)
  }
  sdk.sdkCache.startCache(currentCache)
}

async function saveSdkInternalCache() {
  await sdk.cache.writeCache(INTERNAL_CACHE_FILE, sdk.sdkCache.retriveCache(), { skipR2CacheWrite: true })
}

async function preExit() {
  // try {
  //     await saveSdkInternalCache() // save sdk cache to r2
  // } catch (e) {
  //   if (process.env.NO_EXIT_ON_LONG_RUN_RPC)
  //     sdk.error(e)
  // }
}