Git-Mirrors/blockchain-data-engineering-toolkit
1
0
Fork 0
mirror of https://github.com/autistic-symposium/blockchain-data-engineering-toolkit.git synced 2025-04-25 10:19:13 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix api url

Browse Source
This commit is contained in:
osiris account 2023-03-12 12:57:18 -07:00
parent 8b46c0c6b2
commit 468b86b2c9
8 changed files with 110 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
token-scanner-api/.env.example
View File

@ -32,3 +32,8 @@ MONGODB_DB_NAME = balances
MONGODB_COLLECTION_NAME = balances
#########################
##### api settings
#########################
API_HOST_URL = http://localhost
API_HOST_URL_PORT = 80

8
token-scanner-api/src/blockchains/ethereum.py
View File

@ -12,15 +12,15 @@ from web3.exceptions import BlockNotFound
from web3.providers.rpc import HTTPProvider
from src.utils.arithmetics import convert_hex_to_int
from src.utils.os_utils import send_rpc_request, exit_with_error, create_result_file, \
set_output, open_json, load_config, save_output, log_info
from src.utils.os_utils import send_rpc_request, exit_with_error, log_info, \
set_output, open_json, save_output, create_result_file
class TokenIndexer:
def __init__(self):
def __init__(self, env_vars):
self.env_vars = load_config()
self.env_vars = env_vars
self.web3 = self._set_web3_object()
# contract parameters

20
token-scanner-api/src/main.py
View File

@ -7,9 +7,9 @@ import uvicorn
import argparse
from src.utils.os_utils import load_config
from src.utils.db_processing import populate_db
from src.blockchains.ethereum import TokenIndexer
from src.utils.vercel_utils import upload_to_vercel
from src.utils.db_processing import run_db_processing
from src.utils.test_api import fetch_token_balance as f
from src.utils.data_processing import run_data_processing
@ -50,7 +50,7 @@ def run_menu() -> argparse.ArgumentParser:
def run() -> None:
"""Entry point for this module."""
load_config()
env_vars = load_config()
parser = run_menu()
args = parser.parse_args()
@ -58,18 +58,21 @@ def run() -> None:
# Run historical data indexer
#############################
if args.indexer:
indexer = TokenIndexer()
indexer = TokenIndexer(env_vars)
indexer.run()
elif args.process:
run_data_processing(args.process[0])
elif args.db:
populate_db(args.db[0])
run_db_processing(args.db[0], env_vars)
#############################
# Run deployment tools
#############################
elif args.api:
uvicorn.run("src.server.api:app", host="0.0.0.0", port=8000, reload=True)
uvicorn.run("src.server.api:app", \
host=env_vars['API_HOST_URL'], \
port=env_vars['API_HOST_PORT'], \
reload=True)
elif args.vercel:
upload_to_vercel()
@ -77,12 +80,11 @@ def run() -> None:
# Run api tests
#############################
elif args.balance:
f.fetch_token_balance(args.balance[0])
f.fetch_token_balance(env_vars, args.balance[0])
elif args.top:
f.fetch_top_holders(args.top[0])
f.fetch_top_holders(env_vars, args.top[0])
elif args.change:
f.fetch_change(args.change[0])
f.fetch_change(env_vars, args.change[0])
else:
parser.print_help()

15
token-scanner-api/src/server/database.py
View File

@ -11,10 +11,9 @@ from src.utils import os_utils
# Private methods: database #
############################################
def _get_db_collection():
def _get_db_collection(env_vars):
"""Connect to the database."""
env_vars = os_utils.load_config()
url = env_vars['MONGODB_URL']
db_name = env_vars['MONGODB_DB_NAME']
collection = env_vars['MONGODB_COLLECTION_NAME']
@ -41,10 +40,10 @@ def _balancer_helper(item) -> dict:
# Public methods: database #
############################################
async def retrieve_top_balances(top_number: int) -> list:
async def retrieve_top_balances(top_number: int, env_vars: dict) -> list:
"""Retrieve top balances from the database."""
collection = _get_db_collection()
collection = _get_db_collection(env_vars)
top_balances = collection.find()
result = []
@ -59,10 +58,10 @@ async def retrieve_top_balances(top_number: int) -> list:
return result
async def retrieve_balance(wallet: str) -> dict:
async def retrieve_balance(wallet: str, env_vars: dict) -> dict:
"""Retrieve balance from the database."""
collection = _get_db_collection()
collection = _get_db_collection(env_vars)
balance = collection.find_one({"wallet": wallet})
if balance:
@ -71,9 +70,9 @@ async def retrieve_balance(wallet: str) -> dict:
return {}
async def retrieve_holder_weekly_change(address: str) -> int:
async def retrieve_holder_weekly_change(address: str, env_vars: dict) -> int:
"""Retrieve weekly change of a given address."""
collection = _get_db_collection()
collection = _get_db_collection(env_vars)
# todo
pass

12
token-scanner-api/src/server/routes.py
View File

@ -20,10 +20,10 @@ async def get_notes() -> dict:
@router.get("/balance/{address}")
async def get_token_balance(address: str) -> dict:
async def get_token_balance(env_vars: dict, address: str) -> dict:
"""Get a token balance for a given address."""
futures = [retrieve_balance(address)]
futures = [retrieve_balance(env_vars, address)]
result = await asyncio.gather(*futures)
if result:
return {"result": result}
@ -32,12 +32,12 @@ async def get_token_balance(address: str) -> dict:
@router.get("/top")
async def get_top_holders(top_number=None) -> dict:
async def get_top_holders(env_vars: dict, top_number=None) -> dict:
"""Get top holders of a given token."""
top_number = top_number or 10
futures = [retrieve_top_balances(top_number)]
futures = [retrieve_top_balances(env_vars, top_number)]
result = await asyncio.gather(*futures)
if result:
return {"top_holders": result}
@ -46,10 +46,10 @@ async def get_top_holders(top_number=None) -> dict:
@router.get("/weekly/{address}")
async def get_holder_weekly_change(address: str) -> dict:
async def get_holder_weekly_change( env_vars: dict, address: str) -> dict:
"""Get weekly change of a given address."""
futures = [retrieve_holder_weekly_change(address)]
futures = [retrieve_holder_weekly_change(env_vars, address)]
result = await asyncio.gather(*futures)
if result:
return {"result": result}

3
token-scanner-api/src/utils/db_processing.py
View File

@ -18,12 +18,11 @@ def format_and_load_data(filepath):
return result
def populate_db(filepath):
def run_db_processing(filepath, env_vars):
#################################
# Connect to database via client
#################################
env_vars = os_utils.load_config()
url = env_vars['MONGODB_URL']
db_name = env_vars['MONGODB_DB_NAME']
collection = env_vars['MONGODB_COLLECTION_NAME']

57
token-scanner-api/src/utils/os_utils.py
View File

@ -7,6 +7,7 @@ import sys
import json
import logging
import requests
import urlparse
from pathlib import Path
from dotenv import load_dotenv
from datetime import datetime
@ -49,7 +50,9 @@ def load_config() -> dict:
env_vars['TOKEN_DECIMALS'] = os.getenv("TOKEN_DECIMALS")
env_vars['MONGODB_URI'] = os.getenv("MONGODB_URI")
env_vars['MONGODB_DB_NAME'] = os.getenv("MONGODB_DB_NAME")
env_vars['MONGODB_COLLECTION_NAME'] = os.getenv("MONGODB_COLLECTION_NAME")
env_vars['MONGODB_COLLECTION_NAME'] = os.getenv("MONGODB_COLLECTION_NAME")
env_vars['API_HOST_URL'] = os.getenv("API_HOST_URL")
env_vars['API_HOST_PORT'] = os.getenv("API_HOST_PORT")
set_logging(os.getenv("LOG_LEVEL"))
return env_vars
@ -92,6 +95,12 @@ def format_path(dir_path, filename) -> str:
return os.path.join(dir_path, filename)
def format_url(base_url, endpoint) -> str:
"""Format a URL full filepath."""
return urlparse.urljoin(base_url, endpoint)
def save_output(destination, data, mode="w") -> None:
"""Save data from memory to a destination in disk."""
@ -143,6 +152,37 @@ def create_result_file(prefix) -> str:
return f'{prefix}_{this_time}.json'
def send_post_request(url, headers=None, json=None) -> dict:
"""Send a request to a given URL"""
json = params or {}
headers = headers or {}
try:
response = requests.post(url, headers=headers, json=json)
return response.json()
except requests.exceptions.HTTPError as e:
log_error('Error querying to {0}: {1}'.format(url, e.response.text))
return {}
def send_get_request(url, params=None) -> dict:
"""Send a request to a given URL"""
params = params or {}
try:
response = requests.get(url, params=params)
return response.json()
except requests.exceptions.HTTPError as e:
log_error('Error querying to {0}: {1}'.format(url, e.response.text))
return {}
def send_rpc_request(url, method, params=None) -> dict:
"""Send a JSON-RPC request to a given URL"""
@ -150,14 +190,9 @@ def send_rpc_request(url, method, params=None) -> dict:
data = {'jsonrpc': '2.0', 'method': method, 'params': params, 'id': 1}
log_debug(f'Querying {url} with {data}')
try:
response = requests.post(url, headers={'Content-Type': 'application/json'}, json=data)
if 'result' in response.json():
return response.json()['result']
else:
log_error('Query failed: {}.'.format(response.json()['error']))
response = send_post_request(url, headers={'Content-Type': 'application/json'}, json=data)
if 'result' in response.json():
return response.json()['result']
else:
log_error('Query failed: {}.'.format(response.json()['error']))
except requests.exceptions.HTTPError as e:
log_error('Error querying to {0}: {1}'.format(url, e.response.text))
return {}

36
token-scanner-api/src/utils/test_api.py
View File

@ -1,8 +1,32 @@
def fetch_token_balance():
pass
# -*- encoding: utf-8 -*-
# utils/test_api.py
# This class implements the tests for the API.
def fetch_top_token_holders():
pass
import os_utils
def fetch_change():
pass
def fetch_token_balance(env_vars, wallet):
"""Test the fetch_token_balance function."""
url = os_utils.format_url(f"{env_vars['API_HOST_URL']}:{env_vars['API_HOST_PORT']}", \
f"/balance/{wallet}")
response = os_utils.send_get_request(url)
os_utils.log_info(response)
def fetch_top_token_holders(env_vars, top_number):
"""Test the fetch_top_token_holders function."""
url = os_utils.format_url(f"{env_vars['API_HOST_URL']}:{env_vars['API_HOST_PORT']}", \
"top")
response = os_utils.send_get_request(url)
os_utils.log_info(response)
def fetch_change(env_vars, wallet):
"""Test the fetch_change function."""
url = os_utils.format_url(f"{env_vars['API_HOST_URL']}:{env_vars['API_HOST_PORT']}", \
f"/weekly/{wallet}")
response = os_utils.send_get_request(url)
os_utils.log_info(response)