diff --git a/token-scanner-api/.env.example b/token-scanner-api/.env.example index 45eb36b..5a043c5 100644 --- a/token-scanner-api/.env.example +++ b/token-scanner-api/.env.example @@ -32,3 +32,8 @@ MONGODB_DB_NAME = balances MONGODB_COLLECTION_NAME = balances +######################### +##### api settings +######################### +API_HOST_URL = http://localhost +API_HOST_URL_PORT = 80 diff --git a/token-scanner-api/src/blockchains/ethereum.py b/token-scanner-api/src/blockchains/ethereum.py index 193aea1..46cf2d6 100644 --- a/token-scanner-api/src/blockchains/ethereum.py +++ b/token-scanner-api/src/blockchains/ethereum.py @@ -12,15 +12,15 @@ from web3.exceptions import BlockNotFound from web3.providers.rpc import HTTPProvider from src.utils.arithmetics import convert_hex_to_int -from src.utils.os_utils import send_rpc_request, exit_with_error, create_result_file, \ - set_output, open_json, load_config, save_output, log_info +from src.utils.os_utils import send_rpc_request, exit_with_error, log_info, \ + set_output, open_json, save_output, create_result_file class TokenIndexer: - def __init__(self): + def __init__(self, env_vars): - self.env_vars = load_config() + self.env_vars = env_vars self.web3 = self._set_web3_object() # contract parameters diff --git a/token-scanner-api/src/main.py b/token-scanner-api/src/main.py index 7d903f5..76c6735 100644 --- a/token-scanner-api/src/main.py +++ b/token-scanner-api/src/main.py @@ -7,9 +7,9 @@ import uvicorn import argparse from src.utils.os_utils import load_config -from src.utils.db_processing import populate_db from src.blockchains.ethereum import TokenIndexer from src.utils.vercel_utils import upload_to_vercel +from src.utils.db_processing import run_db_processing from src.utils.test_api import fetch_token_balance as f from src.utils.data_processing import run_data_processing @@ -50,7 +50,7 @@ def run_menu() -> argparse.ArgumentParser: def run() -> None: """Entry point for this module.""" - load_config() + env_vars = load_config() parser = run_menu() args = parser.parse_args() @@ -58,18 +58,21 @@ def run() -> None: # Run historical data indexer ############################# if args.indexer: - indexer = TokenIndexer() + indexer = TokenIndexer(env_vars) indexer.run() elif args.process: run_data_processing(args.process[0]) elif args.db: - populate_db(args.db[0]) + run_db_processing(args.db[0], env_vars) ############################# # Run deployment tools ############################# elif args.api: - uvicorn.run("src.server.api:app", host="0.0.0.0", port=8000, reload=True) + uvicorn.run("src.server.api:app", \ + host=env_vars['API_HOST_URL'], \ + port=env_vars['API_HOST_PORT'], \ + reload=True) elif args.vercel: upload_to_vercel() @@ -77,12 +80,11 @@ def run() -> None: # Run api tests ############################# elif args.balance: - f.fetch_token_balance(args.balance[0]) + f.fetch_token_balance(env_vars, args.balance[0]) elif args.top: - f.fetch_top_holders(args.top[0]) + f.fetch_top_holders(env_vars, args.top[0]) elif args.change: - f.fetch_change(args.change[0]) - + f.fetch_change(env_vars, args.change[0]) else: parser.print_help() diff --git a/token-scanner-api/src/server/database.py b/token-scanner-api/src/server/database.py index f0ceb13..faf54d7 100644 --- a/token-scanner-api/src/server/database.py +++ b/token-scanner-api/src/server/database.py @@ -11,10 +11,9 @@ from src.utils import os_utils # Private methods: database # ############################################ -def _get_db_collection(): +def _get_db_collection(env_vars): """Connect to the database.""" - env_vars = os_utils.load_config() url = env_vars['MONGODB_URL'] db_name = env_vars['MONGODB_DB_NAME'] collection = env_vars['MONGODB_COLLECTION_NAME'] @@ -41,10 +40,10 @@ def _balancer_helper(item) -> dict: # Public methods: database # ############################################ -async def retrieve_top_balances(top_number: int) -> list: +async def retrieve_top_balances(top_number: int, env_vars: dict) -> list: """Retrieve top balances from the database.""" - collection = _get_db_collection() + collection = _get_db_collection(env_vars) top_balances = collection.find() result = [] @@ -59,10 +58,10 @@ async def retrieve_top_balances(top_number: int) -> list: return result -async def retrieve_balance(wallet: str) -> dict: +async def retrieve_balance(wallet: str, env_vars: dict) -> dict: """Retrieve balance from the database.""" - collection = _get_db_collection() + collection = _get_db_collection(env_vars) balance = collection.find_one({"wallet": wallet}) if balance: @@ -71,9 +70,9 @@ async def retrieve_balance(wallet: str) -> dict: return {} -async def retrieve_holder_weekly_change(address: str) -> int: +async def retrieve_holder_weekly_change(address: str, env_vars: dict) -> int: """Retrieve weekly change of a given address.""" - collection = _get_db_collection() + collection = _get_db_collection(env_vars) # todo pass diff --git a/token-scanner-api/src/server/routes.py b/token-scanner-api/src/server/routes.py index e05cd44..a317cbb 100644 --- a/token-scanner-api/src/server/routes.py +++ b/token-scanner-api/src/server/routes.py @@ -20,10 +20,10 @@ async def get_notes() -> dict: @router.get("/balance/{address}") -async def get_token_balance(address: str) -> dict: +async def get_token_balance(env_vars: dict, address: str) -> dict: """Get a token balance for a given address.""" - futures = [retrieve_balance(address)] + futures = [retrieve_balance(env_vars, address)] result = await asyncio.gather(*futures) if result: return {"result": result} @@ -32,12 +32,12 @@ async def get_token_balance(address: str) -> dict: @router.get("/top") -async def get_top_holders(top_number=None) -> dict: +async def get_top_holders(env_vars: dict, top_number=None) -> dict: """Get top holders of a given token.""" top_number = top_number or 10 - futures = [retrieve_top_balances(top_number)] + futures = [retrieve_top_balances(env_vars, top_number)] result = await asyncio.gather(*futures) if result: return {"top_holders": result} @@ -46,10 +46,10 @@ async def get_top_holders(top_number=None) -> dict: @router.get("/weekly/{address}") -async def get_holder_weekly_change(address: str) -> dict: +async def get_holder_weekly_change( env_vars: dict, address: str) -> dict: """Get weekly change of a given address.""" - futures = [retrieve_holder_weekly_change(address)] + futures = [retrieve_holder_weekly_change(env_vars, address)] result = await asyncio.gather(*futures) if result: return {"result": result} diff --git a/token-scanner-api/src/utils/db_processing.py b/token-scanner-api/src/utils/db_processing.py index 8a43c40..e968ae5 100644 --- a/token-scanner-api/src/utils/db_processing.py +++ b/token-scanner-api/src/utils/db_processing.py @@ -18,12 +18,11 @@ def format_and_load_data(filepath): return result -def populate_db(filepath): +def run_db_processing(filepath, env_vars): ################################# # Connect to database via client ################################# - env_vars = os_utils.load_config() url = env_vars['MONGODB_URL'] db_name = env_vars['MONGODB_DB_NAME'] collection = env_vars['MONGODB_COLLECTION_NAME'] diff --git a/token-scanner-api/src/utils/os_utils.py b/token-scanner-api/src/utils/os_utils.py index 3aab940..697ac70 100644 --- a/token-scanner-api/src/utils/os_utils.py +++ b/token-scanner-api/src/utils/os_utils.py @@ -7,6 +7,7 @@ import sys import json import logging import requests +import urlparse from pathlib import Path from dotenv import load_dotenv from datetime import datetime @@ -49,7 +50,9 @@ def load_config() -> dict: env_vars['TOKEN_DECIMALS'] = os.getenv("TOKEN_DECIMALS") env_vars['MONGODB_URI'] = os.getenv("MONGODB_URI") env_vars['MONGODB_DB_NAME'] = os.getenv("MONGODB_DB_NAME") - env_vars['MONGODB_COLLECTION_NAME'] = os.getenv("MONGODB_COLLECTION_NAME") + env_vars['MONGODB_COLLECTION_NAME'] = os.getenv("MONGODB_COLLECTION_NAME") + env_vars['API_HOST_URL'] = os.getenv("API_HOST_URL") + env_vars['API_HOST_PORT'] = os.getenv("API_HOST_PORT") set_logging(os.getenv("LOG_LEVEL")) return env_vars @@ -92,6 +95,12 @@ def format_path(dir_path, filename) -> str: return os.path.join(dir_path, filename) +def format_url(base_url, endpoint) -> str: + """Format a URL full filepath.""" + + return urlparse.urljoin(base_url, endpoint) + + def save_output(destination, data, mode="w") -> None: """Save data from memory to a destination in disk.""" @@ -143,6 +152,37 @@ def create_result_file(prefix) -> str: return f'{prefix}_{this_time}.json' +def send_post_request(url, headers=None, json=None) -> dict: + """Send a request to a given URL""" + + json = params or {} + headers = headers or {} + + try: + response = requests.post(url, headers=headers, json=json) + return response.json() + + except requests.exceptions.HTTPError as e: + log_error('Error querying to {0}: {1}'.format(url, e.response.text)) + + return {} + + +def send_get_request(url, params=None) -> dict: + """Send a request to a given URL""" + + params = params or {} + + try: + response = requests.get(url, params=params) + return response.json() + + except requests.exceptions.HTTPError as e: + log_error('Error querying to {0}: {1}'.format(url, e.response.text)) + + return {} + + def send_rpc_request(url, method, params=None) -> dict: """Send a JSON-RPC request to a given URL""" @@ -150,14 +190,9 @@ def send_rpc_request(url, method, params=None) -> dict: data = {'jsonrpc': '2.0', 'method': method, 'params': params, 'id': 1} log_debug(f'Querying {url} with {data}') - try: - response = requests.post(url, headers={'Content-Type': 'application/json'}, json=data) - if 'result' in response.json(): - return response.json()['result'] - else: - log_error('Query failed: {}.'.format(response.json()['error'])) + response = send_post_request(url, headers={'Content-Type': 'application/json'}, json=data) + if 'result' in response.json(): + return response.json()['result'] + else: + log_error('Query failed: {}.'.format(response.json()['error'])) - except requests.exceptions.HTTPError as e: - log_error('Error querying to {0}: {1}'.format(url, e.response.text)) - - return {} diff --git a/token-scanner-api/src/utils/test_api.py b/token-scanner-api/src/utils/test_api.py index 2b06b34..34f36f1 100644 --- a/token-scanner-api/src/utils/test_api.py +++ b/token-scanner-api/src/utils/test_api.py @@ -1,8 +1,32 @@ -def fetch_token_balance(): - pass +# -*- encoding: utf-8 -*- +# utils/test_api.py +# This class implements the tests for the API. -def fetch_top_token_holders(): - pass +import os_utils -def fetch_change(): - pass \ No newline at end of file + +def fetch_token_balance(env_vars, wallet): + """Test the fetch_token_balance function.""" + + url = os_utils.format_url(f"{env_vars['API_HOST_URL']}:{env_vars['API_HOST_PORT']}", \ + f"/balance/{wallet}") + response = os_utils.send_get_request(url) + os_utils.log_info(response) + + +def fetch_top_token_holders(env_vars, top_number): + """Test the fetch_top_token_holders function.""" + + url = os_utils.format_url(f"{env_vars['API_HOST_URL']}:{env_vars['API_HOST_PORT']}", \ + "top") + response = os_utils.send_get_request(url) + os_utils.log_info(response) + + +def fetch_change(env_vars, wallet): + """Test the fetch_change function.""" + + url = os_utils.format_url(f"{env_vars['API_HOST_URL']}:{env_vars['API_HOST_PORT']}", \ + f"/weekly/{wallet}") + response = os_utils.send_get_request(url) + os_utils.log_info(response)