clean up and add web3

.github/.gitkeep

@@ -0,0 +1 @@
+

.github/workflows/cryptoactions.yml

@@ -0,0 +1,16 @@
+name: crypto action
+run-name: ${{ github.actor }} is testing crypto stuff 🚀
+on: [push]
+jobs:
+  Explore-GitHub-Actions:
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event."
+      - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!"
+      - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}."
+      - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner."
+      - run: echo "🖥️ The workflow is now ready to test your code on the runner."
+      - name: List files in the repository
+        run: |
+          ls ${{ github.workspace }}
+      - run: echo "🍏 This job's status is ${{ job.status }}."

.github/workflows/notes/.gitkeep

@@ -0,0 +1 @@
+

.github/workflows/validate_urls.yml

@@ -0,0 +1,32 @@
+
+name: validate links
+
+on: [push, pull_request]
+
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+
+jobs:
+  unit:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+
+      - name: set up ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.1
+          bundler-cache: true
+
+      - name: install awesome_bot
+        run: gem install awesome_bot
+
+      - name: validate URLs
+        run: awesome_bot README.md --allow-redirect --request-delay 0.4

README.md

@@ -1,52 +1,63 @@
-# ✨🐍 Python: small projects and boilerplates 
+## 🥷🏻🐍⛓️ mev python: playing pvp in the metaweb
+
+<br>
 
 
-## Code in this repo
+### my [ongoing] web3 projects and templates 
 
-### APIs
-
-* [FastAPI Location API deployed in Vercel](https://github.com/go-outside-labs/Scratch-Space-Python/tree/master/fastapi-location-app)
+* [web3-python-toolkit](web3-python-toolkit)
+    - an on-going development of a library and set of python scripts with my fav on-chain ops.
 
 
-### Boilerplates
 
-* [CLI with Click](https://github.com/go-outside-labs/Scratch-Space-Python/tree/master/boilerplates-click)
-* [CLI with Argparse](https://github.com/go-outside-labs/Scratch-Space-Python/tree/master/boilerplates-argparse)
-* [Dashboards with Dash and Plot.ly](https://github.com/go-outside-labs/Scratch-Space-Python/tree/master/boilerplates-dash)
-* [Testing in Python](https://github.com/go-outside-labs/Scratch-Space-Python/tree/master/boilerplates-tests)
-* [Security examples](https://github.com/go-outside-labs/Scratch-Space-Python/tree/master/boilerplates-security)
-* [Concurrence examples](https://github.com/go-outside-labs/Scratch-Space-Python/tree/master/boilerplates-concurrence)
+<br>
 
-### Tests
+---
 
+### my [over the years] web2 projects and templates
+
+#### side and fun projects
+
+* [finding a blob boundary](small-projects/finding-blob-boundary)
+* [magic pen](small-projects/magic-pen)
+* [maze puzzle](projects/maze-puzzle)
+* [parsing medium posts](small-projects/medium)
+* [encoding decimals](enconding-decimals)
+
+<br>
+
+#### boilerplates
+
+* [fast api](fastapi-location-app)
+* [click](boilerplates-click)
+* [argparse](boilerplates-argparse)
+* [dash + plot.ly](boilerplates-dash)
+* [security](boilerplates-security)
+* [concurrency](boilerplates-concurrency)
+* [optimization](boilerplates-optimization)
+
+<br>
+
+#### tests
+
+* [testing in python](boilerplates-tests)
 * [good example of unit tests suite I wrote](https://github.com/go-outside-labs/aws-pipeline/tree/master/tests)
 
 
-### Small projects
-
-* [encoding decimals](https://github.com/go-outside-labs/Scratch-Space-Python/tree/master/small-projects/enconding-decimals)
-* [finding a blob boundary](https://github.com/go-outside-labs/Scratch-Space-Python/tree/master/small-projects/finding-blob-boundary)
-* [magic pen](https://github.com/go-outside-labs/Scratch-Space-Python/tree/master/small-projects/magic-pen)
-* [maze puzzle](https://github.com/go-outside-labs/Scratch-Space-Python/tree/master/small-projects/maze-puzzle)
-* [parsing Medium posts](https://github.com/go-outside-labs/Scratch-Space-Python/tree/master/small-projects/medium)
 
+<br>
 
 ----
 
-### Useful tools, linters, and packages
+### external resources
 
-* [Black](https://github.com/psf/black)
-* [Flake8 ](https://flake8.pycqa.org/en/latest/)
-* [Pre-commit](https://pre-commit.com/)
+* [my book on python and algorithms](https://github.com/go-outside-labs/algorithms-book)
+* [google style guide](https://google.github.io/styleguide/pyguide.html)
+* [black](https://github.com/psf/black)
+* [flake8 ](https://flake8.pycqa.org/en/latest/)
+* [pre-commit](https://pre-commit.com/)
 * [unittest](https://docs.python.org/3/library/unittest.html)
 * [nose](https://nose.readthedocs.io/en/latest/)
 * [tox](https://tox.wiki/en/latest/)
 
 
-----
-
-
-## Good Readings
-
-* [Google Style Guide](https://google.github.io/styleguide/pyguide.html)
-

web3_python_toolkit/ Makefile

@@ -0,0 +1,20 @@
+.PHONY: clean install lint
+
+clean:
+	@find . -iname '*.py[co]' -delete
+	@find . -iname '__pycache__' -delete
+	@rm -rf  '.pytest_cache'
+	@rm -rf dist/
+	@rm -rf build/
+	@rm -rf *.egg-info
+	@rm -rf poetry.lock
+	@rm -rf .tox
+	@rm -rf venv/lib/python*/site-packages/*.egg
+	poetry cache clear
+	poetry env remove --all
+
+install:
+	poetry install
+
+lint:
+	tox -e lint

web3_python_toolkit/.env.example

@@ -0,0 +1,49 @@
+########## 
+# general
+########## 
+
+##### options are: 'error' < 'info' < 'debug'
+LOG_LEVEL=info
+
+PROVIDER_TYPE=
+
+PROVIDER_URL= 
+
+BLOCK_NUMBER=
+
+
+######################
+# get_block_history.py
+###################### 
+
+#### options are 'local_ipc', 'local_http', 'local_ws', 'ipc', 'ws', 'http'
+#### for locally run nodes, an IPC connection is the mosts secure option.
+
+
+PAIR_ADDRESSES=
+
+ABI_JSON_PATH=
+
+
+##############################
+# get_deep_block_data.py
+############################# 
+
+
+
+##############################
+# get_deep_tx_data.py
+############################# 
+
+TRANSACTION=
+
+
+##############################
+# get_contracts_deployed.py
+############################# 
+
+START_BLOCK=
+
+TX_FILE=
+
+

web3_python_toolkit/README.md

@@ -0,0 +1,104 @@
+## 📚 web3-python-toolkit
+
+<br>
+
+an on-going development of a library and set of python scripts with my fav on-chain ops.
+
+<br>
+
+
+### installing
+
+```
+brew install poetry
+make install
+cp .env.example .env
+```
+
+
+<br>
+
+
+----
+
+### scripts
+
+
+#### get contracts deployed to mainnet and testnets
+
+1. add info to `.env`
+2. run 
+   `poetry run python  get_contracts_deployed.py`
+3. any output is saved to `data/`.
+
+
+
+<br>
+
+
+#### get reserve history by block for a pair of addresses
+
+1. add the pair abi to `abi`
+2. add info to `.env`
+3. run 
+   `poetry run python get_reserve_history_by_block.py`
+
+
+
+<br>
+
+#### get deep block data
+
+1. add info to `.env`
+3. run 
+   `poetry run python get_deep_block_data.py`
+
+
+
+<br>
+
+---
+
+### troubleshoot
+
+##### if you see `ethereum-etl not compatible to m1` run:
+
+```
+pip uninstall ethereum-etl 
+pip install --no-binary ethereum-etl 
+```
+
+<br>
+
+---
+
+### resources
+
+
+* [web3.py library](https://web3py.readthedocs.io/en/v5/)
+* [ethereum etl library](https://ethereum-etl.readthedocs.io/en/latest/quickstart/)
+
+
+<br>
+
+---
+
+### relevant info
+
+##### providers
+ 
+- providers are how libraries such as `web3.py` talk to the blockchain. 
+- providers take `JSON-RPC` requests and return responses
+- the most common ways to connect to your node are:
+   - IPC (uses local filesystem, fastest and most secure)
+   - Websockets (works remotely, faster than HTTP)
+   - HTTP (more nodes support it)
+
+<br>
+
+##### middleware
+
+* a web3.py instance can be configured via middleware (sitting between the web3 methods and the provider).
+* middlewares use an onion metaphor: each layer may affect both the request and response from the provider.
+* each middleware layer gets invoked before the request reaches the provider, and then processes the result after the provider returns, in reverse order.
+* we often use `geth_poa_middleware`, to run with geth's Proof-of-Authority (PoA) consensus. this adds support for more than 32 bytes in each block (the `extraData` field).

web3_python_toolkit/abis/LP_pair.json

@@ -0,0 +1,658 @@
+[
+    {
+        "inputs": [],
+        "stateMutability": "nonpayable",
+        "type": "constructor"
+    },
+    {
+        "anonymous": false,
+        "inputs": [
+            {
+                "indexed": true,
+                "internalType": "address",
+                "name": "owner",
+                "type": "address"
+            },
+            {
+                "indexed": true,
+                "internalType": "address",
+                "name": "spender",
+                "type": "address"
+            },
+            {
+                "indexed": false,
+                "internalType": "uint256",
+                "name": "value",
+                "type": "uint256"
+            }
+        ],
+        "name": "Approval",
+        "type": "event"
+    },
+    {
+        "anonymous": false,
+        "inputs": [
+            {
+                "indexed": true,
+                "internalType": "address",
+                "name": "sender",
+                "type": "address"
+            },
+            {
+                "indexed": false,
+                "internalType": "uint256",
+                "name": "amount0",
+                "type": "uint256"
+            },
+            {
+                "indexed": false,
+                "internalType": "uint256",
+                "name": "amount1",
+                "type": "uint256"
+            },
+            {
+                "indexed": true,
+                "internalType": "address",
+                "name": "to",
+                "type": "address"
+            }
+        ],
+        "name": "Burn",
+        "type": "event"
+    },
+    {
+        "anonymous": false,
+        "inputs": [
+            {
+                "indexed": true,
+                "internalType": "address",
+                "name": "sender",
+                "type": "address"
+            },
+            {
+                "indexed": false,
+                "internalType": "uint256",
+                "name": "amount0",
+                "type": "uint256"
+            },
+            {
+                "indexed": false,
+                "internalType": "uint256",
+                "name": "amount1",
+                "type": "uint256"
+            }
+        ],
+        "name": "Mint",
+        "type": "event"
+    },
+    {
+        "anonymous": false,
+        "inputs": [
+            {
+                "indexed": true,
+                "internalType": "address",
+                "name": "sender",
+                "type": "address"
+            },
+            {
+                "indexed": false,
+                "internalType": "uint256",
+                "name": "amount0In",
+                "type": "uint256"
+            },
+            {
+                "indexed": false,
+                "internalType": "uint256",
+                "name": "amount1In",
+                "type": "uint256"
+            },
+            {
+                "indexed": false,
+                "internalType": "uint256",
+                "name": "amount0Out",
+                "type": "uint256"
+            },
+            {
+                "indexed": false,
+                "internalType": "uint256",
+                "name": "amount1Out",
+                "type": "uint256"
+            },
+            {
+                "indexed": true,
+                "internalType": "address",
+                "name": "to",
+                "type": "address"
+            }
+        ],
+        "name": "Swap",
+        "type": "event"
+    },
+    {
+        "anonymous": false,
+        "inputs": [
+            {
+                "indexed": false,
+                "internalType": "uint112",
+                "name": "reserve0",
+                "type": "uint112"
+            },
+            {
+                "indexed": false,
+                "internalType": "uint112",
+                "name": "reserve1",
+                "type": "uint112"
+            }
+        ],
+        "name": "Sync",
+        "type": "event"
+    },
+    {
+        "anonymous": false,
+        "inputs": [
+            {
+                "indexed": true,
+                "internalType": "address",
+                "name": "from",
+                "type": "address"
+            },
+            {
+                "indexed": true,
+                "internalType": "address",
+                "name": "to",
+                "type": "address"
+            },
+            {
+                "indexed": false,
+                "internalType": "uint256",
+                "name": "value",
+                "type": "uint256"
+            }
+        ],
+        "name": "Transfer",
+        "type": "event"
+    },
+    {
+        "inputs": [],
+        "name": "DOMAIN_SEPARATOR",
+        "outputs": [
+            {
+                "internalType": "bytes32",
+                "name": "",
+                "type": "bytes32"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [],
+        "name": "MINIMUM_LIQUIDITY",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [],
+        "name": "PERMIT_TYPEHASH",
+        "outputs": [
+            {
+                "internalType": "bytes32",
+                "name": "",
+                "type": "bytes32"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "address",
+                "name": "",
+                "type": "address"
+            },
+            {
+                "internalType": "address",
+                "name": "",
+                "type": "address"
+            }
+        ],
+        "name": "allowance",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "address",
+                "name": "spender",
+                "type": "address"
+            },
+            {
+                "internalType": "uint256",
+                "name": "value",
+                "type": "uint256"
+            }
+        ],
+        "name": "approve",
+        "outputs": [
+            {
+                "internalType": "bool",
+                "name": "",
+                "type": "bool"
+            }
+        ],
+        "stateMutability": "nonpayable",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "address",
+                "name": "",
+                "type": "address"
+            }
+        ],
+        "name": "balanceOf",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "address",
+                "name": "to",
+                "type": "address"
+            }
+        ],
+        "name": "burn",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "amount0",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "amount1",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "nonpayable",
+        "type": "function"
+    },
+    {
+        "inputs": [],
+        "name": "decimals",
+        "outputs": [
+            {
+                "internalType": "uint8",
+                "name": "",
+                "type": "uint8"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [],
+        "name": "factory",
+        "outputs": [
+            {
+                "internalType": "address",
+                "name": "",
+                "type": "address"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [],
+        "name": "getReserves",
+        "outputs": [
+            {
+                "internalType": "uint112",
+                "name": "_reserve0",
+                "type": "uint112"
+            },
+            {
+                "internalType": "uint112",
+                "name": "_reserve1",
+                "type": "uint112"
+            },
+            {
+                "internalType": "uint32",
+                "name": "_blockTimestampLast",
+                "type": "uint32"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "address",
+                "name": "_token0",
+                "type": "address"
+            },
+            {
+                "internalType": "address",
+                "name": "_token1",
+                "type": "address"
+            }
+        ],
+        "name": "initialize",
+        "outputs": [],
+        "stateMutability": "nonpayable",
+        "type": "function"
+    },
+    {
+        "inputs": [],
+        "name": "kLast",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "address",
+                "name": "to",
+                "type": "address"
+            }
+        ],
+        "name": "mint",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "liquidity",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "nonpayable",
+        "type": "function"
+    },
+    {
+        "inputs": [],
+        "name": "name",
+        "outputs": [
+            {
+                "internalType": "string",
+                "name": "",
+                "type": "string"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "address",
+                "name": "",
+                "type": "address"
+            }
+        ],
+        "name": "nonces",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "address",
+                "name": "owner",
+                "type": "address"
+            },
+            {
+                "internalType": "address",
+                "name": "spender",
+                "type": "address"
+            },
+            {
+                "internalType": "uint256",
+                "name": "value",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "deadline",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint8",
+                "name": "v",
+                "type": "uint8"
+            },
+            {
+                "internalType": "bytes32",
+                "name": "r",
+                "type": "bytes32"
+            },
+            {
+                "internalType": "bytes32",
+                "name": "s",
+                "type": "bytes32"
+            }
+        ],
+        "name": "permit",
+        "outputs": [],
+        "stateMutability": "nonpayable",
+        "type": "function"
+    },
+    {
+        "inputs": [],
+        "name": "price0CumulativeLast",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [],
+        "name": "price1CumulativeLast",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "address",
+                "name": "to",
+                "type": "address"
+            }
+        ],
+        "name": "skim",
+        "outputs": [],
+        "stateMutability": "nonpayable",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "amount0Out",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "amount1Out",
+                "type": "uint256"
+            },
+            {
+                "internalType": "address",
+                "name": "to",
+                "type": "address"
+            },
+            {
+                "internalType": "bytes",
+                "name": "data",
+                "type": "bytes"
+            }
+        ],
+        "name": "swap",
+        "outputs": [],
+        "stateMutability": "nonpayable",
+        "type": "function"
+    },
+    {
+        "inputs": [],
+        "name": "symbol",
+        "outputs": [
+            {
+                "internalType": "string",
+                "name": "",
+                "type": "string"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [],
+        "name": "sync",
+        "outputs": [],
+        "stateMutability": "nonpayable",
+        "type": "function"
+    },
+    {
+        "inputs": [],
+        "name": "token0",
+        "outputs": [
+            {
+                "internalType": "address",
+                "name": "",
+                "type": "address"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [],
+        "name": "token1",
+        "outputs": [
+            {
+                "internalType": "address",
+                "name": "",
+                "type": "address"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [],
+        "name": "totalSupply",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "address",
+                "name": "to",
+                "type": "address"
+            },
+            {
+                "internalType": "uint256",
+                "name": "value",
+                "type": "uint256"
+            }
+        ],
+        "name": "transfer",
+        "outputs": [
+            {
+                "internalType": "bool",
+                "name": "",
+                "type": "bool"
+            }
+        ],
+        "stateMutability": "nonpayable",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "address",
+                "name": "from",
+                "type": "address"
+            },
+            {
+                "internalType": "address",
+                "name": "to",
+                "type": "address"
+            },
+            {
+                "internalType": "uint256",
+                "name": "value",
+                "type": "uint256"
+            }
+        ],
+        "name": "transferFrom",
+        "outputs": [
+            {
+                "internalType": "bool",
+                "name": "",
+                "type": "bool"
+            }
+        ],
+        "stateMutability": "nonpayable",
+        "type": "function"
+    }
+]

web3_python_toolkit/pyproject.toml

@@ -0,0 +1,21 @@
+[tool.poetry]
+name = "web3-python-toolkit"
+version = "0.1.0"
+description="toolkit for web3"
+authors=["steinkirch"]
+readme = "README.md"
+packages = [{include = "scripts"}]
+
+[tool.poetry.dependencies]
+python = "3.10"
+ethereum-etl = "2.1.2"
+python-dotenv = "0.21.0"
+web3 = "5.31.3"
+pandas = "^1.5.2"
+
+[tool.poetry.group.dev.dependencies]
+tox = "^4.0.16"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

web3_python_toolkit/scripts/__init__.py

@@ -0,0 +1 @@
+# -*- encoding: utf-8 -*-

web3_python_toolkit/scripts/get_contracts_deployed.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# author: steinkirch
+
+import os
+import ethereumetl
+import pandas as pd
+from utils.os import load_config, create_dir, run_exec
+from utils.plots import plot_bar, open_csv, save_csv
+
+
+
+def get_data_for_contracts_by_block() -> dict:
+    """Prepare a dict of data to extract contracts by block."""
+    
+    data = {}
+    create_dir('data')
+    env_keys = ['PROVIDER_URL', 
+                'TX_FILE', 
+                'START_BLOCK',
+                'OUTPUT_FILE']
+    env_vars = load_config(env_keys) 
+
+    data['provider_uri'] = env_vars['PROVIDER_URL']
+    data['tx_file'] = env_vars['TX_FILE']
+    data['start_block'] = env_vars['START_BLOCK']
+    data['output_file'] = env_vars['OUTPUT_FILE']
+
+    # adding this manually
+    data['last_block_2015'] = 778482
+    data['last_block_2016'] = 2912406
+    data['last_block_2017'] = 4832685
+    data['last_block_2018'] = 6988614
+    data['last_block_2019'] = 9193265
+    data['last_block_2020'] = 11565018
+    data['last_block_2021'] = 13916165
+    data['last_block_2022'] = 15978869 
+    data['buffer_for_chunk_size'] = 10000
+
+    return data
+
+
+def export_blocks_and_transactions(end_block, data) -> dict:
+    """Run ethereumetl export_blocks_and_transactions."""
+
+    run_exec(['ethereumetl', 'export_blocks_and_transactions', \
+                  f'--start-block {data["start_block"]}', \
+                  f'--end-block {end_block}', \
+                  f'--provider-uri {data["provider_uri"]}', \
+                  f'--transactions-output {data["tx_file"]}'])
+
+    txs = open_csv(data['tx_file'])
+    contracts = txs[txs['to_address'].isnull()]
+    os.remove(data['tx_file'])
+    return contracts + txs['from_address'].unique().tolist()
+
+
+def get_contracts_by_block(data, year) -> dict:
+    """Extract unique contracts by block for a given year."""
+
+    contracts = []
+    last_block_year = data[f'last_block_{year}']
+    start_block = int(data['start_block'])
+    end_block = start_block + 9999
+
+    while (end_block <= last_block_year + data['buffer_for_chunk_size']): 
+        end_block_used = min(end_block, last_block_year)
+        contracts.append(export_blocks_and_transactions(end_block_used, data))
+        start_block += data['buffer_for_chunk_size']
+        end_block += data['buffer_for_chunk_size']
+
+    return contracts
+
+
+def get_unique_contracts(contracts, year) -> None:
+    """Extract and save a list of unique contracts."""
+
+    unique_contract = [*set(contracts)]
+    print(f"✅ Unique contract for {year})): {str(len(unique_contract))}")
+    return pd.DataFrame(unique_contract, columns=["contracts"])
+
+
+if __name__ == "__main__":
+
+    ###########
+    # Set up
+    ###########
+    contracts_by_year = {}
+    all_contracts = pd.DataFrame()
+    years = list(range(2015, 2022))
+    data = get_data_for_contracts_by_block()
+    start_block = int(data['start_block'])
+
+    ###########
+    # Get data
+    ###########
+    for year in years:
+        contracts = get_contracts_by_block(data, year)
+        contracts_by_year[year] = len(contracts)
+        all_contracts.append(get_unique_contracts(contracts, year))
+        start_block += 1
+
+    all_contracts = all_contracts['contract_deployers'].unique()
+    all_contracts_df = pd.DataFrame(all_contracts, columns=['contract'])
+    save_csv(all_contracts_df, data['output_file'])
+                                                     
+    ###########
+    # Plot data
+    ###########
+    y_data = [y for y in contracts_by_year[year] if year == years.revers().pop()]
+    plot_bar({'contract deployed': y_data}, years)
+
+

web3_python_toolkit/scripts/get_deep_block_data.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# author: steinkirch
+
+from utils.strings import pprint
+from utils.os import load_config
+from utils.web3_wrapper import Web3Wrapper
+
+
+def get_data_for_connection() -> dict:
+    """Prepare a dict of data for connection."""
+
+    data = {}
+    env_keys = ['PROVIDER_TYPE', 
+                'PROVIDER_URL',
+                'BLOCK_NUMBER']
+    env_vars = load_config(env_keys) 
+
+    data['network'] = env_vars['PROVIDER_URL']
+    data['block'] = env_vars['BLOCK_NUMBER']
+    data['provider_type'] = env_vars['PROVIDER_TYPE']  
+    return data
+
+
+def get_deep_block_data(data) -> dict:
+    
+    w3 = Web3Wrapper(mode=data['provider_type'],
+                     network=data['network'])  
+    return w3.get_block()
+
+
+if __name__ == "__main__":
+
+    data = get_data_for_connection()
+    results = get_deep_block_data(data)
+    pprint(results)
+    

web3_python_toolkit/scripts/get_deep_tx_data.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# author: steinkirch
+
+from utils.strings import pprint
+from utils.os import load_config
+from utils.web3_wrapper import Web3Wrapper
+
+
+def get_data_for_connection() -> dict:
+    """Prepare a dict of data for connection."""
+
+    data = {}
+    env_keys = ['PROVIDER_TYPE', 
+                'PROVIDER_URL',
+                'TRANSACTION']
+    env_vars = load_config(env_keys) 
+
+    data['network'] = env_vars['PROVIDER_URL']
+    data['provider_type'] = env_vars['PROVIDER_TYPE'] 
+    data['tx'] = env_vars['TRANSACTION']
+    return data
+
+
+def get_deep_tx_data(data) -> dict:
+    
+    w3 = Web3Wrapper(mode=data['provider_type'],
+                     network=data['network'])  
+    tx_data = w3.get_tx(data['tx'])
+    tx_data.update(w3.get_tx_receipt(data['tx']))
+
+    return tx_data
+
+
+if __name__ == "__main__":
+
+    data = get_data_for_connection()
+    tx_data = get_deep_tx_data(data)
+    pprint(tx_data)
+    

web3_python_toolkit/scripts/get_reserve_history_by_block.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# author: steinkirch
+
+from utils.os import load_config, open_json, log_info
+from utils.web3_wrapper import Web3Wrapper
+
+
+def get_data_for_connection() -> dict:
+    """Prepare a dict of data for connection."""
+
+    data = {}
+    env_keys = ['PAIR_ADDRESSES', 
+                'PROVIDER_URL',
+                'BLOCK_NUMBER',
+                'ABI_JSON_PATH',
+                'PROVIDER_TYPE']
+    env_vars = load_config(env_keys) 
+
+    data['addresses'] = env_vars['PAIR_ADDRESSES']
+    data['network'] = env_vars['PROVIDER_URL']
+    data['block'] = env_vars['BLOCK_NUMBER']
+    data['abi'] = env_vars['ABI_JSON_PATH']
+    data['provider_type'] = env_vars['PROVIDER_TYPE']  
+    return data
+
+
+def get_reserve_by_block(data) -> None:
+    """Establish connection to retrieve reserve history."""
+
+    w3 = Web3Wrapper(mode=data['provider_type'],
+                     network=data['network'])
+    w3.inject_middleware()    
+    w3.get_pair_contract(data['addresses'], open_json(data['abi']))
+    return w3.get_reserves(int(data['block']))
+    
+
+if __name__ == "__main__":
+
+    data = get_data_for_connection()
+    reserve1, reserve2 = get_reserve_by_block(data)
+    log_info(f'reserves: {reserve1}, {reserve2}')

web3_python_toolkit/scripts/utils/__init__.py

@@ -0,0 +1 @@
+# -*- encoding: utf-8 -*-

web3_python_toolkit/scripts/utils/arithmetics.py

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+# arithmetics.py
+# This class implements math methods used by the other classes.
+# author: steinkirch
+
+
+from decimal import Decimal, getcontext
+from utils.strings import log_error
+
+
+def div(dividend, divisor) -> Decimal:
+    """Return higher precision division."""
+
+    if divisor == 0:
+        log_error('Found a zero division error. Returning 0.')
+        return 0
+    return to_decimal(dividend) / to_decimal(divisor)
+
+
+def to_decimal(value, precision=None) -> Decimal:
+    """Return Decimal value for higher (defined) precision."""
+
+    precision = precision or 22
+    getcontext().prec = precision
+    return Decimal(value)

web3_python_toolkit/scripts/utils/os.py

@@ -0,0 +1,149 @@
+# -*- encoding: utf-8 -*-
+# This class implements OS/file system util methods used by the other classes.
+
+import os
+import sys
+import json
+import copy
+import logging
+import subprocess
+from pathlib import Path
+from dotenv import load_dotenv
+
+
+def set_logging(log_level) -> None:
+    """Set logging level according to .env config."""
+
+    if log_level == 'info':
+        logging.basicConfig(level=logging.INFO, format='%(message)s')
+
+    elif log_level == 'error':
+        logging.basicConfig(level=logging.ERROR, format='%(message)s')
+
+    elif log_level == 'debug':
+        logging.basicConfig(level=logging.DEBUG, format='%(message)s')
+
+    else:
+        print(f'Logging level {log_level} is not available. Setting to ERROR')
+        logging.basicConfig(level=logging.ERROR, format='%(message)s')
+
+
+def load_config(keys) -> dict:
+    """Load and set environment variables."""
+
+    env_file = Path('..') / '.env'
+    if not os.path.isfile(env_file):
+        exit_with_error('Please create an .env file')
+
+    load_dotenv(env_file)
+    env_vars = {}
+    
+    try:
+        for key in keys:
+            env_vars[key] = os.getenv(key)
+            set_logging(os.getenv("LOG_LEVEL"))
+        return env_vars
+    except KeyError as e:
+        exit_with_error(f'Cannot extract env variables: {e}. Exiting.')
+
+
+def log_error(string) -> None:
+    """Print STDOUT error using the logging library."""
+
+    logging.error('🚨 %s', string)
+
+
+def log_info(string) -> None:
+    """Print STDOUT info using the logging library."""
+
+    logging.info('✅ %s', string)
+
+
+def log_debug(string) -> None:
+    """Print STDOUT debug using the logging library."""
+
+    logging.debug('🟨 %s', string)
+
+
+def open_json(filepath) -> dict:
+    """Load and parse a json file."""
+
+    try:
+        with open(filepath, 'r', encoding='utf-8') as infile:
+            return json.load(infile)
+
+    except (IOError, FileNotFoundError, TypeError) as e:
+        exit_with_error(f'Failed to parse: "{filepath}": {e}')
+
+
+def format_path(dir_path, filename) -> str:
+    """Format a OS full filepath."""
+
+    return os.path.join(dir_path, filename)
+
+
+def format_output_file(name) -> str:
+    """Format the name for the result file."""
+
+    return f'{name}.json'
+
+
+def save_json(destination, data) -> None:
+    """Save data from memory to a json destination in disk."""
+
+    try:
+        with open(destination, 'w', encoding='utf-8') as outfile:
+            json.dump(data, outfile, indent=4)
+
+    except (IOError, TypeError) as e:
+        log_error(f'Could not save {destination}: {e}')
+
+
+def create_dir(result_dir) -> None:
+    """Check whether a directory exists and create it if needed."""
+
+    try:
+        if not os.path.isdir(result_dir):
+            os.mkdir(result_dir)
+
+    except OSError as e:
+        log_error(f'Could not create {result_dir}: {e}')
+
+
+def set_output(env_vars, input_file) -> str:
+    """Create an output destination to save solutions."""
+
+    try:
+        output_dir = env_vars['OUTPUT_DIR']
+        create_dir(output_dir)
+
+        output_str = input_file.split('_')[1].split('.json')[0]
+        output_file_str = env_vars['OUTPUT_FILE_STR']
+        output_file = output_file_str.format(output_str)
+        return format_path(output_dir, output_file)
+
+    except (TypeError, KeyError) as e:
+        exit_with_error(f'Could not format output file: {e}')
+
+
+def deep_copy(dict_to_clone) -> dict:
+    """Deep copy (not reference copy) to a dict."""
+
+    return copy.deepcopy(dict_to_clone)
+
+
+def exit_with_error(message) -> None:
+    """Log an error message and halt the program."""
+
+    log_error(message)
+    sys.exit(1)
+
+
+def run_exec(command) -> None:
+    """Exec a bash commnad (remember: not safe!)."""
+
+    try:
+        this_command = subprocess.run(command)
+        log_info(f'Exit code: {this_command.returncode}')
+    except Exception as e:
+        log_error(f'Error running {command}: {e}')

web3_python_toolkit/scripts/utils/plots.py

@@ -0,0 +1,31 @@
+# -*- encoding: utf-8 -*-
+# This class implements plot scripts
+# author: steinkirch
+
+import pandas as pd
+from utils.os import exit_with_error
+
+
+def open_csv(filepath) -> dict:
+    """Load and parse a csv file."""
+
+    try:
+        return pd.read_csv(filepath)
+    except (IOError, FileNotFoundError, TypeError) as e:
+        exit_with_error(f'Failed to parse: "{filepath}": {e}')
+
+
+def save_csv(destination, data, index=False) -> None:
+    """Save data from memory to a csv destination in disk."""
+
+    try:
+        data.to_csv(destination, index=index)
+
+    except (IOError, TypeError) as e:
+        log_error(f'Could not save {destination}: {e}')
+
+
+def plot_bar(y, x) -> None:
+    """Simplest plot for two sets."""
+    df = pd.DataFrame(y, index=x)
+    df.plot.bar(rot=0, subplots=True)

web3_python_toolkit/scripts/utils/strings.py

@@ -0,0 +1,40 @@
+# -*- encoding: utf-8 -*-
+# This class implements string methods used by the other classes.
+# author: steinkirch
+
+from pprint import PrettyPrinter
+
+from utils.os import log_error
+from utils.arithmetics import to_decimal
+
+
+def to_decimal_str(value) -> str:
+    """Format a reserve amount to a suitable string."""
+
+    return str(to_decimal(value))
+
+
+def to_wei_str(value, decimals=None) -> str:
+    """Parse an order string to wei value."""
+
+    decimals = decimals or 18
+    try:
+        return str(value)[:-decimals] + '_' + str(value)[-decimals:]
+    except ValueError as e:
+        log_error(f'Cannot convert to wei: {e}')
+
+
+def to_solution(value) -> str:
+    """Format decimal wei with an underscore for easier reading."""
+
+    return to_wei_str(to_decimal_str(value))
+
+
+def pprint(data, indent=None) -> None:
+    """Print dicts and data in a suitable format"""
+
+    print()
+    indent = indent or 4
+    pp = PrettyPrinter(indent=indent)
+    pp.pprint(data)
+    print()

web3_python_toolkit/scripts/utils/web3_wrapper.py

@@ -0,0 +1,67 @@
+# -*- encoding: utf-8 -*-
+# This class implements an (ongoing) wrapper for web3 libs.
+# author: steinkirch
+
+from web3 import Web3, HTTPProvider, WebsocketProvider, IPCProvider
+from web3.middleware import geth_poa_middleware
+from utils.os import log_info
+
+
+class Web3Wrapper():
+
+    def __init__(self, mode, network):
+        self.mode = mode
+        self.network = network
+
+        self.w3 = None
+        self.pair_contract = None
+
+        self._setup()
+    
+    ##################
+    # PRIVATE METHODS
+    ##################
+    def _setup(self) -> None:
+        self._get_web3_object()
+
+    def _get_web3_object(self) -> None:
+        log_info(f'Setting mode {self.mode} for {self.network}')
+        if self.mode == 'http' or self.mode == 'local_http':
+            self.w3 = Web3(HTTPProvider(self.network))
+        elif self.mode == 'ws' or self.mode == 'local_ws':
+            self.w3 = Web3(WebsocketProvider(self.network))
+        elif self.mode == 'ipc' or self.mode == 'local_ipc':
+            self.w3 = Web3(IPCProvider(self.network))
+        else:
+            log_info(f'Provider type is invalid: {self.mode}. Fix .env.')
+
+
+    ##################
+    # BLOCK METHODS
+    ##################
+    def get_block(self, block_number='latest') -> dict:
+        return dict(self.w3.eth.get_block(block_number))
+
+
+    ##################
+    # LP PAIR METHODS
+    ##################
+    def get_pair_contract(self, address, abi) -> str:
+        self.pair_contract = self.w3.eth.contract(address=address, abi=abi)
+
+    def inject_middleware(self, layer=0) -> None:
+        self.w3.middleware_onion.inject(geth_poa_middleware, 
+                                        layer=layer)
+    
+    def get_reserves(self, block) -> list:
+        return self.pair_contract.functions.getReserves().call({}, block)[:2]
+
+
+    ##############
+    # TX METHODS
+    ##############
+    def get_tx(self, tx) -> dict:
+        return dict(self.w3.eth.get_transaction(tx))
+
+    def get_tx_receipt(self, tx) -> dict:
+        return dict(self.w3.eth.get_transaction_receipt(tx))