From d32e821bd72f28f77a6816ef8c4e04f9f2b80aea Mon Sep 17 00:00:00 2001 From: Mia von Steinkirch Date: Tue, 17 Dec 2019 09:25:46 -0800 Subject: [PATCH] add blerplate --- dash_app/.env_example | 4 ++ dash_app/Makefile | 26 +++++++++++ dash_app/Procfile | 1 + dash_app/README.md | 62 ++++++++++++++++++++++++-- dash_app/app.py | 70 +++++++++++++++++++++++++++++ dash_app/requirements.txt | 31 +++++++++++++ dash_app/wrappers/k8s_wrapper.py | 75 ++++++++++++++++++++++++++++++++ dash_app/wrappers/settings.py | 12 +++++ dash_app/wrappers/style.py | 45 +++++++++++++++++++ 9 files changed, 323 insertions(+), 3 deletions(-) create mode 100644 dash_app/.env_example create mode 100644 dash_app/Makefile create mode 100644 dash_app/Procfile create mode 100644 dash_app/app.py create mode 100644 dash_app/requirements.txt create mode 100644 dash_app/wrappers/k8s_wrapper.py create mode 100644 dash_app/wrappers/settings.py create mode 100644 dash_app/wrappers/style.py diff --git a/dash_app/.env_example b/dash_app/.env_example new file mode 100644 index 0000000..e86f103 --- /dev/null +++ b/dash_app/.env_example @@ -0,0 +1,4 @@ +# Env constants to be used in the program + +## General constants +PORT=8051 \ No newline at end of file diff --git a/dash_app/Makefile b/dash_app/Makefile new file mode 100644 index 0000000..12f7c72 --- /dev/null +++ b/dash_app/Makefile @@ -0,0 +1,26 @@ +.PHONY: default clean run install test version dist + +default: run + +clean: + @find . -iname '*.py[co]' -delete + @find . -iname '__pycache__' -delete + @rm -rf dist/ + @rm -rf build/ + @rm -rf *.egg-info + +run: + python3 app.py + +install: + pip3 install -r requirements.txt + +test: + pytest -vvv + +dist: clean + python3 setup.py sdist + python3 setup.py bdist_wheel + +version: dist + python3 setup.py --version diff --git a/dash_app/Procfile b/dash_app/Procfile new file mode 100644 index 0000000..2c3403a --- /dev/null +++ b/dash_app/Procfile @@ -0,0 +1 @@ +web: gunicorn app:server --workers 4 diff --git a/dash_app/README.md b/dash_app/README.md index 5c2dd36..f9b1e50 100644 --- a/dash_app/README.md +++ b/dash_app/README.md @@ -1,7 +1,63 @@ -# Dashboards in Python +# Infrastructure Dashboards + +This repository contains the source code for the infrastructure dashboards developed with [plot.ly and dash](https://dash.plot.ly/). + +### Why Plotly + +Plotly allows you to make beautiful and interactive dashboards in just a few lines of code, with data virtually any source that has a Python API. + +### How do the Infrastructure Dashboards work? + +Plotly objects consist of one or more data components and a layout component. Both have subcomponents. Most, but not all, of the formatting is controlled in the layout. + +This app is divided into the following resources: + +* `wrappers/`: where the API wrappers, `style.py` and `settings.py` live. +* `.env`: where all the constants and keys/secrets are set. +* `app.py`: entry point for the dashboard app: where the layout elements and the callback functions are set. + +----- + +## Running locally in dev mode + +### Setting up + +Add an `.env` file: + +``` +cp .env_example .env +``` + +Create an virtual environment and install dependencies: + +``` +virtualenv venv +source venv/bin/activate +``` + +### Installing + +``` +make install +``` + +### Running + +Run server at localhost: + +``` +make run +``` + +The dahsboard should be available at `http://127.0.0.1:8051/` (note that the port is set in `.env`). +------------- -### Learning Resources +## Learning Resources + +* [Build Your own Data Dashboard](https://towardsdatascience.com/build-your-own-data-dashboard-93e4848a0dcf). +* [A Python Programmers’ Guide to Dashboarding](https://medium.com/@drimik99/a-python-programmers-guide-to-dashboarding-part-1-8db0c48eee9d). +* [Interactive Python Dashboards with Plotly and Dash](https://www.udemy.com/course/interactive-python-dashboards-with-plotly-and-dash). +* [Make Your Data Visualizations Interactive with Plotly](https://towardsdatascience.com/its-2019-make-your-data-visualizations-interactive-with-plotly-b361e7d45dc6). -* [Interactive Python Dashboards with plotly and dash](https://www.udemy.com/course/interactive-python-dashboards-with-plotly-and-dash/). diff --git a/dash_app/app.py b/dash_app/app.py new file mode 100644 index 0000000..5dc8ceb --- /dev/null +++ b/dash_app/app.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python3 + +import os +import dash +import plotly + +import pandas as pd +import dash_table as dt +import dash_core_components as dcc +import dash_html_components as html +from dash.dependencies import Input, Output, State + + +# ----------------------------- +# ------ Instantiate app ------ +# ----------------------------- +app = dash.Dash(__name__) + + +# ------------------------------- +# ---- Start app HTML layout ---- +# ------------------------------- + +app.layout = html.Div(children=[ + + # Main Title + html.Div([ + html.H1(s.title_h1, + style=s.style_title_h1 + ) + ], + ), + + # Tabs + dcc.Tabs([ + + dcc.Tab(label='tab1', children=[ + html.Div([ + html.H2('title 1', + style=s.style_title_h2 + ) + ], + ), + ]), + + dcc.Tab(label='tab2', children=[ + html.Div([ + html.H2('tab 2', + style=s.style_title_h2 + ) + ], + ), + ]), +], style=s.style_all) + + +# --------------------------------- +# ----- App Callback methods ------ +# --------------------------------- + + + +# --------------------------------- +# ----------- Run web app --------- +# --------------------------------- +if __name__ == '__main__': + app.scripts.config.serve_locally = True + app.css.config.serve_locally = True + app.run_server(debug=True, port=se.PORT) diff --git a/dash_app/requirements.txt b/dash_app/requirements.txt new file mode 100644 index 0000000..7744883 --- /dev/null +++ b/dash_app/requirements.txt @@ -0,0 +1,31 @@ +certifi==2018.8.13 +chardet==3.0.4 +click==6.7 +dash==0.34.0 +dash-core-components==0.27.1 +dash-html-components==0.11.0 +dash-renderer==0.13.0 +decorator==4.3.0 +Flask==1.0.2 +Flask-Compress==1.4.0 +gunicorn==19.9.0 +idna==2.7 +ipython-genutils==0.2.0 +itsdangerous==0.24 +Jinja2==2.10 +jsonschema==2.6.0 +jupyter-core==4.4.0 +MarkupSafe==1.0 +nbformat==4.4.0 +plotly==3.1.1 +pytz==2018.5 +requests==2.19.1 +retrying==1.3.3 +six==1.11.0 +traitlets==4.3.2 +urllib3==1.23 +Werkzeug==0.14.1 +numpy==1.17.4 +pandas==0.25.3 +dash-table-experiments==0.6.0 +python-dotenv==0.10.3 diff --git a/dash_app/wrappers/k8s_wrapper.py b/dash_app/wrappers/k8s_wrapper.py new file mode 100644 index 0000000..ee501c0 --- /dev/null +++ b/dash_app/wrappers/k8s_wrapper.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# Wrapper for K8s API + +import os + +from kubernetes import client, config +from wrappers.settings import ORG, K8S_ENV_LIST + + +def get_pod_data(namespace): + ''' Retrieve and return data from K8s from a given namepasce.''' + + config.load_kube_config() + api_instance = client.CoreV1Api() + + try: + ret = api_instance.list_pod_for_all_namespaces(watch=False) + + ip_list, name_list, stat_list = [], [], [] + for item in ret.items: + if (item.metadata.namespace != namespace): + continue + + ip_list.append(item.status.pod_ip) + name_list.append(item.metadata.name) + stat_list.append(item.status.phase) + + dtag_list, dimage_list = [], [] + for pod in name_list: + dtag_str, dimage_str = get_pod_info(namespace, api_instance, pod) + dtag_list.append(dtag_str) + dimage_list.append(dimage_str) + + + return { + 'Status': stat_list, + 'Name': name_list, + 'Pod IP': ip_list, + 'Docker Image': dimage_list, + 'Docker Tag': dtag_list + } + + except Exception as e: + print("Error retrieving K8s data: \n{}".format(e)) + return None + +def generate_env_dict(): + ''' Retrieve K8s namespaces from env, and return a dict of it. ''' + env_list = K8S_ENV_LIST.split(',') + return [{'label': i.strip(), 'value': i.strip()} for i in env_list] + + +def get_pod_info(namespace, api_instance, pod_name): + ''' Retrieve the Docker info from a given pod. ''' + api_response = api_instance.read_namespaced_pod(pod_name, namespace) + + dtag_str, dimage_str = '', '' + if len(api_response.spec.containers) < 2: + dimage_str, dtag_str = api_response.spec.containers[0].image.split(':') + else: + for container in api_response.spec.containers: + im, tg = container.image.split(':') + dimage_str += '{}; '.format(im) + dtag_str += '{}; '.format(tg) + + + return dtag_str, dimage_str + + +if __name__ == '__main__': + + test_namespace = 'staging' + print('Printing k8s data for giving namespace') + print(get_pod_data(test_namespace)) + diff --git a/dash_app/wrappers/settings.py b/dash_app/wrappers/settings.py new file mode 100644 index 0000000..e45b671 --- /dev/null +++ b/dash_app/wrappers/settings.py @@ -0,0 +1,12 @@ +# Load all env variables from .env file + +import os + +from dotenv import load_dotenv +from pathlib import Path + +env_path = Path('.') / '.env' +load_dotenv(dotenv_path=env_path) + +# General constants +PORT = os.getenv('PORT') \ No newline at end of file diff --git a/dash_app/wrappers/style.py b/dash_app/wrappers/style.py new file mode 100644 index 0000000..7ae22c0 --- /dev/null +++ b/dash_app/wrappers/style.py @@ -0,0 +1,45 @@ +# -*- coding: utf8 -*- +# Define dashboard styling + +# ----------------------------- +# --- Titles and strings ------ +# ----------------------------- +title_h1 = 'Big Title' + + +# ----------------------------- +# --- elements ids ------------ +# ----------------------------- + + +# ----------------------------- +# --- CSS style dictionaries -- +# ----------------------------- + +color_palette = ['#052a4e', # dark blue + '#fd8283', # light red + '#e4faff'] # light blue + +style_all = {'margin':'auto', + 'padding':20, + 'width':'95%', + 'fontFamily':'helvetica', + 'fontSize':'14', + 'color':color_palette[0], + 'background':color_palette[2], + } + +style_title_h1 = {'textAlign':'left', + 'color':color_palette[1], + 'font-weight':'bold', + 'fontSize':'32', + 'text-transform':'uppercase', + 'padding':20, + } + +style_title_h2 = {'textAlign':'center', + 'font-weight':'bold', + 'text-transform':'uppercase', + 'fontSize':'20', + } +