From 59a85500ce8c1c5e652704c6456f0793928a936c Mon Sep 17 00:00:00 2001 From: "Mia von Steinkirch, PhD, MSc" <1130416+bt3gl@users.noreply.github.com> Date: Sun, 8 Mar 2020 17:48:58 -0700 Subject: [PATCH 1/6] Create pythonapp.yml --- .github/workflows/pythonapp.yml | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/pythonapp.yml diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml new file mode 100644 index 0000000..25a32ef --- /dev/null +++ b/.github/workflows/pythonapp.yml @@ -0,0 +1,37 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python application + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + pip install pytest + pytest From 2fe37dc4d007b2b1ea1a181eec2008e28083bc38 Mon Sep 17 00:00:00 2001 From: "Mia von Steinkirch, PhD, MSc" <1130416+bt3gl@users.noreply.github.com> Date: Sun, 8 Mar 2020 17:49:15 -0700 Subject: [PATCH 2/6] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2940783..8e79405 100644 --- a/README.md +++ b/README.md @@ -14,4 +14,5 @@ --- + Buy Me A Coffee From aefc5cc7d881b8a10641c616d47b1b04a0b2c0aa Mon Sep 17 00:00:00 2001 From: "Mia von Steinkirch, PhD, MSc" <1130416+bt3gl@users.noreply.github.com> Date: Mon, 9 Mar 2020 17:15:29 -0700 Subject: [PATCH 3/6] Update README.md --- Security_examples/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Security_examples/README.md b/Security_examples/README.md index ba6d3ef..352deed 100644 --- a/Security_examples/README.md +++ b/Security_examples/README.md @@ -1,6 +1,6 @@ -# Medium Examples +# Security Examples -This directory holds any code and snippet that I have published in Medium: +This directory contains the source code published in my Medium artciles: * [Learn Networking with Python’s Socket and Threading Module 🚀](https://medium.com/python-for-the-utopian/learning-networking-with-pythons-socket-and-threading-module-30dc77e1fc59). From 86c2aff32395bc5d0e67803794bb5125901e7ff1 Mon Sep 17 00:00:00 2001 From: "Mia von Steinkirch, PhD, MSc" <1130416+bt3gl@users.noreply.github.com> Date: Mon, 9 Mar 2020 17:15:37 -0700 Subject: [PATCH 4/6] Update README.md --- Security_examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Security_examples/README.md b/Security_examples/README.md index 352deed..83c055a 100644 --- a/Security_examples/README.md +++ b/Security_examples/README.md @@ -1,6 +1,6 @@ # Security Examples -This directory contains the source code published in my Medium artciles: +This directory contains the source code published in my Medium articles: * [Learn Networking with Python’s Socket and Threading Module 🚀](https://medium.com/python-for-the-utopian/learning-networking-with-pythons-socket-and-threading-module-30dc77e1fc59). From 614452d4623f54b5e2818a185379f662bd4ff0d3 Mon Sep 17 00:00:00 2001 From: "Mia von Steinkirch, PhD, MSc" <1130416+bt3gl@users.noreply.github.com> Date: Mon, 9 Mar 2020 17:16:40 -0700 Subject: [PATCH 5/6] Update README.md --- Concurrence_examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Concurrence_examples/README.md b/Concurrence_examples/README.md index c3575f3..cd2e5b6 100644 --- a/Concurrence_examples/README.md +++ b/Concurrence_examples/README.md @@ -1,3 +1,3 @@ ### Concurrence in Python -Please check out this article for the examples in this directory: [Python + Concurrence: Here is What You Need to Know](https://medium.com/python-for-the-utopian/python-concurrence-here-is-what-you-need-to-know-c771d86eda95). +* This directory contains the examples for my Medium article: [Python + Concurrence: Here is What You Need to Know](https://medium.com/python-for-the-utopian/python-concurrence-here-is-what-you-need-to-know-c771d86eda95). From 516c922e0223663b6e22a659cf3f04918cc5d19c Mon Sep 17 00:00:00 2001 From: Mia von Steinkirch Date: Sat, 21 Mar 2020 14:24:55 -0700 Subject: [PATCH 6/6] add concurrence examples --- Concurrence_examples/README.md | 2 +- .../asyncio_simple_example.py | 13 +++-- .../concurrence_future_example.py | 25 --------- .../concurrent_future_example.py | 19 +++++++ Concurrence_examples/concurrent_tasks.py | 51 ------------------- .../concurrent_tasks_with_math.py | 49 ------------------ Concurrence_examples/daemon_example.py | 6 +-- Concurrence_examples/deadlock_example.py | 12 +++++ Concurrence_examples/getenv_example.py | 24 --------- Concurrence_examples/logging_example.py | 17 ++++--- .../multiprocessing_example.py | 6 +-- Concurrence_examples/pool_example.py | 7 +-- Concurrence_examples/safe_thread_example.py | 26 ---------- Concurrence_examples/threads_with_queues.py | 33 +++--------- Concurrence_examples/unsafe_thread_example.py | 22 -------- 15 files changed, 64 insertions(+), 248 deletions(-) delete mode 100644 Concurrence_examples/concurrence_future_example.py create mode 100644 Concurrence_examples/concurrent_future_example.py delete mode 100644 Concurrence_examples/concurrent_tasks.py delete mode 100644 Concurrence_examples/concurrent_tasks_with_math.py create mode 100644 Concurrence_examples/deadlock_example.py delete mode 100644 Concurrence_examples/getenv_example.py delete mode 100644 Concurrence_examples/safe_thread_example.py delete mode 100644 Concurrence_examples/unsafe_thread_example.py diff --git a/Concurrence_examples/README.md b/Concurrence_examples/README.md index cd2e5b6..b85600e 100644 --- a/Concurrence_examples/README.md +++ b/Concurrence_examples/README.md @@ -1,3 +1,3 @@ ### Concurrence in Python -* This directory contains the examples for my Medium article: [Python + Concurrence: Here is What You Need to Know](https://medium.com/python-for-the-utopian/python-concurrence-here-is-what-you-need-to-know-c771d86eda95). +Examples for my Medium article: [Python + Concurrence: A Mnemonic Guide🚦](https://medium.com/python-for-the-utopian/python-concurrence-a-mnemonic-guide-7304867cbfb7). diff --git a/Concurrence_examples/asyncio_simple_example.py b/Concurrence_examples/asyncio_simple_example.py index 3c2a508..d527ec2 100644 --- a/Concurrence_examples/asyncio_simple_example.py +++ b/Concurrence_examples/asyncio_simple_example.py @@ -2,14 +2,13 @@ import asyncio + async def delayed_hello(): - print("Hello ") + print('Hello ') await asyncio.sleep(1) - print("World!") + print('World!') -if __name__ == "__main__": - - loop = asyncio.get_event_loop() - loop.run_until_complete(delayed_hello()) - loop.close() \ No newline at end of file +loop = asyncio.get_event_loop() +loop.run_until_complete(delayed_hello()) +loop.close() \ No newline at end of file diff --git a/Concurrence_examples/concurrence_future_example.py b/Concurrence_examples/concurrence_future_example.py deleted file mode 100644 index 09f83b6..0000000 --- a/Concurrence_examples/concurrence_future_example.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 - -import random -import logging -import concurrent.futures - -WORKER_COUNT = 10 -JOB_COUNT = 10 - -class Job: - def __init__(self, number): - self.number = number - -def process_job(job): - # Wait between 0 and 0.01 seconds. - time.sleep(random.random()/100.0) - logging.info("Job number {:d}".format(job.number)) - -def main(): - with concurrent.futures.ThreadPoolExecutor( - max_workers=WORKER_COUNT) as executor: - futures = [executor.submit(process_job, Job(i)) - for i in range(JOB_COUNT)] - for future in concurrent.futures.as_completed(futures): - pass \ No newline at end of file diff --git a/Concurrence_examples/concurrent_future_example.py b/Concurrence_examples/concurrent_future_example.py new file mode 100644 index 0000000..107bc2f --- /dev/null +++ b/Concurrence_examples/concurrent_future_example.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 + +from time import sleep +from concurrent.futures import ThreadPoolExecutor + + +def return_after_5_secs(message): + sleep(5) + return message + + +pool = ThreadPoolExecutor(3) +future = pool.submit(return_after_5_secs, ('Future message')) + +print(future.done()) + +sleep(5) +print(future.done()) +print(future.result()) \ No newline at end of file diff --git a/Concurrence_examples/concurrent_tasks.py b/Concurrence_examples/concurrent_tasks.py deleted file mode 100644 index 7a6e143..0000000 --- a/Concurrence_examples/concurrent_tasks.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 - -import os -import time -import threading -import multiprocessing - -NUM_WORKERS = 4 - -def run_sleep(): - print("PID: %s, Process Name: %s, Thread Name: %s" % ( - os.getpid(), - multiprocessing.current_process().name, - threading.current_thread().name) - ) - time.sleep(1) - - -# Run tasks serially -start_time = time.time() - -for _ in range(NUM_WORKERS): - run_sleep() - -end_time = time.time() - -print("Serial time=", end_time - start_time) - - -# Run tasks using threads -start_time = time.time() - -threads = [threading.Thread(target=run_sleep) for _ in range(NUM_WORKERS)] -[thread.start() for thread in threads] -[thread.join() for thread in threads] - -end_time = time.time() - -print("Threads time=", end_time - start_time) - - -# Run tasks using processes -start_time = time.time() - -processes = [multiprocessing.Process(target=run_sleep()) for _ in range(NUM_WORKERS)] -[process.start() for process in processes] -[process.join() for process in processes] - -end_time = time.time() - -print("Parallel time=", end_time - start_time) \ No newline at end of file diff --git a/Concurrence_examples/concurrent_tasks_with_math.py b/Concurrence_examples/concurrent_tasks_with_math.py deleted file mode 100644 index 04a1753..0000000 --- a/Concurrence_examples/concurrent_tasks_with_math.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 - -import os -import time -import threading -import multiprocessing - -NUM_WORKERS = 4 - -def run_numbers(): - print("PID: %s, Process Name: %s, Thread Name: %s" % ( - os.getpid(), - multiprocessing.current_process().name, - threading.current_thread().name) - ) - x = 0 - while x < 10000000: - x += 1 - -start_time = time.time() - -for _ in range(NUM_WORKERS): - run_numbers() - -end_time = time.time() - -print("Serial time=", end_time - start_time) - - -start_time = time.time() - -threads = [threading.Thread(target=run_numbers) for _ in range(NUM_WORKERS)] -[thread.start() for thread in threads] -[thread.join() for thread in threads] - -end_time = time.time() - -print("Threads time=", end_time - start_time) - - -start_time = time.time() - -processes = [multiprocessing.Process(target=run_numbers) for _ in range(NUM_WORKERS)] -[process.start() for process in processes] -[process.join() for process in processes] -end_time = time.time() - - -print("Parallel time=", end_time - start_time) \ No newline at end of file diff --git a/Concurrence_examples/daemon_example.py b/Concurrence_examples/daemon_example.py index 3f5721d..0059eb9 100644 --- a/Concurrence_examples/daemon_example.py +++ b/Concurrence_examples/daemon_example.py @@ -7,10 +7,10 @@ import multiprocessing def daemon(): p = multiprocessing.current_process() - print('Starting: {}, {}'.format(p.name, p.pid)) + sys.stdout.flush() - time.sleep(2) + time.sleep(1) print('Exiting : {}, {}'.format(p.name, p.pid)) sys.stdout.flush() @@ -18,8 +18,8 @@ def daemon(): def non_daemon(): p = multiprocessing.current_process() - print('Starting: {}, {}'.format(p.name, p.pid)) + sys.stdout.flush() print('Exiting : {}, {}'.format(p.name, p.pid)) diff --git a/Concurrence_examples/deadlock_example.py b/Concurrence_examples/deadlock_example.py new file mode 100644 index 0000000..af1009a --- /dev/null +++ b/Concurrence_examples/deadlock_example.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 + +import threading + +l = threading.Lock() +print("Before first lock acquire.") + +l.acquire() +print("Before second lock acquire.") + +l.acquire() +print("Lock was acquired twice") diff --git a/Concurrence_examples/getenv_example.py b/Concurrence_examples/getenv_example.py deleted file mode 100644 index d815339..0000000 --- a/Concurrence_examples/getenv_example.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python3 - -import time -from gevent.pool import Pool -from gevent import monkey - -# Note that you can spawn many workers with gevent since the cost of creating and switching is very low -NUM_WORKERS = 4 - -# Monkey-Patch socket module for HTTP requests -monkey.patch_socket() - -start_time = time.time() - -pool = Pool(NUM_WORKERS) -for address in WEBSITE_LIST: - pool.spawn(check_website, address) - -# Wait for stuff to finish -pool.join() - -end_time = time.time() - -print("Time for GreenSquirrel: %ssecs" % (end_time - start_time)) diff --git a/Concurrence_examples/logging_example.py b/Concurrence_examples/logging_example.py index 9af8b4c..e666941 100644 --- a/Concurrence_examples/logging_example.py +++ b/Concurrence_examples/logging_example.py @@ -1,15 +1,16 @@ #!/usr/bin/env python3 -import multiprocessing -import logging import sys +import logging +import multiprocessing + def worker(): - print 'Doing some work' + print('Doing some work...') sys.stdout.flush() -if __name__ == '__main__': - multiprocessing.log_to_stderr(logging.DEBUG) - p = multiprocessing.Process(target=worker) - p.start() - p.join() \ No newline at end of file + +multiprocessing.log_to_stderr(logging.DEBUG) +p = multiprocessing.Process(target=worker) +p.start() +p.join() \ No newline at end of file diff --git a/Concurrence_examples/multiprocessing_example.py b/Concurrence_examples/multiprocessing_example.py index fda3c45..004ad7d 100644 --- a/Concurrence_examples/multiprocessing_example.py +++ b/Concurrence_examples/multiprocessing_example.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - import time import random import multiprocessing @@ -12,5 +10,5 @@ def worker(n): for i in range(5): - t = multiprocessing.Process(target=worker, args=(i,)) - t.start() + p = multiprocessing.Process(target=worker, args=(i,)) + p.start() diff --git a/Concurrence_examples/pool_example.py b/Concurrence_examples/pool_example.py index ebcb3c4..14f4f2d 100644 --- a/Concurrence_examples/pool_example.py +++ b/Concurrence_examples/pool_example.py @@ -2,9 +2,10 @@ from multiprocessing import Pool + def f(x): return x*x -if __name__ == '__main__': - p = Pool(5) - print(p.map(f, [1, 2, 3])) + +p = Pool(5) +print(p.map(f, [1, 2, 3])) \ No newline at end of file diff --git a/Concurrence_examples/safe_thread_example.py b/Concurrence_examples/safe_thread_example.py deleted file mode 100644 index 9d7389c..0000000 --- a/Concurrence_examples/safe_thread_example.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 - -import threading - -counter = 0 -threads = [] - -lock = threading.Lock() - - -def count_with_lock(): - - global counter - - for _ in range(100): - with lock: - counter += 1 - - -for _ in range(100): - thread = threading.Thread(target=count_with_lock) - thread.start() - threads.append(thread) - - -print(counter) \ No newline at end of file diff --git a/Concurrence_examples/threads_with_queues.py b/Concurrence_examples/threads_with_queues.py index 921b144..6653c39 100644 --- a/Concurrence_examples/threads_with_queues.py +++ b/Concurrence_examples/threads_with_queues.py @@ -1,36 +1,19 @@ -#!/usr/bin/env python3 - -import time from queue import Queue from threading import Thread - + + NUM_WORKERS = 4 task_queue = Queue() - + + def worker(): - # Constantly check the queue for addresses while True: address = task_queue.get() run_function(address) - - # Mark the processed task as done task_queue.task_done() - -start_time = time.time() - -# Create the worker threads + + threads = [Thread(target=worker) for _ in range(NUM_WORKERS)] - -# Add the websites to the task queue -[task_queue.put(item) for item in SOME_LIST] - -# Start all the workers +[task_queue.put(item) for item in threads] [thread.start() for thread in threads] - -# Wait for all the tasks in the queue to be processed -task_queue.join() - - -end_time = time.time() - -print('Time: {} secs'.format(end_time - start_time)) \ No newline at end of file +task_queue.join() \ No newline at end of file diff --git a/Concurrence_examples/unsafe_thread_example.py b/Concurrence_examples/unsafe_thread_example.py deleted file mode 100644 index 9aab375..0000000 --- a/Concurrence_examples/unsafe_thread_example.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 - -import time -import threading - -counter = 0 -threads = [] - -def count(): - global counter - for _ in range(100): - counter += 1 - -for _ in range(100): - thread = threading.Thread(target=count) - thread.start() - threads.append(thread) - -for thread in threads: - thread.join() - -print(f"Count: {counter}") \ No newline at end of file