From b07d8125d04a5f311dd4ef4d19fa1e2a6740852e Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 28 Jul 2025 16:23:51 -0700 Subject: [PATCH 1/4] skip psycopg.rows.scalar_row --- brozzler/ydl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index bbe0a72..c1dd3cb 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -165,9 +165,9 @@ class VideoDataClient: ), ) try: - results = self._execute_pg_query( - pg_query, row_factory=psycopg.rows.scalar_row, fetchall=True - ) + results = [row[0] for row in self._execute_pg_query( + pg_query, fetchall=True + )] except Exception as e: logger.warn("postgres query failed: %s", e) results = [] From 923ae1f2a247ed87c624255158fe3da7c65bc686 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 28 Jul 2025 16:31:10 -0700 Subject: [PATCH 2/4] fix logged errors --- brozzler/ydl.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index c1dd3cb..4291eef 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -76,9 +76,6 @@ class VideoCaptureRecord: class VideoDataClient: - import psycopg - from psycopg_pool import ConnectionPool, PoolTimeout - VIDEO_DATA_SOURCE = os.getenv("VIDEO_DATA_SOURCE") def __init__(self): @@ -165,9 +162,9 @@ class VideoDataClient: ), ) try: - results = [row[0] for row in self._execute_pg_query( - pg_query, fetchall=True - )] + results = [ + row[0] for row in self._execute_pg_query(pg_query, fetchall=True) + ] except Exception as e: logger.warn("postgres query failed: %s", e) results = [] @@ -666,8 +663,8 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints): ) except Exception as e: logger.warning("hit exception processing worker._video_data: %s", e) - if uncaptured_youtube_watch_pages: - outlinks.update(uncaptured_youtube_watch_pages) + if uncaptured_watch_pages: + outlinks.update(uncaptured_watch_pages) else: outlinks = { "https://www.youtube.com/watch?v=%s" % e["id"] From 59f7830711aa243f2266c12edce988c7418e659c Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 28 Jul 2025 16:40:42 -0700 Subject: [PATCH 3/4] import psycopg only in class VideoDataClient --- brozzler/ydl.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 4291eef..6f25327 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -28,11 +28,9 @@ from dataclasses import dataclass from typing import Any, List, Optional import doublethink -import psycopg import structlog import urlcanon import yt_dlp -from psycopg_pool import ConnectionPool, PoolTimeout from yt_dlp.utils import ExtractorError, match_filter_func import brozzler @@ -76,6 +74,9 @@ class VideoCaptureRecord: class VideoDataClient: + import psycopg + from psycopg_pool import ConnectionPool, PoolTimeout + VIDEO_DATA_SOURCE = os.getenv("VIDEO_DATA_SOURCE") def __init__(self): From e8dc41ca30ea46bd390d37c2b439bdca697b912e Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 28 Jul 2025 16:51:09 -0700 Subject: [PATCH 4/4] import psycopg only at top of file --- brozzler/ydl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 6f25327..4291eef 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -28,9 +28,11 @@ from dataclasses import dataclass from typing import Any, List, Optional import doublethink +import psycopg import structlog import urlcanon import yt_dlp +from psycopg_pool import ConnectionPool, PoolTimeout from yt_dlp.utils import ExtractorError, match_filter_func import brozzler @@ -74,9 +76,6 @@ class VideoCaptureRecord: class VideoDataClient: - import psycopg - from psycopg_pool import ConnectionPool, PoolTimeout - VIDEO_DATA_SOURCE = os.getenv("VIDEO_DATA_SOURCE") def __init__(self):