Always save remote image data (#4875)

* Always save remote image data

* cleanup

---------

Co-authored-by: Felix Ableitner <me@nutomic.com>
Co-authored-by: Dessalines <dessalines@users.noreply.github.com>
This commit is contained in:
Sander Saarend 2024-09-23 10:58:49 +03:00 committed by GitHub
parent 8cdfc148d7
commit 25df9d255b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 23 additions and 44 deletions

View File

@ -17,7 +17,7 @@ use lemmy_db_schema::{
community::{Community, CommunityModerator, CommunityUpdateForm}, community::{Community, CommunityModerator, CommunityUpdateForm},
community_block::CommunityBlock, community_block::CommunityBlock,
email_verification::{EmailVerification, EmailVerificationForm}, email_verification::{EmailVerification, EmailVerificationForm},
images::RemoteImage, images::{ImageDetails, RemoteImage},
instance::Instance, instance::Instance,
instance_block::InstanceBlock, instance_block::InstanceBlock,
local_site::LocalSite, local_site::LocalSite,
@ -1012,6 +1012,7 @@ pub async fn process_markdown(
if context.settings().pictrs_config()?.image_mode() == PictrsImageMode::ProxyAllImages { if context.settings().pictrs_config()?.image_mode() == PictrsImageMode::ProxyAllImages {
let (text, links) = markdown_rewrite_image_links(text); let (text, links) = markdown_rewrite_image_links(text);
RemoteImage::create(&mut context.pool(), links.clone()).await?;
// Create images and image detail rows // Create images and image detail rows
for link in links { for link in links {
@ -1021,7 +1022,7 @@ pub async fn process_markdown(
let proxied = let proxied =
build_proxied_image_url(&link, &context.settings().get_protocol_and_hostname())?; build_proxied_image_url(&link, &context.settings().get_protocol_and_hostname())?;
let details_form = details.build_image_details_form(&proxied); let details_form = details.build_image_details_form(&proxied);
RemoteImage::create(&mut context.pool(), &details_form).await?; ImageDetails::create(&mut context.pool(), &details_form).await?;
} }
} }
Ok(text) Ok(text)
@ -1057,13 +1058,15 @@ async fn proxy_image_link_internal(
if link.domain() == Some(&context.settings().hostname) { if link.domain() == Some(&context.settings().hostname) {
Ok(link.into()) Ok(link.into())
} else if image_mode == PictrsImageMode::ProxyAllImages { } else if image_mode == PictrsImageMode::ProxyAllImages {
RemoteImage::create(&mut context.pool(), vec![link.clone()]).await?;
let proxied = build_proxied_image_url(&link, &context.settings().get_protocol_and_hostname())?; let proxied = build_proxied_image_url(&link, &context.settings().get_protocol_and_hostname())?;
// This should fail softly, since pictrs might not even be running // This should fail softly, since pictrs might not even be running
let details_res = fetch_pictrs_proxied_image_details(&link, context).await; let details_res = fetch_pictrs_proxied_image_details(&link, context).await;
if let Ok(details) = details_res { if let Ok(details) = details_res {
let details_form = details.build_image_details_form(&proxied); let details_form = details.build_image_details_form(&proxied);
RemoteImage::create(&mut context.pool(), &details_form).await?; ImageDetails::create(&mut context.pool(), &details_form).await?;
}; };
Ok(proxied.into()) Ok(proxied.into())
@ -1209,7 +1212,7 @@ mod tests {
assert!( assert!(
RemoteImage::validate(&mut context.pool(), remote_image.into()) RemoteImage::validate(&mut context.pool(), remote_image.into())
.await .await
.is_err() .is_ok()
); );
} }
} }

View File

@ -1,14 +1,7 @@
use crate::{ use crate::{
newtypes::DbUrl, newtypes::DbUrl,
schema::{image_details, local_image, remote_image}, schema::{image_details, local_image, remote_image},
source::images::{ source::images::{ImageDetails, ImageDetailsForm, LocalImage, LocalImageForm, RemoteImage},
ImageDetails,
ImageDetailsForm,
LocalImage,
LocalImageForm,
RemoteImage,
RemoteImageForm,
},
utils::{get_conn, DbPool}, utils::{get_conn, DbPool},
}; };
use diesel::{ use diesel::{
@ -20,7 +13,8 @@ use diesel::{
NotFound, NotFound,
QueryDsl, QueryDsl,
}; };
use diesel_async::{AsyncPgConnection, RunQueryDsl}; use diesel_async::RunQueryDsl;
use url::Url;
impl LocalImage { impl LocalImage {
pub async fn create( pub async fn create(
@ -38,7 +32,7 @@ impl LocalImage {
.get_result::<Self>(conn) .get_result::<Self>(conn)
.await; .await;
ImageDetails::create(conn, image_details_form).await?; ImageDetails::create(&mut conn.into(), image_details_form).await?;
local_insert local_insert
}) as _ }) as _
@ -60,26 +54,16 @@ impl LocalImage {
} }
impl RemoteImage { impl RemoteImage {
pub async fn create(pool: &mut DbPool<'_>, form: &ImageDetailsForm) -> Result<usize, Error> { pub async fn create(pool: &mut DbPool<'_>, links: Vec<Url>) -> Result<usize, Error> {
let conn = &mut get_conn(pool).await?; let conn = &mut get_conn(pool).await?;
conn let forms = links
.build_transaction() .into_iter()
.run(|conn| { .map(|url| remote_image::dsl::link.eq::<DbUrl>(url.into()))
Box::pin(async move { .collect::<Vec<_>>();
let remote_image_form = RemoteImageForm { insert_into(remote_image::table)
link: form.link.clone(), .values(forms)
}; .on_conflict_do_nothing()
let remote_insert = insert_into(remote_image::table) .execute(conn)
.values(remote_image_form)
.on_conflict_do_nothing()
.execute(conn)
.await;
ImageDetails::create(conn, form).await?;
remote_insert
}) as _
})
.await .await
} }
@ -100,10 +84,9 @@ impl RemoteImage {
} }
impl ImageDetails { impl ImageDetails {
pub(crate) async fn create( pub async fn create(pool: &mut DbPool<'_>, form: &ImageDetailsForm) -> Result<usize, Error> {
conn: &mut AsyncPgConnection, let conn = &mut get_conn(pool).await?;
form: &ImageDetailsForm,
) -> Result<usize, Error> {
insert_into(image_details::table) insert_into(image_details::table)
.values(form) .values(form)
.on_conflict_do_nothing() .on_conflict_do_nothing()

View File

@ -51,13 +51,6 @@ pub struct RemoteImage {
pub published: DateTime<Utc>, pub published: DateTime<Utc>,
} }
#[derive(Debug, Clone)]
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
#[cfg_attr(feature = "full", diesel(table_name = remote_image))]
pub struct RemoteImageForm {
pub link: DbUrl,
}
#[skip_serializing_none] #[skip_serializing_none]
#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
#[cfg_attr(feature = "full", derive(Queryable, Selectable, Identifiable, TS))] #[cfg_attr(feature = "full", derive(Queryable, Selectable, Identifiable, TS))]