Merge branch 'main' into plugin-system

* Update rustls

* Format code

.gitignore

@@ -36,4 +36,4 @@ dev_pgdata/
 *.sqldump
 
 # compiled example plugin
-example_plugin/plugin.wasm
+plugins/plugin.wasm

.woodpecker.yml

@@ -229,7 +229,7 @@ steps:
       DO_WRITE_HOSTS_FILE: "1"
     commands:
       - *install_pnpm
-      - apt update && apt install -y bash curl postgresql-client golang tinygo
+      - apt update && apt install -y bash curl postgresql-client golang
       - bash api_tests/prepare-drone-federation-test.sh
       - cd api_tests/
       - pnpm i

Cargo.toml

@@ -157,10 +157,10 @@ ts-rs = { version = "7.1.1", features = [
   "chrono-impl",
   "no-serde-warnings",
 ] }
-rustls = { version = "0.21.11", features = ["dangerous_configuration"] }
+rustls = { version = "0.23.5", features = ["ring"] }
 futures-util = "0.3.30"
 tokio-postgres = "0.7.10"
-tokio-postgres-rustls = "0.10.0"
+tokio-postgres-rustls = "0.12.0"
 urlencoding = "2.1.3"
 enum-map = "2.7"
 moka = { version = "0.12.7", features = ["future"] }
@@ -204,7 +204,11 @@ chrono = { workspace = true }
 prometheus = { version = "0.13.3", features = ["process"] }
 serial_test = { workspace = true }
 clap = { workspace = true }
+serde.workspace = true
 actix-web-prom = "0.7.0"
+actix-http = "3.6.0"
+extism = "1.2.0"
+extism-convert = "1.2.0"
 
 [dev-dependencies]
 pretty_assertions = { workspace = true }

api_tests/prepare-drone-federation-test.sh

@@ -74,10 +74,9 @@ LEMMY_CONFIG_LOCATION=./docker/federation/lemmy_delta.hjson \
   target/lemmy_server >$LOG_DIR/lemmy_delta.out 2>&1 &
 
 # plugin setup
-pushd example_plugin
-# TODO: not in ubuntu repos, better to use only `go`
-# TODO: prevent it from creating useless `go` folder in home dir
-tinygo build -o plugin.wasm -target wasi main.go
+pushd plugins
+# need to use tinygo because apparently go has only experimental support for wasm target
+GOPATH=$HOME/.local/share/go tinygo build -o plugin.wasm -target wasi main.go
 popd
 
 echo "start epsilon"

crates/api_common/Cargo.toml

@@ -25,7 +25,7 @@ full = [
   "lemmy_db_views_moderator/full",
   "lemmy_utils/full",
   "activitypub_federation",
-  "encoding",
+  "encoding_rs",
   "reqwest-middleware",
   "webpage",
   "ts-rs",
@@ -69,7 +69,7 @@ mime = { version = "0.3.17", optional = true }
 webpage = { version = "1.6", default-features = false, features = [
   "serde",
 ], optional = true }
-encoding = { version = "0.2.33", optional = true }
+encoding_rs = { version = "0.8.34", optional = true }
 jsonwebtoken = { version = "8.3.0", optional = true }
 # necessary for wasmt compilation
 getrandom = { version = "0.2.14", features = ["js"] }

crates/api_common/src/request.rs

@@ -6,7 +6,7 @@ use crate::{
   utils::{local_site_opt_to_sensitive, proxy_image_link, proxy_image_link_opt_apub},
 };
 use activitypub_federation::config::Data;
-use encoding::{all::encodings, DecoderTrap};
+use encoding_rs::{Encoding, UTF_8};
 use lemmy_db_schema::{
   newtypes::DbUrl,
   source::{
@@ -160,11 +160,9 @@ fn extract_opengraph_data(html_bytes: &[u8], url: &Url) -> LemmyResult<OpenGraph
   // proper encoding. If the specified encoding cannot be found, fall back to the original UTF-8
   // version.
   if let Some(charset) = page.meta.get("charset") {
-    if charset.to_lowercase() != "utf-8" {
-      if let Some(encoding_ref) = encodings().iter().find(|e| e.name() == charset) {
-        if let Ok(html_with_encoding) = encoding_ref.decode(html_bytes, DecoderTrap::Replace) {
-          page = HTML::from_string(html_with_encoding, None)?;
-        }
+    if charset != UTF_8.name() {
+      if let Some(encoding) = Encoding::for_label(charset.as_bytes()) {
+        page = HTML::from_string(encoding.decode(html_bytes).0.into(), None)?;
       }
     }
   }

crates/api_crud/Cargo.toml

@@ -28,11 +28,8 @@ uuid = { workspace = true }
 moka.workspace = true
 once_cell.workspace = true
 anyhow.workspace = true
-serde.workspace = true
 webmention = "0.5.0"
 accept-language = "3.1.0"
-extism = "1.2.0"
-extism-convert = "1.2.0"
 
 [package.metadata.cargo-machete]
 ignored = ["futures"]

crates/api_crud/src/post/create.rs

@@ -1,6 +1,5 @@
 use activitypub_federation::config::Data;
 use actix_web::web::Json;
-use extism::*;
 use lemmy_api_common::{
   build_response::build_post_response,
   context::LemmyContext,
@@ -47,20 +46,16 @@ use lemmy_utils::{
     },
   },
 };
-use serde::Serialize;
-use std::{ffi::OsStr, fs::read_dir};
 use tracing::Instrument;
 use url::Url;
 use webmention::{Webmention, WebmentionError};
 
 #[tracing::instrument(skip(context))]
 pub async fn create_post(
-  mut data: Json<CreatePost>,
+  data: Json<CreatePost>,
   context: Data<LemmyContext>,
   local_user_view: LocalUserView,
 ) -> LemmyResult<Json<PostResponse>> {
-  plugin_hook("api_before_create_post", &mut (*data))?;
-
   let local_site = LocalSite::read(&mut context.pool()).await?;
 
   honeypot_check(&data.honeypot)?;
@@ -205,45 +200,5 @@ pub async fn create_post(
     }
   };
 
-  let mut res = build_post_response(&context, community_id, &local_user_view.person, post_id)
-    .await?
-    .0;
-
-  plugin_hook("api_after_create_post", &mut res)?;
-  Ok(Json(res))
-}
-
-fn load_plugins() -> LemmyResult<Plugin> {
-  // TODO: make dir configurable via env var
-  // TODO: should only read fs once at startup for performance
-  let plugin_paths = read_dir("example_plugin")?;
-
-  let mut wasm_files = vec![];
-  for path in plugin_paths {
-    let path = path?.path();
-    if path.extension() == Some(OsStr::new("wasm")) {
-      wasm_files.push(path);
-    }
-  }
-  let manifest = Manifest::new(wasm_files);
-  let plugin = Plugin::new(manifest, [], true)?;
-  Ok(plugin)
-}
-
-fn plugin_hook<T: Serialize + for<'de> serde::Deserialize<'de> + Clone>(
-  name: &'static str,
-  data: &mut T,
-) -> LemmyResult<()> {
-  let mut plugin = load_plugins()?;
-  if plugin.function_exists(name) {
-    *data = plugin
-      .call::<extism_convert::Json<T>, extism_convert::Json<T>>(name, (*data).clone().into())
-      .map_err(|e| {
-        dbg!(&e);
-        LemmyErrorType::PluginError(e.to_string())
-      })?
-      .0
-      .into();
-  }
-  Ok(())
+  build_post_response(&context, community_id, &local_user_view.person, post_id).await
 }

crates/db_schema/src/utils.rs

@@ -33,13 +33,22 @@ use lemmy_utils::{
 use once_cell::sync::Lazy;
 use regex::Regex;
 use rustls::{
-  client::{ServerCertVerified, ServerCertVerifier},
-  ServerName,
+  client::danger::{
+    DangerousClientConfigBuilder,
+    HandshakeSignatureValid,
+    ServerCertVerified,
+    ServerCertVerifier,
+  },
+  crypto::{self, verify_tls12_signature, verify_tls13_signature},
+  pki_types::{CertificateDer, ServerName, UnixTime},
+  ClientConfig,
+  DigitallySignedStruct,
+  SignatureScheme,
 };
 use std::{
   ops::{Deref, DerefMut},
   sync::Arc,
-  time::{Duration, SystemTime},
+  time::Duration,
 };
 use tracing::error;
 use url::Url;
@@ -312,10 +321,11 @@ pub fn diesel_option_overwrite_to_url_create(opt: &Option<String>) -> LemmyResul
 
 fn establish_connection(config: &str) -> BoxFuture<ConnectionResult<AsyncPgConnection>> {
   let fut = async {
-    let rustls_config = rustls::ClientConfig::builder()
-      .with_safe_defaults()
-      .with_custom_certificate_verifier(Arc::new(NoCertVerifier {}))
-      .with_no_client_auth();
+    let rustls_config = DangerousClientConfigBuilder {
+      cfg: ClientConfig::builder(),
+    }
+    .with_custom_certificate_verifier(Arc::new(NoCertVerifier {}))
+    .with_no_client_auth();
 
     let tls = tokio_postgres_rustls::MakeRustlsConnect::new(rustls_config);
     let (client, conn) = tokio_postgres::connect(config, tls)
@@ -338,21 +348,55 @@ fn establish_connection(config: &str) -> BoxFuture<ConnectionResult<AsyncPgConne
   fut.boxed()
 }
 
+#[derive(Debug)]
 struct NoCertVerifier {}
 
 impl ServerCertVerifier for NoCertVerifier {
   fn verify_server_cert(
     &self,
-    _end_entity: &rustls::Certificate,
-    _intermediates: &[rustls::Certificate],
+    _end_entity: &CertificateDer,
+    _intermediates: &[CertificateDer],
     _server_name: &ServerName,
-    _scts: &mut dyn Iterator<Item = &[u8]>,
-    _ocsp_response: &[u8],
-    _now: SystemTime,
+    _ocsp: &[u8],
+    _now: UnixTime,
   ) -> Result<ServerCertVerified, rustls::Error> {
     // Will verify all (even invalid) certs without any checks (sslmode=require)
     Ok(ServerCertVerified::assertion())
   }
+
+  fn verify_tls12_signature(
+    &self,
+    message: &[u8],
+    cert: &CertificateDer,
+    dss: &DigitallySignedStruct,
+  ) -> Result<HandshakeSignatureValid, rustls::Error> {
+    verify_tls12_signature(
+      message,
+      cert,
+      dss,
+      &crypto::ring::default_provider().signature_verification_algorithms,
+    )
+  }
+
+  fn verify_tls13_signature(
+    &self,
+    message: &[u8],
+    cert: &CertificateDer,
+    dss: &DigitallySignedStruct,
+  ) -> Result<HandshakeSignatureValid, rustls::Error> {
+    verify_tls13_signature(
+      message,
+      cert,
+      dss,
+      &crypto::ring::default_provider().signature_verification_algorithms,
+    )
+  }
+
+  fn supported_verify_schemes(&self) -> Vec<SignatureScheme> {
+    crypto::ring::default_provider()
+      .signature_verification_algorithms
+      .supported_schemes()
+  }
 }
 
 pub async fn build_db_pool() -> LemmyResult<ActualDbPool> {

plugins/main.go

@@ -16,8 +16,8 @@ type CreatePost struct {
 	Custom_thumbnail *string `json:"custom_thumbnail,omitempty"`
   }
 
-//export api_before_create_post
-func api_before_create_post() int32 {
+//export api_before_post_post
+func api_before_post_post() int32 {
 	params := CreatePost{}
 	// use json input helper, which automatically unmarshals the plugin input into your struct
 	err := pdk.InputJSON(&params)

src/api_routes_http.rs

@@ -1,3 +1,4 @@
+use crate::plugin_middleware::PluginMiddleware;
 use actix_web::{guard, web};
 use lemmy_api::{
   comment::{
@@ -139,6 +140,7 @@ use lemmy_utils::rate_limit::RateLimitCell;
 pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimitCell) {
   cfg.service(
     web::scope("/api/v3")
+      .wrap(PluginMiddleware::new())
       .route("/image_proxy", web::get().to(image_proxy))
       // Site
       .service(

src/lib.rs

@@ -1,5 +1,6 @@
 pub mod api_routes_http;
 pub mod code_migrations;
+pub mod plugin_middleware;
 pub mod prometheus_metrics;
 pub mod root_span_builder;
 pub mod scheduled_tasks;

src/plugin_middleware.rs

@@ -0,0 +1,124 @@
+use actix_http::h1::Payload;
+use actix_web::{
+  body::MessageBody,
+  dev::{forward_ready, Service, ServiceRequest, ServiceResponse, Transform},
+  web::Bytes,
+  Error,
+};
+use core::future::Ready;
+use extism::{Manifest, Plugin};
+use futures_util::future::LocalBoxFuture;
+use lemmy_utils::{error::LemmyResult, LemmyErrorType};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::{ffi::OsStr, fs::read_dir, future::ready, rc::Rc};
+use tracing::info;
+
+#[derive(Clone)]
+pub struct PluginMiddleware {}
+
+impl PluginMiddleware {
+  pub fn new() -> Self {
+    PluginMiddleware {}
+  }
+}
+impl<S, B> Transform<S, ServiceRequest> for PluginMiddleware
+where
+  S: Service<ServiceRequest, Response = ServiceResponse<B>, Error = Error> + 'static,
+  S::Future: 'static,
+  B: MessageBody + 'static,
+{
+  type Response = ServiceResponse<B>;
+  type Error = Error;
+  type Transform = SessionService<S>;
+  type InitError = ();
+  type Future = Ready<Result<Self::Transform, Self::InitError>>;
+
+  fn new_transform(&self, service: S) -> Self::Future {
+    ready(Ok(SessionService {
+      service: Rc::new(service),
+    }))
+  }
+}
+
+pub struct SessionService<S> {
+  service: Rc<S>,
+}
+
+impl<S, B> Service<ServiceRequest> for SessionService<S>
+where
+  S: Service<ServiceRequest, Response = ServiceResponse<B>, Error = Error> + 'static,
+  S::Future: 'static,
+  B: 'static,
+{
+  type Response = ServiceResponse<B>;
+  type Error = Error;
+  type Future = LocalBoxFuture<'static, Result<Self::Response, Self::Error>>;
+
+  forward_ready!(service);
+
+  fn call(&self, mut service_req: ServiceRequest) -> Self::Future {
+    let svc = self.service.clone();
+
+    Box::pin(async move {
+      let method = service_req.method();
+      let path = service_req.path().replace("/api/v3/", "").replace("/", "_");
+      // TODO: naming can be a bit silly, `POST /api/v3/post` becomes `api_before_post_post`
+      let plugin_hook = format!("api_before_{method}_{path}").to_lowercase();
+
+      info!("Calling plugin hook {}", &plugin_hook);
+      if let Some(mut plugins) = load_plugins()? {
+        if plugins.function_exists(&plugin_hook) {
+          let payload = service_req.extract::<Bytes>().await?;
+
+          let mut json: Value = serde_json::from_slice(&payload.to_vec())?;
+          call_plugin(plugins, &plugin_hook, &mut json)?;
+
+          let (_, mut new_payload) = Payload::create(true);
+          new_payload.unread_data(Bytes::from(serde_json::to_vec_pretty(&json)?));
+          service_req.set_payload(new_payload.into());
+        }
+      }
+      let res = svc.call(service_req).await?;
+
+      // TODO: add after hook
+
+      Ok(res)
+    })
+  }
+}
+
+fn load_plugins() -> LemmyResult<Option<Plugin>> {
+  // TODO: make dir configurable via env var
+  // TODO: should only read fs once at startup for performance
+  let plugin_paths = read_dir("plugins")?;
+
+  let mut wasm_files = vec![];
+  for path in plugin_paths {
+    let path = path?.path();
+    if path.extension() == Some(OsStr::new("wasm")) {
+      wasm_files.push(path);
+    }
+  }
+  if !wasm_files.is_empty() {
+    // TODO: what if theres more than one plugin for the same hook?
+    let manifest = Manifest::new(wasm_files);
+    let plugin = Plugin::new(manifest, [], true)?;
+    Ok(Some(plugin))
+  } else {
+    Ok(None)
+  }
+}
+
+pub fn call_plugin<T: Serialize + for<'de> Deserialize<'de> + Clone>(
+  mut plugins: Plugin,
+  name: &str,
+  data: &mut T,
+) -> LemmyResult<()> {
+  *data = plugins
+    .call::<extism_convert::Json<T>, extism_convert::Json<T>>(name, (*data).clone().into())
+    .map_err(|e| LemmyErrorType::PluginError(e.to_string()))?
+    .0
+    .into();
+  Ok(())
+}