qubes-mirage-firewall/dispatcher.ml

open Lwt.Infix
open Fw_utils
module Netback = Backend.Make (Xenstore.Make (Xen_os.Xs))
module ClientEth = Ethernet.Make (Netback)
module UplinkEth = Ethernet.Make (Netif)

let src = Logs.Src.create "dispatcher" ~doc:"Networking dispatch"

module Log = (val Logs.src_log src : Logs.LOG)

module Make
    (R : Mirage_random.S)
    (Clock : Mirage_clock.MCLOCK)
    (Time : Mirage_time.S) =
struct
  module Arp = Arp.Make (UplinkEth) (Time)
  module I = Static_ipv4.Make (R) (Clock) (UplinkEth) (Arp)
  module U = Udp.Make (I) (R)

  let clients : Cleanup.t Dao.VifMap.t ref = ref Dao.VifMap.empty

  class client_iface eth ~domid ~gateway_ip ~client_ip client_mac : client_link
    =
    let log_header = Fmt.str "dom%d:%a" domid Ipaddr.V4.pp client_ip in
    object
      val mutable rules = []
      method get_rules = rules
      method set_rules new_db = rules <- Dao.read_rules new_db client_ip
      method my_mac = ClientEth.mac eth
      method other_mac = client_mac
      method my_ip = gateway_ip
      method other_ip = client_ip

      method writev proto fillfn =
        Lwt.catch
          (fun () ->
            ClientEth.write eth client_mac proto fillfn >|= function
            | Ok () -> ()
            | Error e ->
                Log.err (fun f ->
                    f "error trying to send to client: @[%a@]"
                      ClientEth.pp_error e))
          (fun ex ->
            (* Usually Netback_shutdown, because the client disconnected *)
            Log.err (fun f ->
                f "uncaught exception trying to send to client: @[%s@]"
                  (Printexc.to_string ex));
            Lwt.return_unit)

      method log_header = log_header
    end

  class netvm_iface eth mac ~my_ip ~other_ip : interface =
    object
      method my_mac = UplinkEth.mac eth
      method my_ip = my_ip
      method other_ip = other_ip

      method writev ethertype fillfn =
        Lwt.catch
          (fun () ->
            mac >>= fun dst ->
            UplinkEth.write eth dst ethertype fillfn
            >|= or_raise "Write to uplink" UplinkEth.pp_error)
          (fun ex ->
            Log.err (fun f ->
                f "uncaught exception trying to send to uplink: @[%s@]"
                  (Printexc.to_string ex));
            Lwt.return_unit)
    end

  type uplink = {
    net : Netif.t;
    eth : UplinkEth.t;
    arp : Arp.t;
    interface : interface;
    mutable fragments : Fragments.Cache.t;
    ip : I.t;
    udp : U.t;
  }

  type t = {
    uplink_connected : unit Lwt_condition.t;
    uplink_disconnect : unit Lwt_condition.t;
    uplink_disconnected : unit Lwt_condition.t;
    mutable config : Dao.network_config;
    clients : Client_eth.t;
    nat : My_nat.t;
    mutable uplink : uplink option;
  }

  let create ~config ~clients ~nat ~uplink =
    {
      uplink_connected = Lwt_condition.create ();
      uplink_disconnect = Lwt_condition.create ();
      uplink_disconnected = Lwt_condition.create ();
      config;
      clients;
      nat;
      uplink;
    }

  let update t ~config ~uplink =
    t.config <- config;
    t.uplink <- uplink;
    Lwt.return_unit

  let target t buf =
    let dst_ip = buf.Ipv4_packet.dst in
    match Client_eth.lookup t.clients dst_ip with
    | Some client_link -> Some (client_link :> interface)
    | None -> ( (* if dest is not a client, transfer it to our uplink *)
        match t.uplink with
        | None -> (
            match Client_eth.lookup t.clients t.config.netvm_ip with
            | Some uplink ->
              Some (uplink :> interface)
            | None ->
              Log.err (fun f -> f "We have a command line configuration %a but it's currently not connected to us (please check its netvm property)...%!" Ipaddr.V4.pp t.config.netvm_ip);
              None)
        | Some uplink -> Some uplink.interface)

  let add_client t = Client_eth.add_client t.clients
  let remove_client t = Client_eth.remove_client t.clients

  let classify t ip =
    if ip = Ipaddr.V4 t.config.our_ip then `Firewall
    else if ip = Ipaddr.V4 t.config.netvm_ip then `NetVM
    else (Client_eth.classify t.clients ip :> Packet.host)

  let resolve t = function
    | `Firewall -> Ipaddr.V4 t.config.our_ip
    | `NetVM -> Ipaddr.V4 t.config.netvm_ip
    | #Client_eth.host as host -> Client_eth.resolve t.clients host

  (* Transmission *)

  let transmit_ipv4 packet iface =
    Lwt.catch
      (fun () ->
        let fragments = ref [] in
        iface#writev `IPv4 (fun b ->
            match Nat_packet.into_cstruct packet b with
            | Error e ->
                Log.warn (fun f ->
                    f "Failed to write packet to %a: %a" Ipaddr.V4.pp
                      iface#other_ip Nat_packet.pp_error e);
                0
            | Ok (n, frags) ->
                fragments := frags;
                n)
        >>= fun () ->
        Lwt_list.iter_s
          (fun f ->
            let size = Cstruct.length f in
            iface#writev `IPv4 (fun b ->
                Cstruct.blit f 0 b 0 size;
                size))
          !fragments)
      (fun ex ->
        Log.warn (fun f ->
            f "Failed to write packet to %a: %s" Ipaddr.V4.pp iface#other_ip
              (Printexc.to_string ex));
        Lwt.return_unit)

  let forward_ipv4 t packet =
    let (`IPv4 (ip, _)) = packet in
    Lwt.catch
    (fun () ->
      match target t ip with
      | Some iface -> transmit_ipv4 packet iface
      | None -> Lwt.return_unit)
    (fun ex ->
      let dst_ip = ip.Ipv4_packet.dst in
      Log.warn (fun f ->
          f "Failed to lookup for target %a: %s" Ipaddr.V4.pp dst_ip
            (Printexc.to_string ex));
      Lwt.return_unit)

  (* NAT *)

  let translate t packet = My_nat.translate t.nat packet

  (* Add a NAT rule for the endpoints in this frame, via a random port on the firewall. *)
  let add_nat_and_forward_ipv4 t packet =
    let xl_host = t.config.our_ip in
    match My_nat.add_nat_rule_and_translate t.nat ~xl_host `NAT packet with
    | Ok packet -> forward_ipv4 t packet
    | Error e ->
        Log.warn (fun f ->
            f "Failed to add NAT rewrite rule: %s (%a)" e Nat_packet.pp packet);
        Lwt.return_unit

  (* Add a NAT rule to redirect this conversation to [host:port] instead of us. *)
  let nat_to t ~host ~port packet =
    match resolve t host with
    | Ipaddr.V6 _ ->
        Log.warn (fun f -> f "Cannot NAT with IPv6");
        Lwt.return_unit
    | Ipaddr.V4 target -> (
        let xl_host = t.config.our_ip in
        match
          My_nat.add_nat_rule_and_translate t.nat ~xl_host
            (`Redirect (target, port))
            packet
        with
        | Ok packet -> forward_ipv4 t packet
        | Error e ->
            Log.warn (fun f ->
                f "Failed to add NAT redirect rule: %s (%a)" e Nat_packet.pp
                  packet);
            Lwt.return_unit)

  let apply_rules t (rules : ('a, 'b) Packet.t -> Packet.action Lwt.t) ~dst
      (annotated_packet : ('a, 'b) Packet.t) : unit Lwt.t =
    let packet = Packet.to_mirage_nat_packet annotated_packet in
    rules annotated_packet >>= fun action ->
    match (action, dst) with
    | `Accept, `Client client_link -> transmit_ipv4 packet client_link
    | `Accept, (`External _ | `NetVM) -> (
        match t.uplink with
        | Some uplink -> transmit_ipv4 packet uplink.interface
        | None -> (
            match Client_eth.lookup t.clients t.config.netvm_ip with
            | Some iface -> transmit_ipv4 packet iface
            | None ->
                Log.warn (fun f ->
                    f "No output interface for %a : drop" Nat_packet.pp packet);
                Lwt.return_unit))
    | `Accept, `Firewall ->
        Log.warn (fun f ->
            f "Bad rule: firewall can't accept packets %a" Nat_packet.pp packet);
        Lwt.return_unit
    | `NAT, _ ->
        Log.debug (fun f -> f "adding NAT rule for %a" Nat_packet.pp packet);
        add_nat_and_forward_ipv4 t packet
    | `NAT_to (host, port), _ -> nat_to t packet ~host ~port
    | `Drop reason, _ ->
        Log.debug (fun f ->
            f "Dropped packet (%s) %a" reason Nat_packet.pp packet);
        Lwt.return_unit

  let ipv4_from_netvm t packet =
    match Memory_pressure.status () with
    | `Memory_critical -> Lwt.return_unit
    | `Ok -> (
        let (`IPv4 (ip, _transport)) = packet in
        let src = classify t (Ipaddr.V4 ip.Ipv4_packet.src) in
        let dst = classify t (Ipaddr.V4 ip.Ipv4_packet.dst) in
        match Packet.of_mirage_nat_packet ~src ~dst packet with
        | None -> Lwt.return_unit
        | Some _ -> (
            match src with
            | `Client _ | `Firewall ->
                Log.warn (fun f ->
                    f "Frame from NetVM has internal source IP address! %a"
                      Nat_packet.pp packet);
                Lwt.return_unit
            | (`External _ | `NetVM) as src -> (
                match translate t packet with
                | Some frame -> forward_ipv4 t frame
                | None -> (
                    match Packet.of_mirage_nat_packet ~src ~dst packet with
                    | None -> Lwt.return_unit
                    | Some packet -> apply_rules t Rules.from_netvm ~dst packet)
                )))

  let ipv4_from_client resolver dns_servers t ~src packet =
    match Memory_pressure.status () with
    | `Memory_critical -> Lwt.return_unit
    | `Ok -> (
        (* Check for existing NAT entry for this packet *)
        match translate t packet with
        | Some frame ->
            forward_ipv4 t frame (* Some existing connection or redirect *)
        | None -> (
            (* No existing NAT entry. Check the firewall rules. *)
            let (`IPv4 (ip, _transport)) = packet in
            match classify t (Ipaddr.V4 ip.Ipv4_packet.src) with
            | `Client _ | `Firewall -> (
                let dst = classify t (Ipaddr.V4 ip.Ipv4_packet.dst) in
                match
                  Packet.of_mirage_nat_packet ~src:(`Client src) ~dst packet
                with
                | None -> Lwt.return_unit
                | Some firewall_packet ->
                    apply_rules t
                      (Rules.from_client resolver dns_servers)
                      ~dst firewall_packet)
            | `NetVM -> ipv4_from_netvm t packet
            | `External _ ->
                Log.warn (fun f ->
                    f "Frame from Inside has external source IP address! %a"
                      Nat_packet.pp packet);
                Lwt.return_unit))

  (** Handle an ARP message from the client. *)
  let client_handle_arp ~fixed_arp ~iface request =
    match Arp_packet.decode request with
    | Error e ->
        Log.warn (fun f ->
            f "Ignored unknown ARP message: %a" Arp_packet.pp_error e);
        Lwt.return_unit
    | Ok arp -> (
        match Client_eth.ARP.input fixed_arp arp with
        | None -> Lwt.return_unit
        | Some response ->
          Lwt.catch
            (fun () ->
                  iface#writev `ARP (fun b ->
                      Arp_packet.encode_into response b;
                      Arp_packet.size))
            (fun ex ->
              Log.warn (fun f ->
                  f "Failed to write APR to %a: %s" Ipaddr.V4.pp iface#other_ip
                    (Printexc.to_string ex));
              Lwt.return_unit)
     )

  (** Handle an IPv4 packet from the client. *)
  let client_handle_ipv4 get_ts cache ~iface ~router dns_client dns_servers
      packet =
    let cache', r = Nat_packet.of_ipv4_packet !cache ~now:(get_ts ()) packet in
    cache := cache';
    match r with
    | Error e ->
        Log.warn (fun f ->
            f "Ignored unknown IPv4 message: %a" Nat_packet.pp_error e);
        Lwt.return_unit
    | Ok None -> Lwt.return_unit
    | Ok (Some packet) ->
        let (`IPv4 (ip, _)) = packet in
        let src = ip.Ipv4_packet.src in
        if src = iface#other_ip then
          ipv4_from_client dns_client dns_servers router ~src:iface packet
        else if iface#other_ip = router.config.netvm_ip then
          (* This can occurs when used with *BSD as netvm (and a gateway is set) *)
          ipv4_from_netvm router packet
        else (
          Log.warn (fun f ->
              f "Incorrect source IP %a in IP packet from %a (dropping)"
                Ipaddr.V4.pp src Ipaddr.V4.pp iface#other_ip);
          Lwt.return_unit)

  (** Connect to a new client's interface and listen for incoming frames and firewall rule changes. *)
  let add_vif get_ts { Dao.ClientVif.domid; device_id } dns_client dns_servers
      ~client_ip ~router ~cleanup_tasks qubesDB =
    Netback.make ~domid ~device_id >>= fun backend ->
    Log.info (fun f ->
        f "Client %d (IP: %s) ready" domid (Ipaddr.V4.to_string client_ip));
    ClientEth.connect backend >>= fun eth ->
    let client_mac = Netback.frontend_mac backend in
    let client_eth = router.clients in
    let gateway_ip = Client_eth.client_gw client_eth in
    let iface = new client_iface eth ~domid ~gateway_ip ~client_ip client_mac in
    (* update the rules whenever QubesDB notices a change for this IP *)
    let qubesdb_updater =
      Lwt.catch
        (fun () ->
          let rec update current_db current_rules =
            Qubes.DB.got_new_commit qubesDB (Dao.db_root client_ip) current_db
            >>= fun new_db ->
            iface#set_rules new_db;
            let new_rules = iface#get_rules in
            if current_rules = new_rules then
              Log.info (fun m ->
                  m "Rules did not change for %s"
                    (Ipaddr.V4.to_string client_ip))
            else (
              Log.info (fun m ->
                  m "New firewall rules for %s@.%a"
                    (Ipaddr.V4.to_string client_ip)
                    Fmt.(list ~sep:(any "@.") Pf_qubes.Parse_qubes.pp_rule)
                    new_rules);
              (* empty NAT table if rules are updated: they might deny old connections *)
              My_nat.remove_connections router.nat client_ip);
            update new_db new_rules
          in
          update Qubes.DB.KeyMap.empty [])
        (function Lwt.Canceled -> Lwt.return_unit | e -> Lwt.fail e)
    in
    Cleanup.on_cleanup cleanup_tasks (fun () -> Lwt.cancel qubesdb_updater);
    add_client router iface >>= fun () ->
    Cleanup.on_cleanup cleanup_tasks (fun () -> remove_client router iface);
    let fixed_arp = Client_eth.ARP.create ~net:client_eth iface in
    let fragment_cache = ref (Fragments.Cache.empty (256 * 1024)) in
    let listener =
      Lwt.catch
        (fun () ->
          Netback.listen backend ~header_size:Ethernet.Packet.sizeof_ethernet
            (fun frame ->
              match Ethernet.Packet.of_cstruct frame with
              | Error err ->
                  Log.warn (fun f -> f "Invalid Ethernet frame: %s" err);
                  Lwt.return_unit
              | Ok (eth, payload) -> (
                  match eth.Ethernet.Packet.ethertype with
                  | `ARP -> client_handle_arp ~fixed_arp ~iface payload
                  | `IPv4 ->
                      client_handle_ipv4 get_ts fragment_cache ~iface ~router
                        dns_client dns_servers payload
                  | `IPv6 -> Lwt.return_unit (* TODO: oh no! *)))
          >|= or_raise "Listen on client interface" Netback.pp_error)
        (function Lwt.Canceled -> Lwt.return_unit | e -> Lwt.fail e)
    in
    Cleanup.on_cleanup cleanup_tasks (fun () -> Lwt.cancel listener);
    Lwt.pick [ qubesdb_updater; listener ]

  (** A new client VM has been found in XenStore. Find its interface and connect to it. *)
  let add_client get_ts dns_client dns_servers ~router vif client_ip qubesDB =
    let cleanup_tasks = Cleanup.create () in
    Log.info (fun f ->
        f "add client vif %a with IP %a" Dao.ClientVif.pp vif Ipaddr.V4.pp
          client_ip);
    Lwt.async (fun () ->
        Lwt.catch
          (fun () ->
            add_vif get_ts vif dns_client dns_servers ~client_ip ~router
              ~cleanup_tasks qubesDB)
          (fun ex ->
            Log.warn (fun f ->
                f "Error with client %a: %s" Dao.ClientVif.pp vif
                  (Printexc.to_string ex));
            Lwt.return_unit));
    cleanup_tasks

  (** Watch XenStore for notifications of new clients. *)
  let wait_clients get_ts dns_client dns_servers qubesDB router =
    Dao.watch_clients (fun new_set ->
        (* Check for removed clients *)
        !clients
        |> Dao.VifMap.iter (fun key cleanup ->
               if not (Dao.VifMap.mem key new_set) then (
                 clients := !clients |> Dao.VifMap.remove key;
                 Log.info (fun f -> f "client %a has gone" Dao.ClientVif.pp key);
                 Cleanup.cleanup cleanup));
        (* Check for added clients *)
        new_set
        |> Dao.VifMap.iter (fun key ip_addr ->
               if not (Dao.VifMap.mem key !clients) then (
                 let cleanup =
                   add_client get_ts dns_client dns_servers ~router key ip_addr
                     qubesDB
                 in
                 Log.debug (fun f -> f "client %a arrived" Dao.ClientVif.pp key);
                 clients := !clients |> Dao.VifMap.add key cleanup)))

  let send_dns_client_query t ~src_port ~dst ~dst_port buf =
    match t.uplink with
    | None ->
        Log.err (fun f -> f "No uplink interface");
        Lwt.return (Error (`Msg "failure"))
    | Some uplink -> (
        Lwt.catch
          (fun () ->
            U.write ~src_port ~dst ~dst_port uplink.udp buf >|= function
            | Error s ->
                Log.err (fun f -> f "error sending udp packet: %a" U.pp_error s);
                Error (`Msg "failure")
            | Ok () -> Ok ())
          (fun ex ->
            Log.err (fun f ->
                f "uncaught exception trying to send DNS request to uplink: @[%s@]"
                  (Printexc.to_string ex));
            Lwt.return (Error (`Msg "DNS request not sent"))))

  (** Wait for packet from our uplink (we must have an uplink here...). *)
  let rec uplink_listen get_ts dns_responses router =
    Lwt_condition.wait router.uplink_connected >>= fun () ->
    match router.uplink with
    | None ->
          Log.err (fun f ->
              f
                "Uplink is connected but not found in the router, retrying...%!");
        uplink_listen get_ts dns_responses router
    | Some uplink ->
        let listen =
        Lwt.catch
        (fun () ->
          Netif.listen uplink.net ~header_size:Ethernet.Packet.sizeof_ethernet
            (fun frame ->
              (* Handle one Ethernet frame from NetVM *)
              UplinkEth.input uplink.eth ~arpv4:(Arp.input uplink.arp)
                ~ipv4:(fun ip ->
                  let cache, r =
                    Nat_packet.of_ipv4_packet uplink.fragments ~now:(get_ts ())
                      ip
                  in
                  uplink.fragments <- cache;
                  begin match r with
                  | Error e ->
                      Log.warn (fun f ->
                          f "Ignored unknown IPv4 message from uplink: %a"
                            Nat_packet.pp_error e);
                      Lwt.return ()
                  | Ok None -> Lwt.return_unit
                  | Ok (Some (`IPv4 (header, packet))) ->
                      let open Udp_packet in
                      Log.debug (fun f ->
                          f "received ipv4 packet from %a on uplink" Ipaddr.V4.pp
                            header.Ipv4_packet.src);
                      begin match packet with
                      | `UDP (header, packet) when My_nat.dns_port router.nat header.dst_port ->
                          Log.debug (fun f ->
                              f
                                "found a DNS packet whose dst_port (%d) was in the list of \
                                 dns_client ports"
                                header.dst_port);
                          Lwt_mvar.put dns_responses (header, packet)
                      | _ -> ipv4_from_netvm router (`IPv4 (header, packet))
                      end
                   end)
                ~ipv6:(fun _ip -> Lwt.return_unit)
                frame)
          >|= or_raise "Uplink listen loop" Netif.pp_error)
          (function Lwt.Canceled ->
            (* We can be cancelled if reconnect_uplink is achieved (via the Lwt_condition), so we need to disconnect and broadcast when it's done
               currently we delay 1s as Netif.disconnect is non-blocking... (need to fix upstream?) *)
            Log.info (fun f ->
               f "disconnecting from our uplink");
            U.disconnect uplink.udp >>= fun () ->
            I.disconnect uplink.ip  >>= fun () ->
            (* mutable fragments : Fragments.Cache.t; *)
            (* interface : interface; *)
            Arp.disconnect uplink.arp >>= fun () ->
            UplinkEth.disconnect uplink.eth >>= fun () ->
            Netif.disconnect uplink.net >>= fun () ->
            Lwt_condition.broadcast router.uplink_disconnected ();
            Lwt.return_unit
          | e -> Lwt.fail e)
        in
        let reconnect_uplink =
          Lwt_condition.wait router.uplink_disconnect >>= fun () ->
          Log.info (fun f ->
             f "we need to reconnect to the new uplink");
          Lwt.return_unit
        in
        Lwt.pick [ listen ; reconnect_uplink ] >>= fun () ->
        uplink_listen get_ts dns_responses router

  (** Connect to our uplink backend (we must have an uplink here...). *)
  let connect config =
    let my_ip = config.Dao.our_ip in
    let gateway = config.Dao.netvm_ip in
    Netif.connect "0" >>= fun net ->
    UplinkEth.connect net >>= fun eth ->
    Arp.connect eth >>= fun arp ->
    Arp.add_ip arp my_ip >>= fun () ->
    let cidr = Ipaddr.V4.Prefix.make 0 my_ip in
    I.connect ~cidr ~gateway eth arp >>= fun ip ->
    U.connect ip >>= fun udp ->
    let netvm_mac =
      Arp.query arp gateway >|= or_raise "Getting MAC of our NetVM" Arp.pp_error
    in
    let interface =
      new netvm_iface eth netvm_mac ~my_ip ~other_ip:config.Dao.netvm_ip
    in
    let fragments = Fragments.Cache.empty (256 * 1024) in
    Lwt.return { net; eth; arp; interface; fragments; ip; udp }

  (** Wait Xenstore for our uplink changes (we must have an uplink here...). *)
  let uplink_wait_update qubesDB router =
    let rec aux current_db =
      let netvm = "/qubes-gateway" in
      Log.info (fun f -> f "Waiting for netvm changes to %S...%!" netvm);
      Qubes.DB.after qubesDB current_db >>= fun new_db ->
      (match (router.uplink, Qubes.DB.KeyMap.find_opt netvm new_db) with
      | Some uplink, Some netvm
        when not
               (String.equal netvm
                  (Ipaddr.V4.to_string uplink.interface#other_ip)) ->
          Log.info (fun f ->
              f "Our netvm IP has changed, before it was %s, now it's: %s%!"
                (Ipaddr.V4.to_string uplink.interface#other_ip)
                netvm);
          Lwt_condition.broadcast router.uplink_disconnect ();
          (* wait for uplink disconnexion *)
          Lwt_condition.wait router.uplink_disconnected >>= fun () ->
          Dao.read_network_config qubesDB >>= fun config ->
          Dao.print_network_config config;
          connect config >>= fun uplink ->
          update router ~config ~uplink:(Some uplink) >>= fun () ->
          Lwt_condition.broadcast router.uplink_connected ();
          Lwt.return_unit
      | None, Some _ ->
          (* a new interface is attributed to qubes-mirage-firewall *)
          Log.info (fun f -> f "Going from netvm not connected to %s%!" netvm);
          Dao.read_network_config qubesDB >>= fun config ->
          Dao.print_network_config config;
          connect config >>= fun uplink ->
          update router ~config ~uplink:(Some uplink) >>= fun () ->
          Lwt_condition.broadcast router.uplink_connected ();
          Lwt.return_unit
      | Some _, None ->
          (* This currently is never triggered :( *)
          Log.info (fun f ->
              f "TODO: Our netvm disapeared, troubles are coming!%!");
          Lwt.return_unit
      | Some _, Some _ (* The new netvm IP is unchanged (it's our old netvm IP) *)
      | None, None  ->
          Log.info (fun f ->
              f "QubesDB has changed but not the situation of our netvm!%!");
          Lwt.return_unit)
      >>= fun () -> aux new_db
    in
    aux Qubes.DB.KeyMap.empty
end