re-implement daemonize (#8011)

This has long been something I've wanted to do. Basically the `Daemonize` code
is both too flexible and not flexible enough, in that it offers a bunch of
features that we don't use (changing UID, closing FDs in the child, logging to
syslog) and doesn't offer a bunch that we could do with (redirecting stdout/err
to a file instead of /dev/null; having the parent not exit until the child is
running).

As a first step, I've lifted the Daemonize code and removed the bits we don't
use. This should be a non-functional change. Fixing everything else will come
later.
This commit is contained in:
Richard van der Hoff 2020-08-04 10:03:41 +01:00 committed by GitHub
parent 481f76c7aa
commit 916cf2d439
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 135 additions and 14 deletions

1
changelog.d/8011.misc Normal file
View File

@ -0,0 +1 @@
Replace daemonize library with a local implementation.

View File

@ -12,7 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import gc import gc
import logging import logging
import os import os
@ -22,7 +21,6 @@ import sys
import traceback import traceback
from typing import Iterable from typing import Iterable
from daemonize import Daemonize
from typing_extensions import NoReturn from typing_extensions import NoReturn
from twisted.internet import defer, error, reactor from twisted.internet import defer, error, reactor
@ -34,6 +32,7 @@ from synapse.config.server import ListenerConfig
from synapse.crypto import context_factory from synapse.crypto import context_factory
from synapse.logging.context import PreserveLoggingContext from synapse.logging.context import PreserveLoggingContext
from synapse.util.async_helpers import Linearizer from synapse.util.async_helpers import Linearizer
from synapse.util.daemonize import daemonize_process
from synapse.util.rlimit import change_resource_limit from synapse.util.rlimit import change_resource_limit
from synapse.util.versionstring import get_version_string from synapse.util.versionstring import get_version_string
@ -129,17 +128,8 @@ def start_reactor(
if print_pidfile: if print_pidfile:
print(pid_file) print(pid_file)
daemon = Daemonize( daemonize_process(pid_file, logger)
app=appname, run()
pid=pid_file,
action=run,
auto_close_fds=False,
verbose=True,
logger=logger,
)
daemon.start()
else:
run()
def quit_with_error(error_string: str) -> NoReturn: def quit_with_error(error_string: str) -> NoReturn:

View File

@ -59,7 +59,6 @@ REQUIREMENTS = [
"pyyaml>=3.11", "pyyaml>=3.11",
"pyasn1>=0.1.9", "pyasn1>=0.1.9",
"pyasn1-modules>=0.0.7", "pyasn1-modules>=0.0.7",
"daemonize>=2.3.1",
"bcrypt>=3.1.0", "bcrypt>=3.1.0",
"pillow>=4.3.0", "pillow>=4.3.0",
"sortedcontainers>=1.4.4", "sortedcontainers>=1.4.4",

131
synapse/util/daemonize.py Normal file
View File

@ -0,0 +1,131 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2012, 2013, 2014 Ilya Otyutskiy <ilya.otyutskiy@icloud.com>
# Copyright 2020 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import atexit
import fcntl
import logging
import os
import signal
import sys
def daemonize_process(pid_file: str, logger: logging.Logger, chdir: str = "/") -> None:
"""daemonize the current process
This calls fork(), and has the main process exit. When it returns we will be
running in the child process.
"""
# If pidfile already exists, we should read pid from there; to overwrite it, if
# locking will fail, because locking attempt somehow purges the file contents.
if os.path.isfile(pid_file):
with open(pid_file, "r") as pid_fh:
old_pid = pid_fh.read()
# Create a lockfile so that only one instance of this daemon is running at any time.
try:
lock_fh = open(pid_file, "w")
except IOError:
print("Unable to create the pidfile.")
sys.exit(1)
try:
# Try to get an exclusive lock on the file. This will fail if another process
# has the file locked.
fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
print("Unable to lock on the pidfile.")
# We need to overwrite the pidfile if we got here.
#
# XXX better to avoid overwriting it, surely. this looks racey as the pid file
# could be created between us trying to read it and us trying to lock it.
with open(pid_file, "w") as pid_fh:
pid_fh.write(old_pid)
sys.exit(1)
# Fork, creating a new process for the child.
process_id = os.fork()
if process_id != 0:
# parent process
sys.exit(0)
# This is the child process. Continue.
# Stop listening for signals that the parent process receives.
# This is done by getting a new process id.
# setpgrp() is an alternative to setsid().
# setsid puts the process in a new parent group and detaches its controlling
# terminal.
os.setsid()
# point stdin, stdout, stderr at /dev/null
devnull = "/dev/null"
if hasattr(os, "devnull"):
# Python has set os.devnull on this system, use it instead as it might be
# different than /dev/null.
devnull = os.devnull
devnull_fd = os.open(devnull, os.O_RDWR)
os.dup2(devnull_fd, 0)
os.dup2(devnull_fd, 1)
os.dup2(devnull_fd, 2)
os.close(devnull_fd)
# now that we have redirected stderr to /dev/null, any uncaught exceptions will
# get sent to /dev/null, so make sure we log them.
#
# (we don't normally expect reactor.run to raise any exceptions, but this will
# also catch any other uncaught exceptions before we get that far.)
def excepthook(type_, value, traceback):
logger.critical("Unhanded exception", exc_info=(type_, value, traceback))
sys.excepthook = excepthook
# Set umask to default to safe file permissions when running as a root daemon. 027
# is an octal number which we are typing as 0o27 for Python3 compatibility.
os.umask(0o27)
# Change to a known directory. If this isn't done, starting a daemon in a
# subdirectory that needs to be deleted results in "directory busy" errors.
os.chdir(chdir)
try:
lock_fh.write("%s" % (os.getpid()))
lock_fh.flush()
except IOError:
logger.error("Unable to write pid to the pidfile.")
print("Unable to write pid to the pidfile.")
sys.exit(1)
# write a log line on SIGTERM.
def sigterm(signum, frame):
logger.warning("Caught signal %s. Stopping daemon." % signum)
sys.exit(0)
signal.signal(signal.SIGTERM, sigterm)
# Cleanup pid file at exit.
def exit():
logger.warning("Stopping daemon.")
os.remove(pid_file)
sys.exit(0)
atexit.register(exit)
logger.warning("Starting daemon.")