feat(logging): switch to hat-syslog

Dockerfile

@@ -569,6 +569,7 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va
   python3-libsass \
   python3-pip-whl \
   python3-socks \
+  busybox-syslogd \
   curl \
   indent \
   less \
@@ -719,6 +720,10 @@ RUN --mount=type=tmpfs,target=/cache \
 # Copy root
 COPY config/root /
 
+# patch hat-syslog
+COPY patches/hat/ \
+    /usr/local/lib/python3/dist-packages/hat/
+
 # patch yt_dlp
 COPY patches/yt_dlp/ \
     /usr/local/lib/python3/dist-packages/yt_dlp/

Pipfile

@@ -28,3 +28,4 @@ html5lib = "*"
 bgutil-ytdlp-pot-provider = "~=1.3.1"
 yt-dlp-remote-cipher = "*"
 babi = "*"
+hat-syslog = "*"

config/root/etc/nginx/nginx.conf

@@ -61,8 +61,10 @@ http {
 
     # Logging
     log_format host '$remote_addr - $remote_user [$time_local] "[$host] $request" $status $bytes_sent "$http_referer" "$http_user_agent" "$gzip_ratio"';
-    access_log /dev/stdout;
-    error_log stderr;
+    ##access_log /dev/stdout;
+    access_log syslog:server=unix:/dev/log,facility=local1,nohostname;
+    error_log syslog:server=unix:/dev/log,facility=local1,nohostname info;
+    error_log stderr error;
 
     # GZIP
     gzip on;
@@ -99,7 +101,7 @@ http {
         default 80;
         "https" 443;
     }
- 
+
     # Extract the remote port from the HTTP Host header.
     # Uses default_http_port from above,
     #     when no port was found in the header.
@@ -167,11 +169,41 @@ http {
             internal;
             alias /downloads/;
         }
+
+        # Web server logs
+        location /ws {
+            proxy_pass http://127.0.0.1:23020/ws;
+
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+
+            proxy_set_header Host $host;
+            proxy_set_header X-Forwarded-Host $x_forwarded_host;
+            proxy_set_header X-Forwarded-Port $x_forwarded_port;
+            proxy_set_header X-Forwarded-Proto $x_forwarded_proto;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_redirect off;
+        }
+        location /web-logs/ {
+            proxy_pass http://127.0.0.1:23020/;
+
+            proxy_cache gunicorn;
+
+            proxy_set_header Host $host;
+            proxy_set_header X-Forwarded-Host $x_forwarded_host;
+            proxy_set_header X-Forwarded-Port $x_forwarded_port;
+            proxy_set_header X-Forwarded-Proto $x_forwarded_proto;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_redirect off;
+        }
     }
 
     # YouTube JavaScript Cipher Challenge Solving Server
     include /etc/nginx/cipher_server.conf;
-    
+
     # Proof-of-Origin Token Server
     include /etc/nginx/token_server.conf;
 

config/root/etc/s6-overlay/s6-rc.d/busybox-syslogd/run

@@ -0,0 +1,25 @@
+#!/command/with-contenv bash
+
+install_pkg() {
+    apt-get update && apt-get install -y busybox busybox-syslogd
+}
+
+set -e
+cd /
+
+install -d -o root -g app -m 0750 /run/app/log
+
+command -v busybox >/dev/null || install_pkg
+
+HOST=127.0.0.1
+PORT=6514
+export HOST PORT
+
+config='/run/app/log/.syslog.conf'
+logfile='/run/app/log/messages'
+
+: >| "${config}"
+
+exec busybox syslogd -n -S -l 8 -f "${config}" \
+    -b 9 -s 2048 -O "${logfile}" \
+    -L -R "${HOST}:${PORT}"

config/root/etc/s6-overlay/s6-rc.d/busybox-syslogd/type

@@ -0,0 +1 @@
+longrun

config/root/etc/s6-overlay/s6-rc.d/hat-syslog-server/run

@@ -0,0 +1,15 @@
+#!/command/with-contenv bash
+
+set -e
+cd /app
+
+DIR="${XDG_STATE_HOME:-/config/state}/hat"
+DATABASE="${DIR}/syslog.db"
+
+mkdir -p "${DIR}"
+chown -R app:app "${DIR}"
+chmod 0700 "${DIR}"
+
+exec s6-setuidgid app \
+    hat-syslog-server --log-level 'DEBUG' \
+        --db-enable-archive --db-path "${DATABASE}"

config/root/etc/s6-overlay/s6-rc.d/hat-syslog-server/type

@@ -0,0 +1 @@
+longrun

config/root/etc/s6-overlay/s6-rc.d/tubesync-app-init/run

@@ -0,0 +1,26 @@
+#!/command/with-contenv bash
+
+# Copy local_settings.py for the user
+if [ -f /config/tubesync/local_settings.py ]
+then
+    # from the image for an example or comparison
+    cp -v -p /app/tubesync/local_settings.py \
+        /config/tubesync/local_settings.py.image
+
+    # to the image to apply the user's settings
+    cp -v -p /config/tubesync/local_settings.py \
+        /app/tubesync/local_settings.py
+
+    chown app:app \
+        /config/tubesync/local_settings.py.image
+    chmod 0755 \
+        /config/tubesync/local_settings.py.image
+fi
+
+# Reset permissions
+chown -R root:app /app
+chmod -R 0750 /app
+chmod 0755 /app/*.py /app/*.sh
+find /app -mindepth 2 -type f -execdir chmod 640 '{}' +
+chown -R app:app /app/common/static
+chown -R app:app /app/static

config/root/etc/s6-overlay/s6-rc.d/tubesync-app-init/type

@@ -0,0 +1 @@
+oneshot

config/root/etc/s6-overlay/s6-rc.d/tubesync-app-init/up

@@ -0,0 +1,3 @@
+#!/command/execlineb -P
+
+/etc/s6-overlay/s6-rc.d/tubesync-app-init/run

config/root/etc/s6-overlay/s6-rc.d/tubesync-config-init/run

@@ -0,0 +1,17 @@
+#!/command/with-contenv bash
+
+# Change runtime user UID and GID
+groupmod -o -g "${PGID:=911}" app
+usermod -o -u "${PUID:=911}" app
+
+# Ensure existence and reset permissions
+mkdir -v -p /run/app
+chown -R app:app /run/app
+chmod -R 0700 /run/app
+
+# Ensure /config directories exist
+mkdir -v -p /config/{cache,media,state,tasks,tubesync}
+
+# Reset permissions
+chown -R app:app /config
+chmod -R 0755 /config

config/root/etc/s6-overlay/s6-rc.d/tubesync-config-init/type

@@ -0,0 +1 @@
+oneshot

config/root/etc/s6-overlay/s6-rc.d/tubesync-config-init/up

@@ -0,0 +1,3 @@
+#!/command/execlineb -P
+
+/etc/s6-overlay/s6-rc.d/tubesync-config-init/run

config/root/etc/s6-overlay/s6-rc.d/tubesync-downloads-init/run

@@ -0,0 +1,12 @@
+#!/command/with-contenv bash
+
+# Optionally reset the download dir permissions
+if [[ 'True' != "${TUBESYNC_RESET_DOWNLOAD_DIR:-True}" ]]
+then
+    exit 0
+fi
+
+echo "TUBESYNC_RESET_DOWNLOAD_DIR=True, Resetting /downloads directory permissions"
+
+chown -R app:app /downloads
+chmod -R 0755 /downloads

config/root/etc/s6-overlay/s6-rc.d/tubesync-downloads-init/type

@@ -0,0 +1 @@
+oneshot

config/root/etc/s6-overlay/s6-rc.d/tubesync-downloads-init/up

@@ -0,0 +1,3 @@
+#!/command/execlineb -P
+
+/etc/s6-overlay/s6-rc.d/tubesync-downloads-init/run

config/root/etc/s6-overlay/s6-rc.d/tubesync-init/type

@@ -1 +1 @@
-oneshot
+bundle

config/root/etc/s6-overlay/s6-rc.d/tubesync-migrations-init/run

@@ -0,0 +1,12 @@
+#!/command/with-contenv bash
+
+if [[ 'True' == "${TUBESYNC_DEBUG:-False}" ]]
+then
+    s6-setuidgid app \
+        /usr/bin/python3 /app/manage.py \
+            showmigrations -v 3 --list
+fi
+
+# Run migrations
+exec s6-setuidgid app \
+    /usr/bin/python3 /app/manage.py migrate

config/root/etc/s6-overlay/s6-rc.d/tubesync-migrations-init/type

@@ -0,0 +1 @@
+oneshot

config/root/etc/s6-overlay/s6-rc.d/tubesync-migrations-init/up

@@ -0,0 +1,3 @@
+#!/command/execlineb -P
+
+/etc/s6-overlay/s6-rc.d/tubesync-migrations-init/run