From b9e6d67d15bbcc2ffada13f26f2c35e897cea6a5 Mon Sep 17 00:00:00 2001
From: asonix <asonix@asonix.dog>
Date: Sat, 22 Jul 2023 18:37:25 -0500
Subject: [PATCH] Add retention configuration

This doesn't do anything yet, but is part of the Media Proxy and Variant Cleanup tickets
---
 defaults.toml             |   6 +++
 src/config/commandline.rs |  45 ++++++++++++++++
 src/config/defaults.rs    |  18 +++++++
 src/config/file.rs        |  11 +++-
 src/config/primitives.rs  | 110 ++++++++++++++++++++++++++++++++++++++
 5 files changed, 189 insertions(+), 1 deletion(-)

diff --git a/defaults.toml b/defaults.toml
index 9a0be3c..6ac6616 100644
--- a/defaults.toml
+++ b/defaults.toml
@@ -19,6 +19,8 @@ buffer_capacity = 102400
 service_name = "pict-rs"
 targets = "info"
 
+[metrics]
+
 [old_db]
 path = "/mnt"
 
@@ -32,6 +34,10 @@ filters = [
     "thumbnail",
 ]
 
+[media.retention]
+variants = "7d"
+proxy = "7d"
+
 [media.image]
 max_width = 10000
 max_height = 10000
diff --git a/src/config/commandline.rs b/src/config/commandline.rs
index fc5421c..282be5b 100644
--- a/src/config/commandline.rs
+++ b/src/config/commandline.rs
@@ -7,6 +7,8 @@ use clap::{Parser, Subcommand};
 use std::{net::SocketAddr, path::PathBuf};
 use url::Url;
 
+use super::primitives::RetentionValue;
+
 impl Args {
     pub(super) fn into_output(self) -> Output {
         let Args {
@@ -51,6 +53,8 @@ impl Args {
                 metrics_prometheus_address,
                 media_preprocess_steps,
                 media_max_file_size,
+                media_retention_variants,
+                media_retention_proxy,
                 media_image_max_width,
                 media_image_max_height,
                 media_image_max_area,
@@ -109,6 +113,11 @@ impl Args {
                     prometheus_address: metrics_prometheus_address,
                 };
 
+                let retention = Retention {
+                    variants: media_retention_variants,
+                    proxy: media_retention_proxy,
+                };
+
                 let image_quality = ImageQuality {
                     avif: media_image_quality_avif,
                     jpeg: media_image_quality_jpeg,
@@ -170,6 +179,7 @@ impl Args {
                     max_file_size: media_max_file_size,
                     preprocess_steps: media_preprocess_steps,
                     filters: media_filters,
+                    retention: retention.set(),
                     image: image.set(),
                     animation: animation.set(),
                     video: video.set(),
@@ -453,6 +463,8 @@ struct Media {
     #[serde(skip_serializing_if = "Option::is_none")]
     filters: Option<Vec<String>>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    retention: Option<Retention>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     image: Option<Image>,
     #[serde(skip_serializing_if = "Option::is_none")]
     animation: Option<Animation>,
@@ -460,6 +472,27 @@ struct Media {
     video: Option<Video>,
 }
 
+#[derive(Debug, Default, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+struct Retention {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    variants: Option<RetentionValue>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    proxy: Option<RetentionValue>,
+}
+
+impl Retention {
+    fn set(self) -> Option<Self> {
+        let any_set = self.variants.is_some() || self.proxy.is_some();
+
+        if any_set {
+            Some(self)
+        } else {
+            None
+        }
+    }
+}
+
 #[derive(Debug, Default, serde::Serialize)]
 #[serde(rename_all = "snake_case")]
 struct Image {
@@ -768,6 +801,18 @@ struct Run {
     #[arg(long)]
     media_max_file_size: Option<usize>,
 
+    /// How long to keep image "variants" around
+    ///
+    /// A variant is any processed version of an original image
+    #[arg(long)]
+    media_retention_variants: Option<RetentionValue>,
+
+    /// How long to keep "proxied" images around
+    ///
+    /// Proxied images are any images ingested using the media proxy functionality
+    #[arg(long)]
+    media_retention_proxy: Option<RetentionValue>,
+
     /// The maximum width, in pixels, for uploaded images
     #[arg(long)]
     media_image_max_width: Option<u16>,
diff --git a/src/config/defaults.rs b/src/config/defaults.rs
index af8e364..98c7a1f 100644
--- a/src/config/defaults.rs
+++ b/src/config/defaults.rs
@@ -74,11 +74,19 @@ struct OldDbDefaults {
 struct MediaDefaults {
     max_file_size: usize,
     filters: Vec<String>,
+    retention: RetentionDefaults,
     image: ImageDefaults,
     animation: AnimationDefaults,
     video: VideoDefaults,
 }
 
+#[derive(Clone, Debug, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+struct RetentionDefaults {
+    variants: &'static str,
+    proxy: &'static str,
+}
+
 #[derive(Clone, Debug, serde::Serialize)]
 #[serde(rename_all = "snake_case")]
 struct ImageDefaults {
@@ -237,6 +245,7 @@ impl Default for MediaDefaults {
                 "resize".into(),
                 "thumbnail".into(),
             ],
+            retention: Default::default(),
             image: Default::default(),
             animation: Default::default(),
             video: Default::default(),
@@ -244,6 +253,15 @@ impl Default for MediaDefaults {
     }
 }
 
+impl Default for RetentionDefaults {
+    fn default() -> Self {
+        RetentionDefaults {
+            variants: "7d",
+            proxy: "7d",
+        }
+    }
+}
+
 impl Default for ImageDefaults {
     fn default() -> Self {
         ImageDefaults {
diff --git a/src/config/file.rs b/src/config/file.rs
index b4fe068..af4b9dc 100644
--- a/src/config/file.rs
+++ b/src/config/file.rs
@@ -1,5 +1,5 @@
 use crate::{
-    config::primitives::{Filesystem, LogFormat, Targets},
+    config::primitives::{Filesystem, LogFormat, RetentionValue, Targets},
     formats::{AnimationFormat, AudioCodec, ImageFormat, VideoCodec},
     serde_str::Serde,
 };
@@ -173,6 +173,8 @@ pub(crate) struct Media {
 
     pub(crate) filters: BTreeSet<String>,
 
+    pub(crate) retention: Retention,
+
     pub(crate) image: Image,
 
     pub(crate) animation: Animation,
@@ -180,6 +182,13 @@ pub(crate) struct Media {
     pub(crate) video: Video,
 }
 
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub(crate) struct Retention {
+    pub(crate) variants: RetentionValue,
+
+    pub(crate) proxy: RetentionValue,
+}
+
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
 pub(crate) struct Image {
     pub(crate) max_width: u16,
diff --git a/src/config/primitives.rs b/src/config/primitives.rs
index 293d78f..b37378e 100644
--- a/src/config/primitives.rs
+++ b/src/config/primitives.rs
@@ -3,6 +3,116 @@ use std::{fmt::Display, path::PathBuf, str::FromStr};
 use tracing::Level;
 use url::Url;
 
+#[derive(Clone, Debug)]
+pub(crate) struct RetentionValue {
+    value: u32,
+    units: TimeUnit,
+}
+
+#[derive(Clone, Debug)]
+enum TimeUnit {
+    Minute,
+    Hour,
+    Day,
+    Year,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub(crate) enum RetentionValueError {
+    #[error("No number in retention value")]
+    NoValue,
+    #[error("Provided value is invalid")]
+    InvalidValue,
+    #[error("No units in retention value")]
+    NoUnits,
+    #[error("Provided units are invalid")]
+    InvalidUnits,
+}
+
+impl RetentionValue {
+    pub(crate) fn to_duration(&self) -> time::Duration {
+        match self.units {
+            TimeUnit::Minute => time::Duration::minutes(i64::from(self.value)),
+            TimeUnit::Hour => time::Duration::hours(i64::from(self.value)),
+            TimeUnit::Day => time::Duration::days(i64::from(self.value)),
+            TimeUnit::Year => time::Duration::days(i64::from(self.value) * 365),
+        }
+    }
+}
+
+impl std::str::FromStr for RetentionValue {
+    type Err = RetentionValueError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let num_str = s.chars().take_while(|c| c.is_digit(10)).collect::<String>();
+
+        if num_str.is_empty() {
+            return Err(RetentionValueError::NoValue);
+        }
+
+        let value: u32 = num_str
+            .parse()
+            .map_err(|_| RetentionValueError::InvalidValue)?;
+
+        let units = s.trim_start_matches(&num_str).to_lowercase();
+
+        if units.is_empty() {
+            return Err(RetentionValueError::NoUnits);
+        }
+
+        let units = match units.as_str() {
+            "m" | "minute" => TimeUnit::Minute,
+            "h" | "hour" => TimeUnit::Hour,
+            "d" | "day" => TimeUnit::Day,
+            "y" | "year" => TimeUnit::Year,
+            _ => return Err(RetentionValueError::InvalidUnits),
+        };
+
+        Ok(RetentionValue { value, units })
+    }
+}
+
+impl std::fmt::Display for TimeUnit {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Minute => write!(f, "m"),
+            Self::Hour => write!(f, "h"),
+            Self::Day => write!(f, "d"),
+            Self::Year => write!(f, "y"),
+        }
+    }
+}
+
+impl std::fmt::Display for RetentionValue {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}{}", self.value, self.units)
+    }
+}
+
+impl<'de> serde::Deserialize<'de> for RetentionValue {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        use serde::de::Error;
+
+        let s = String::deserialize(deserializer)?;
+
+        s.parse().map_err(D::Error::custom)
+    }
+}
+
+impl serde::Serialize for RetentionValue {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        let s = self.to_string();
+
+        s.serialize(serializer)
+    }
+}
+
 #[derive(
     Clone,
     Copy,