add s3 TLS option

README.md

@@ -76,9 +76,9 @@ Additionally, you can use the following options when starting an instance:
 
 - `s3`: set up an S3 server and configure to Nextcloud to use it as primary
   storage.
-- `<path to app.tar.gz>`: by specifying the path to an app package this package
+  - `s3s`: enable TLS for the S3 setup.
-  will be extracted into the apps. directory of the new instance (overwriting
+  - `s3mb`: enable multi-bucket S3 setup.
-  any existing app code). This can be used to quickly test a packaged app.
+  - `s3m`: enable multi-instance S3 setup.
 - `ldap`: set up an LDAP server.
 - `office`: set up a Nextcloud Office server.
 - `onlyoffice` setup an onlyoffice document server.
@@ -100,6 +100,9 @@ Additionally, you can use the following options when starting an instance:
   configure it the mail server.
 - `redis`: start a separate container for redis.
 - `redis-tls`: connect to redis over TLS.
+- `<path to app.tar.gz>`: by specifying the path to an app package this package
+  will be extracted into the apps. directory of the new instance (overwriting
+  any existing app code). This can be used to quickly test a packaged app.
 - The name of any configured preset.
 
 #### Run tests in a new instance

certificates/s3/private.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCQuTk+irdXRGbY
+JUu+AFmMM4/CCtBIBBgGIG18tesgQUeHEPdRHbBypvFKhpXvlKQbVbiaZxFjtlvn
+L2ReN7gYwjiTjLWuaDgOzGQyObwSJpedlcd5Q957WNlc5OjpoK8zZq9EGnmvjIqe
+5VVBne85RZVw6+i4ljEiWoXCiy0iOPIL4jKO8kfO8EmTTh6ge2sLCT5jFT/V63/B
+hjEobA7+vPbAmEo+Qs4adnBSrlX9nLtL1j4gqawAQMmGl/Ti75T0uQvNxdq2gddc
+n0wyWhyERnZNeMv5sfkwCOTuetNNtGLf/lTkXNdaOCRvW5jJ0EjYYdpQG5R0do1U
+XFLH1cZZAgMBAAECggEAR0xpTk2Ku5yASlY9dXK4qyCv3znymLgjmckaB4mcN7zR
+X1JVdYn55tImJ8AcV/bTzn+xvaevYn9x0XiAqwYqVVBCDTcSPsUrcObzKedVp1+J
+7GHg7vYnwn7oPyKrOIYoKluZVyTv9DN6C4QSN4x2UbHdSM+ATIf51uHf6hMk/ilv
+4uw3csxSCpLOqqsYCQarES7SypcETjFpNnIfTPt7q2Y2DbIDttTTjzrm0/GBP0WE
+OYNvzZZPZRPJfy3et3r9vJbqWzGHvOttzQ+EFHjwPTMfW4tsHcCsEKSGWwLpG4bN
+FPNx0+QCqDiChesdiCHFNSk+u7pRZrHdjuDJEuKSzQKBgQDMDgdCGBQfgaNnkRz/
+aiv3V200/vXegnc0Jz49Dye5AxEVu0X1m2xZpJv+qEwbOx5B+1PV3gfVP/iRf+FK
+MAwFbmb7hGcDE8AGNsSpQydjwzKoi/M67YXv7T8dgWKnwz0eyU4K2IOGInGxuFty
+Ik7+DTqz+Ikh1RiAoGbKfw9yhwKBgQC1kKhjxB7r/uSLcfOSg1mLcR5lTrNDQAPQ
+GnsIje0nD5Tv52/k6U4tk15vjL4t6KZUFo9SJ4O1kM1veyuOJuol2AfPXS+H/Izo
+5BjmoZ0jOONOZZiRIB1moQSy1qhTAeZB9S1ORxQ3dIBPqm+oyADPTTsNV67Cwnt4
+woeZRUPYHwKBgQDE0AcKJcVK+jQMUXfBlrsfTvDjO8MTwYyN/gfWxsZOeXnCFyYM
+FcO01sMrJVJ6tVOi2nFrB0NQ2Om8FLbMYnlFx82GbJca7bK5i5u1kjLs0zoKPSn1
+vWEBIDhPEhuAqhxKlGk0ps580r4MZz+0XwkHmuTy7xX9TtbaQVvDljflAwKBgDKy
+3hJdpTTIzBCUFSuIOezR/WbUfwH8UhQ+ELTmzJ3nn/MNcRU+gHIBgJEtf71aBXfd
+hM+v8Ps2H+dNQXBENYWzuRqSLr+OKdquNrXP0w0OyYoOnHeJvCv4MlOt1Pq4wQ8R
+40DEYETL5zhXoy5CCtfX/PFQ1p/Tpp6l0y9dRACJAoGALwUbyyDy85b2xRQB6RtU
+I+5Vz5cd/1eQdCkoU9mX4qWA/hWpgc7Z2Jd67LW/WWtVjlF9hva/WNDSfGsXo2ew
+C8OofvlfIuFDOCXrodYdHE1Q4g5TZdESr0XAqopq+QzBs89qbIy05kM9iuE4yFUo
+xeimCY9oDWTeGw/XrLdHZF4=
+-----END PRIVATE KEY-----

certificates/s3/public.crt

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDXTCCAkWgAwIBAgIULiChaTwmVx6nRTHohmPuf55/4jUwDQYJKoZIhvcNAQEL
+BQAwPjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxDTALBgNVBAoM
+BEhhemUxCzAJBgNVBAMMAnMzMB4XDTI2MDMwOTE3NTgwOVoXDTI3MDMwOTE3NTgw
+OVowPjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxDTALBgNVBAoM
+BEhhemUxCzAJBgNVBAMMAnMzMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
+AQEAkLk5Poq3V0Rm2CVLvgBZjDOPwgrQSAQYBiBtfLXrIEFHhxD3UR2wcqbxSoaV
+75SkG1W4mmcRY7Zb5y9kXje4GMI4k4y1rmg4DsxkMjm8EiaXnZXHeUPee1jZXOTo
+6aCvM2avRBp5r4yKnuVVQZ3vOUWVcOvouJYxIlqFwostIjjyC+IyjvJHzvBJk04e
+oHtrCwk+YxU/1et/wYYxKGwO/rz2wJhKPkLOGnZwUq5V/Zy7S9Y+IKmsAEDJhpf0
+4u+U9LkLzcXatoHXXJ9MMlochEZ2TXjL+bH5MAjk7nrTTbRi3/5U5FzXWjgkb1uY
+ydBI2GHaUBuUdHaNVFxSx9XGWQIDAQABo1MwUTAdBgNVHQ4EFgQUJJ8HiT2zmuF5
+6WHHFsTHgkrayxYwHwYDVR0jBBgwFoAUJJ8HiT2zmuF56WHHFsTHgkrayxYwDwYD
+VR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAUF0lB/qIrxkgZ4sqNrw4
+CInHCK29XVaMoqk1QZyS/KhWDM+zgbA92OxxuhCKw4iJEajZvgg0S9RtGkBNmquU
+l0rf0JdALd0jPkWr7+3OeqlcgOs2EH7PTqrrbXTGsR12D+Ot+OerQeWXmO28Zrl8
+4O67TwQtslXwZzeCrtiwAA2DrIYpSLzh+qDtwbY5hMG5zmqqjBM20Ysgxszh4rhl
+KR6skXwZwkVVhKpK76qwnU02PIMr8auL1csx8/uBTd/UzX2veqlkOP5V/Gg6eEbI
+4fTOzq7k+FyuzSkrEX4Vc9GbWcRvoVZh+qAKUKstqlE2iCrqmZ+Wal6GA8JA5SZ+
+bQ==
+-----END CERTIFICATE-----

nix/image/configs/nc/s3s.php

@@ -0,0 +1,15 @@
+	'objectstore' => [
+		'class' => 'OC\Files\ObjectStore\S3',
+		'arguments' => [
+			'bucket' => 'nextcloud',
+			'autocreate' => true,
+			'key'    => 'minio',
+			'secret' => 'minio123',
+			'hostname' => 's3',
+			'port' => 9000,
+			'use_ssl' => true,
+			'use_path_style' => true,
+			'uploadPartSize' => 52428800,
+			'use_nextcloud_bundle' => true,
+		],
+	],

nix/image/scripts/nc-auto-config

@@ -32,6 +32,11 @@ then
 	sed -i '/\/\/PLACEHOLDER/ r /etc/nc/s3.php' /var/www/html/config/config.php
 fi
 
+if [ -n "${S3S:-}" ]
+then
+	sed -i '/\/\/PLACEHOLDER/ r /etc/nc/s3s.php' /var/www/html/config/config.php
+fi
+
 if [ -n "${S3MB:-}" ]
 then
 	sed -i '/\/\/PLACEHOLDER/ r /etc/nc/s3mb.php' /var/www/html/config/config.php

src/cloud.rs

@@ -498,6 +498,22 @@ impl Cloud {
             }
         };
 
+        for pre_setup in options
+            .services
+            .iter()
+            .flat_map(|service| service.pre_setup(docker, &id, config).into_iter().flatten())
+        {
+            exec(
+                docker,
+                &container,
+                &uid.to_string(),
+                pre_setup,
+                vec!["NC_IS_CONFIG_READ_ONLY=1"],
+                Some(stdout()),
+            )
+            .await?;
+        }
+
         containers.push(container);
 
         let options_clone = options.clone();

src/service.rs

@@ -100,6 +100,15 @@ pub trait ServiceTrait {
         Ok(HashMap::default())
     }
 
+    fn pre_setup(
+        &self,
+        _docker: &Docker,
+        _cloud_id: &str,
+        _config: &HazeConfig,
+    ) -> Result<Vec<Vec<String>>> {
+        Ok(Vec::new())
+    }
+
     async fn post_setup(
         &self,
         _docker: &Docker,
@@ -205,6 +214,8 @@ impl ServiceTrait for RedisTls {
 pub enum ServiceType {
     /// S3 Primary storage and external storage
     S3,
+    /// S3 Primary storage with TLS
+    S3s,
     /// S3 multi-object store Primary storage and external storage
     S3m,
     /// S3 multi-bucket Primary storage and external storage
@@ -307,6 +318,7 @@ impl Service {
         if let Ok(ty) = ServiceType::from_str(ty) {
             match ty {
                 ServiceType::S3 => Some(vec![Service::ObjectStore(ObjectStore::S3)]),
+                ServiceType::S3s => Some(vec![Service::ObjectStore(ObjectStore::S3s)]),
                 ServiceType::S3m => Some(vec![Service::ObjectStore(ObjectStore::S3m)]),
                 ServiceType::S3mb => Some(vec![Service::ObjectStore(ObjectStore::S3mb)]),
                 ServiceType::Azure => Some(vec![Service::ObjectStore(ObjectStore::Azure)]),

src/service/objectstore.rs

@@ -10,11 +10,15 @@ use bollard::models::{
 use bollard::query_parameters::CreateContainerOptions;
 use bollard::Docker;
 use maplit::hashmap;
-use miette::IntoDiagnostic;
+use miette::{IntoDiagnostic, WrapErr};
+use serde_json::Value;
+use std::collections::HashMap;
+use std::fs::{create_dir_all, write};
 
 #[derive(Debug, Clone, Eq, PartialEq)]
 pub enum ObjectStore {
     S3,
+    S3s,
     S3m,
     S3mb,
     Azure,
@@ -23,7 +27,7 @@ pub enum ObjectStore {
 impl ObjectStore {
     fn image(&self) -> &str {
         match self {
-            ObjectStore::S3 | ObjectStore::S3m | ObjectStore::S3mb => {
+            ObjectStore::S3 | ObjectStore::S3m | ObjectStore::S3mb | ObjectStore::S3s => {
                 "minio/minio:RELEASE.2024-07-16T23-46-41Z"
             }
             ObjectStore::Azure => "arafato/azurite:2.6.5",
@@ -32,7 +36,7 @@ impl ObjectStore {
 
     fn self_env(&self) -> Vec<&str> {
         match self {
-            ObjectStore::S3 | ObjectStore::S3m | ObjectStore::S3mb => {
+            ObjectStore::S3 | ObjectStore::S3m | ObjectStore::S3mb | ObjectStore::S3s => {
                 vec!["MINIO_ACCESS_KEY=minio", "MINIO_SECRET_KEY=minio123"]
             }
             ObjectStore::Azure => vec![],
@@ -41,17 +45,54 @@ impl ObjectStore {
 
     fn host_name(&self) -> &str {
         match self {
-            ObjectStore::S3 | ObjectStore::S3m | ObjectStore::S3mb => "s3",
+            ObjectStore::S3 | ObjectStore::S3m | ObjectStore::S3mb | ObjectStore::S3s => "s3",
             ObjectStore::Azure => "azure",
         }
     }
 
     fn args(&self) -> &[&str] {
         match self {
-            ObjectStore::S3 | ObjectStore::S3m | ObjectStore::S3mb => &["server", "/data"],
+            ObjectStore::S3 | ObjectStore::S3m | ObjectStore::S3mb | ObjectStore::S3s => {
+                &["server", "/data"]
+            }
             _ => &[],
         }
     }
+
+    fn volumes(&self, config: &HazeConfig) -> Option<Vec<String>> {
+        match self {
+            ObjectStore::S3s => {
+                let cert_dir = config.work_dir.join("certificates/s3");
+                create_dir_all(&cert_dir)
+                    .into_diagnostic()
+                    .wrap_err("Failed to create redis certificate directory")
+                    .unwrap();
+                let s3_cert_path = config.work_dir.join("certificates/s3/public.crt");
+                let s3_key_path = config.work_dir.join("certificates/s3/private.key");
+                if !s3_cert_path.exists() {
+                    write(
+                        &s3_cert_path,
+                        include_bytes!("../../certificates/s3/public.crt"),
+                    )
+                    .into_diagnostic()
+                    .wrap_err("Failed to write s3 certificate")
+                    .unwrap();
+                }
+                if !s3_key_path.exists() {
+                    write(
+                        &s3_key_path,
+                        include_bytes!("../../certificates/s3/private.key"),
+                    )
+                    .into_diagnostic()
+                    .wrap_err("Failed to write s3 key")
+                    .unwrap();
+                }
+
+                Some(vec![format!("{cert_dir}:/root/.minio/certs:ro")])
+            }
+            _ => None,
+        }
+    }
 }
 
 #[async_trait::async_trait]
@@ -59,6 +100,7 @@ impl ServiceTrait for ObjectStore {
     fn name(&self) -> &str {
         match self {
             ObjectStore::S3 => "s3",
+            ObjectStore::S3s => "s3s",
             ObjectStore::S3m => "s3m",
             ObjectStore::S3mb => "s3mb",
             ObjectStore::Azure => "azure",
@@ -68,8 +110,9 @@ impl ServiceTrait for ObjectStore {
     fn env(&self) -> &[&str] {
         match self {
             ObjectStore::S3 => &["S3=1"],
+            ObjectStore::S3s => &["S3S=1"],
             ObjectStore::S3m => &["S3M=1"],
-            ObjectStore::S3mb => &["S3MB=1"],
+            ObjectStore::S3mb => &["S3MB =1"],
             ObjectStore::Azure => &["AZURE=1"],
         }
     }
@@ -79,7 +122,7 @@ impl ServiceTrait for ObjectStore {
         docker: &Docker,
         cloud_id: &str,
         network: &str,
-        _config: &HazeConfig,
+        config: &HazeConfig,
         _options: &CloudOptions,
     ) -> Result<Vec<String>> {
         pull_image(docker, self.image()).await?;
@@ -92,6 +135,7 @@ impl ServiceTrait for ObjectStore {
             env: Some(self.self_env().into_iter().map(String::from).collect()),
             host_config: Some(HostConfig {
                 network_mode: Some(network.to_string()),
+                binds: self.volumes(config),
                 ..Default::default()
             }),
             labels: Some(hashmap! {
@@ -165,14 +209,47 @@ impl ServiceTrait for ObjectStore {
         &["files_external"]
     }
 
+    fn config(
+        &self,
+        _docker: &Docker,
+        _cloud_id: &str,
+        _config: &HazeConfig,
+    ) -> Result<HashMap<String, Value>> {
+        match self {
+            ObjectStore::S3s => Ok(hashmap![
+                "default_certificates_bundle_path".into() => Value::String("/var/www/html/data/ca-bundle.crt".into()),
+            ]),
+            _ => Ok(HashMap::default()),
+        }
+    }
+
+    fn pre_setup(
+        &self,
+        _docker: &Docker,
+        _cloud_id: &str,
+        _config: &HazeConfig,
+    ) -> Result<Vec<Vec<String>>> {
+        match self {
+            ObjectStore::S3s => Ok(vec![
+                vec!["mkdir".into(), "-p".into(), "/var/www/html/data".into()],
+                vec![
+                    "sh".into(),
+                    "-c".into(),
+                    "cat /var/www/html/resources/config/ca-bundle.crt /certificates/s3/public.crt > /var/www/html/data/ca-bundle.crt".into(),
+                ],
+            ]),
+            _ => Ok(Vec::new()),
+        }
+    }
+
     async fn post_setup(
         &self,
         _docker: &Docker,
         _cloud_id: &str,
         _config: &HazeConfig,
     ) -> Result<Vec<String>> {
-        if *self == ObjectStore::S3 {
+        match self {
-            Ok(vec![
+            ObjectStore::S3 => Ok(vec![
                 "occ files_external:create s3 amazons3 amazons3::accesskey".into(),
                 "occ files_external:config 1 bucket ext".into(),
                 "occ files_external:config 1 hostname s3".into(),
@@ -182,15 +259,25 @@ impl ServiceTrait for ObjectStore {
                 "occ files_external:config 1 key minio".into(),
                 "occ files_external:config 1 secret minio123".into(),
                 "mc alias set s3 http://s3:9000 minio minio123".into(),
-            ])
+            ]),
-        } else {
+            // ObjectStore::S3s => Ok(vec![
-            Ok(Vec::new())
+            //     "occ files_external:create s3 amazons3 amazons3::accesskey".into(),
+            //     "occ files_external:config 1 bucket ext".into(),
+            //     "occ files_external:config 1 hostname s3".into(),
+            //     "occ files_external:config 1 port 9000".into(),
+            //     "occ files_external:config 1 use_ssl true".into(),
+            //     "occ files_external:config 1 use_path_style true".into(),
+            //     "occ files_external:config 1 key minio".into(),
+            //     "occ files_external:config 1 secret minio123".into(),
+            //     "mc alias set s3 https://s3:9000 minio minio123".into(),
+            // ]),
+            _ => Ok(Vec::new()),
         }
     }
 
     fn proxy_port(&self) -> u16 {
         match self {
-            ObjectStore::S3 | ObjectStore::S3m | ObjectStore::S3mb => 9000,
+            ObjectStore::S3 | ObjectStore::S3m | ObjectStore::S3mb | ObjectStore::S3s => 9000,
             ObjectStore::Azure => 10000,
         }
     }