From eaf54efb25df598ac8db7b1a0c89a6e5be442179 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Fri, 29 Oct 2021 18:05:47 +0200 Subject: Add doc for Nextcloud --- doc/book/src/SUMMARY.md | 19 ++-- doc/book/src/connect/apps.md | 157 +++++++++++++++++++++++++++++ doc/book/src/connect/cli-nextcloud-gui.png | Bin 0 -> 201685 bytes doc/book/src/connect/cli.md | 127 +++++++++++++++++++++++ doc/book/src/connect/code.md | 1 + doc/book/src/connect/index.md | 38 +++++++ doc/book/src/connect/repositories.md | 1 + doc/book/src/connect/websites.md | 1 + doc/book/src/cookbook/clients.md | 105 ------------------- doc/book/src/cookbook/reverse_proxy.md | 1 + 10 files changed, 338 insertions(+), 112 deletions(-) create mode 100644 doc/book/src/connect/apps.md create mode 100644 doc/book/src/connect/cli-nextcloud-gui.png create mode 100644 doc/book/src/connect/cli.md create mode 100644 doc/book/src/connect/code.md create mode 100644 doc/book/src/connect/index.md create mode 100644 doc/book/src/connect/repositories.md create mode 100644 doc/book/src/connect/websites.md delete mode 100644 doc/book/src/cookbook/clients.md create mode 100644 doc/book/src/cookbook/reverse_proxy.md diff --git a/doc/book/src/SUMMARY.md b/doc/book/src/SUMMARY.md index 771a2cad..236ceeec 100644 --- a/doc/book/src/SUMMARY.md +++ b/doc/book/src/SUMMARY.md @@ -5,14 +5,19 @@ - [Quick start](./quick_start/index.md) - [Cookbook](./cookbook/index.md) - - [Deploying Garage](./cookbook/real_world.md) - - [Configuring S3 clients](./cookbook/clients.md) - - [Hosting a website](./cookbook/website.md) - - [Recovering from failures](./cookbook/recovering.md) - [Building from source](./cookbook/from_source.md) - - [Starting with Systemd](./cookbook/systemd.md) - - [Integrate as a media backend]() - - [Operate a cluster]() + - [Integration with systemd](./cookbook/systemd.md) + - [Configuring a reverse proxy](./cookbook/reverse_proxy.md) + - [Production Deployment](./cookbook/real_world.md) + - [Recovering from failures](./cookbook/recovering.md) + +- [Integrations](./connect/index.md) + - [Apps (Nextcloud, Peertube...)](./connect/apps.md) + - [Websites (Hugo, Jekyll, Publii...)](./connect/websites.md) + - [Repositories (Docker, Nix...)](./connect/repositories.md) + - [CLI tools (rclone, awscli, mc...)](./connect/cli.md) + - [Your code (PHP, JS, Go...)](./connect/code.md) + - [Reference Manual](./reference_manual/index.md) - [Garage configuration file](./reference_manual/configuration.md) diff --git a/doc/book/src/connect/apps.md b/doc/book/src/connect/apps.md new file mode 100644 index 00000000..9ad99a7d --- /dev/null +++ b/doc/book/src/connect/apps.md @@ -0,0 +1,157 @@ +# Apps (Nextcloud, Peertube...) + +In this section, we cover the following software: [Nextcloud](#nextcloud), [Peertube](#peertube), [Mastodon](#mastodon), [Matrix](#matrix) + +## Nextcloud + +Nextcloud is a popular file synchronisation and backup service. +By default, Nextcloud stores its data on the local filesystem. +If you want to expand your storage to aggregate multiple servers, Garage is the way to go. + +A S3 backend can be configured in two ways on Nextcloud, either as Primary Storage or as an External Storage. +Primary storage will store all your data on S3, in an opaque manner, and will provide the best performances. +External storage enable you to select which data will be store on S3, you file hierarchy will be preserved in S3, but it might be slower. + +In the following, we cover both methods but before reading our guide, we suppose you have done some preliminary steps. +First, we expect you have an already installed and configured Nextcloud instance. +Second, we suppose you have created a key and a bucket. + +As a reminder, you can create a key for your nextcloud instance as follow: + +```bash +garage key new --name nextcloud-key +``` + +Keep the Key ID and the Secret key in a pad, they will be needed later. +Then you can create a bucket and give read/write rights to your key on this bucket with: + +```bash +garage bucket create nextcloud +garage bucket allow nextcloud --read --write --key nextcloud-key +``` + + +### Primary Storage + +Now edit your Nextcloud configuration file to enable object storage. +On my installation, the config. file is located at the following path: `/var/www/nextcloud/config/config.php`. +We will add a new root key to the `$CONFIG` dictionnary named `objectstore`: + +```php + [ + 'class' => '\\OC\\Files\\ObjectStore\\S3', + 'arguments' => [ + 'bucket' => 'nextcloud', // Your bucket name, must be created before + 'autocreate' => false, // Garage does not support autocreate + 'key' => 'xxxxxxxxx', // The Key ID generated previously + 'secret' => 'xxxxxxxxx', // The Secret key generated previously + 'hostname' => '127.0.0.1', // Can also be a domain name, eg. garage.example.com + 'port' => 3900, // Put your reverse proxy port or your S3 API port + 'use_ssl' => false, // Set it to true if you have a TLS enabled reverse proxy + 'region' => 'garage', // Garage has only one region named "garage" + 'use_path_style' => true // Garage supports only path style, must be set to true + ], +], +``` + +That's all, your Nextcloud will store all your data to S3. +To test your new configuration, just reload your Nextcloud webpage and start sending data. + +*External link:* [Nextcloud Documentation > Primary Storage](https://docs.nextcloud.com/server/latest/admin_manual/configuration_files/primary_storage.html) + +### External Storage + +**From the GUI.** Activate the "External storage support" app from the "Applications" page (click on your account icon on the top right corner of your screen to display the menu). Go to your parameters page (also located below your account icon). Click on external storage (or the corresponding translation in your language). + +[![Screenshot of the External Storage form](./cli-nextcloud-gui.png)](./cli-nextcloud-gui.png) +*Click on the picture to zoom* + +Add a new external storage. Put what you want in "folder name" (eg. "shared"). Select "Amazon S3". Keep "Access Key" for the Authentication field. +In Configuration, put your bucket name (eg. nextcloud), the host (eg. 127.0.0.1), the port (eg. 3900 or 443), the region (garage). Tick the SSL box if you have put an HTTPS proxy in front of garage. You must tick the "Path access" box and you must leave the "Legacy authentication (v2)" box empty. Put your Key ID (eg. GK...) and your Secret Key in the last two input boxes. Finally click on the tick symbol on the right of your screen. + +Now go to your "Files" app and a new "linked folder" has appeared with the name you chose earlier (eg. "shared"). + +*External link:* [Nextcloud Documentation > External Storage Configuration GUI](https://docs.nextcloud.com/server/latest/admin_manual/configuration_files/external_storage_configuration_gui.html) + +**From the CLI.** First install the external storage application: + +```bash +php occ app:install files_external +``` + +Then add a new mount point with: + +```bash + php occ files_external:create \ + -c bucket=nextcloud \ + -c hostname=127.0.0.1 \ + -c port=3900 \ + -c region=garage \ + -c use_ssl=false \ + -c use_path_style=true \ + -c legacy_auth=false \ + -c key=GKxxxx \ + -c secret=xxxx \ + shared amazons3 amazons3::accesskey +``` + +Adapt the `hostname`, `port`, `use_ssl`, `key`, and `secret` entries to your configuration. +Do not change the `use_path_style` and `legacy_auth` entries, other configurations are not supported. + +*External link:* [Nextcloud Documentation > occ command > files external](https://docs.nextcloud.com/server/latest/admin_manual/configuration_server/occ_command.html#files-external-label) + + +## Peertube + +https://docs.joinpeertube.org/admin-remote-storage + + +## Mastodon + +https://docs.joinmastodon.org/admin/config/#cdn + +## Matrix + +### synapse-s3-storage-provider + +https://github.com/matrix-org/synapse-s3-storage-provider + +### matrix-media-repo + +https://github.com/turt2live/matrix-media-repo + +## Pixelfed + +https://docs.pixelfed.org/technical-documentation/env.html#filesystem + +## Pleroma + +https://docs-develop.pleroma.social/backend/configuration/cheatsheet/#pleromauploaderss3 + +## Lemmy + +via pict-rs +https://git.asonix.dog/asonix/pict-rs/commit/f9f4fc63d670f357c93f24147c2ee3e1278e2d97 + +## Funkwhale + +https://docs.funkwhale.audio/admin/configuration.html#s3-storage + +## Misskey + +https://github.com/misskey-dev/misskey/commit/9d944243a3a59e8880a360cbfe30fd5a3ec8d52d + +## Prismo + +https://gitlab.com/prismosuite/prismo/-/blob/dev/.env.production.sample#L26-33 + +## Owncloud Infinite Scale (ocis) + +## Unsupported + + - Mobilizon: No S3 integration + - WriteFreely: No S3 integration + - Plume: No S3 integration diff --git a/doc/book/src/connect/cli-nextcloud-gui.png b/doc/book/src/connect/cli-nextcloud-gui.png new file mode 100644 index 00000000..7a58a3ab Binary files /dev/null and b/doc/book/src/connect/cli-nextcloud-gui.png differ diff --git a/doc/book/src/connect/cli.md b/doc/book/src/connect/cli.md new file mode 100644 index 00000000..168f3ff5 --- /dev/null +++ b/doc/book/src/connect/cli.md @@ -0,0 +1,127 @@ +# CLI tools + +CLI tools allow you to query the S3 API without too many abstractions. +These tools are particularly suitable for debug, backups, website deployments or any scripted task that need to handle data. + +## Minio client (recommended) + +Use the following command to set an "alias", i.e. define a new S3 server to be +used by the Minio client: + +```bash +mc alias set \ + garage \ + \ + \ + \ + --api S3v4 +``` + +Remember that `mc` is sometimes called `mcli` (such as on Arch Linux), to avoid conflicts +with Midnight Commander. + +Some commands: + +```bash +# list buckets +mc ls garage/ + +# list objets in a bucket +mc ls garage/my_files + +# copy from your filesystem to garage +mc cp /proc/cpuinfo garage/my_files/cpuinfo.txt + +# copy from garage to your filesystem +mc cp garage/my_files/cpuinfo.txt /tmp/cpuinfo.txt + +# mirror a folder from your filesystem to garage +mc mirror --overwrite ./book garage/garagehq.deuxfleurs.fr +``` + + +## AWS CLI + +Create a file named `~/.aws/credentials` and put: + +```toml +[default] +aws_access_key_id=xxxx +aws_secret_access_key=xxxx +``` + +Then a file named `~/.aws/config` and put: + +```toml +[default] +region=garage +``` + +Now, supposing Garage is listening on `http://127.0.0.1:3901`, you can list your buckets with: + +```bash +aws --endpoint-url http://127.0.0.1:3901 s3 ls +``` + +Passing the `--endpoint-url` parameter to each command is annoying but AWS developers do not provide a corresponding configuration entry. +As a workaround, you can redefine the aws command by editing the file `~/.bashrc`: + +``` +function aws { command aws --endpoint-url http://127.0.0.1:3911 $@ ; } +``` + +*Do not forget to run `source ~/.bashrc` or to start a new terminal before running the next commands.* + +Now you can simply run: + +```bash +# list buckets +aws s3 ls + +# list objects of a bucket +aws s3 ls s3://my_files + +# copy from your filesystem to garage +aws s3 cp /proc/cpuinfo s3://my_files/cpuinfo.txt + +# copy from garage to your filesystem +aws s3 cp s3/my_files/cpuinfo.txt /tmp/cpuinfo.txt +``` + +## `rclone` + +`rclone` can be configured using the interactive assistant invoked using `rclone configure`. + +You can also configure `rclone` by writing directly its configuration file. +Here is a template `rclone.ini` configuration file: + +```ini +[garage] +type = s3 +provider = Other +env_auth = false +access_key_id = +secret_access_key = +region = +endpoint = +force_path_style = true +acl = private +bucket_acl = private +``` + +## Cyberduck + +TODO + +## `s3cmd` + +Here is a template for the `s3cmd.cfg` file to talk with Garage: + +```ini +[default] +access_key = +secret_key = +host_base = +host_bucket = +use_https = False | True +``` diff --git a/doc/book/src/connect/code.md b/doc/book/src/connect/code.md new file mode 100644 index 00000000..88a74ed9 --- /dev/null +++ b/doc/book/src/connect/code.md @@ -0,0 +1 @@ +# Your code (PHP, JS, Go...) diff --git a/doc/book/src/connect/index.md b/doc/book/src/connect/index.md new file mode 100644 index 00000000..56c41255 --- /dev/null +++ b/doc/book/src/connect/index.md @@ -0,0 +1,38 @@ +# Connect it to + +To configure an S3 client to interact with Garage, you will need the following +parameters: + +- An **API endpoint**: this corresponds to the HTTP or HTTPS address + used to contact the Garage server. When runing Garage locally this will usually + be `http://127.0.0.1:3900`. In a real-world setting, you would usually have a reverse-proxy + that adds TLS support and makes your Garage server available under a public hostname + such as `https://garage.example.com`. + +- An **API access key** and its associated **secret key**. These usually look something + like this: `GK3515373e4c851ebaad366558` (access key), + `7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34` (secret key). + These keys are created and managed using the `garage` CLI, as explained in the + [quick start](../quick_start/index.md) guide. + +Most S3 clients can be configured easily with these parameters, +provided that you follow the following guidelines: + +- **Force path style:** Garage does not support DNS-style buckets, which are now by default + on Amazon S3. Instead, Garage uses the legacy path-style bucket addressing. + Remember to configure your client to acknowledge this fact. + +- **Configuring the S3 region:** Garage requires your client to talk to the correct "S3 region", + which is set in the configuration file. This is often set just to `garage`. + If this is not configured explicitly, clients usually try to talk to region `us-east-1`. + Garage should normally redirect your client to the correct region, + but in case your client does not support this you might have to configure it manually. + +We will now provide example configurations for the most common clients per category: + + - [Apps](./apps.md) + - [Websites](./websites.md) + - [Repositories](./repositories.md) + - [CLI tools](./cli.md) + - [Your code](./code.md) + diff --git a/doc/book/src/connect/repositories.md b/doc/book/src/connect/repositories.md new file mode 100644 index 00000000..1e6d2f4f --- /dev/null +++ b/doc/book/src/connect/repositories.md @@ -0,0 +1 @@ +# Repositories (Docker, Nix...) diff --git a/doc/book/src/connect/websites.md b/doc/book/src/connect/websites.md new file mode 100644 index 00000000..c7c3d554 --- /dev/null +++ b/doc/book/src/connect/websites.md @@ -0,0 +1 @@ +# Websites (Hugo, Jekyll, Publii...) diff --git a/doc/book/src/cookbook/clients.md b/doc/book/src/cookbook/clients.md deleted file mode 100644 index 364c982c..00000000 --- a/doc/book/src/cookbook/clients.md +++ /dev/null @@ -1,105 +0,0 @@ -# Configuring S3 clients to interact with Garage - -To configure an S3 client to interact with Garage, you will need the following -parameters: - -- An **API endpoint**: this corresponds to the HTTP or HTTPS address - used to contact the Garage server. When runing Garage locally this will usually - be `http://127.0.0.1:3900`. In a real-world setting, you would usually have a reverse-proxy - that adds TLS support and makes your Garage server available under a public hostname - such as `https://garage.example.com`. - -- An **API access key** and its associated **secret key**. These usually look something - like this: `GK3515373e4c851ebaad366558` (access key), - `7d37d093435a41f2aab8f13c19ba067d9776c90215f56614adad6ece597dbb34` (secret key). - These keys are created and managed using the `garage` CLI, as explained in the - [quick start](../quick_start/index.md) guide. - -Most S3 clients can be configured easily with these parameters, -provided that you follow the following guidelines: - -- **Force path style:** Garage does not support DNS-style buckets, which are now by default - on Amazon S3. Instead, Garage uses the legacy path-style bucket addressing. - Remember to configure your client to acknowledge this fact. - -- **Configuring the S3 region:** Garage requires your client to talk to the correct "S3 region", - which is set in the configuration file. This is often set just to `garage`. - If this is not configured explicitly, clients usually try to talk to region `us-east-1`. - Garage should normally redirect your client to the correct region, - but in case your client does not support this you might have to configure it manually. - -We will now provide example configurations for the most common S3 clients. - -## AWS CLI - -Export the following environment variables: - -```bash -export AWS_ACCESS_KEY_ID= -export AWS_SECRET_ACCESS_KEY= -export AWS_DEFAULT_REGION= -``` - -Now invoke `aws` as follows: - -```bash -aws --endpoint-url s3 -``` - -For instance: `aws --endpoint-url http://127.0.0.1:3901 s3 ls s3://my-bucket/`. - -## Minio client - -Use the following command to set an "alias", i.e. define a new S3 server to be -used by the Minio client: - -```bash -mc alias set \ - garage \ - \ - \ - \ - --api S3v4 -``` - -Remember that `mc` is sometimes called `mcli` (such as on Arch Linux), to avoid conflicts -with the Midnight Commander. - - -## `rclone` - -`rclone` can be configured using the interactive assistant invoked using `rclone configure`. - -You can also configure `rclone` by writing directly its configuration file. -Here is a template `rclone.ini` configuration file: - -```ini -[garage] -type = s3 -provider = Other -env_auth = false -access_key_id = -secret_access_key = -region = -endpoint = -force_path_style = true -acl = private -bucket_acl = private -``` - -## Cyberduck - -TODO - -## `s3cmd` - -Here is a template for the `s3cmd.cfg` file to talk with Garage: - -```ini -[default] -access_key = -secret_key = -host_base = -host_bucket = -use_https = False | True -``` diff --git a/doc/book/src/cookbook/reverse_proxy.md b/doc/book/src/cookbook/reverse_proxy.md new file mode 100644 index 00000000..658f8421 --- /dev/null +++ b/doc/book/src/cookbook/reverse_proxy.md @@ -0,0 +1 @@ +# Configuring a reverse proxy -- cgit v1.2.3