From 9bf94faaa147e0209188db27fc9e5d6ee49656b3 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 12 Jan 2023 20:46:17 +0100 Subject: Add docs about running pict-rs with garage --- doc/book/connect/apps/index.md | 65 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 2 deletions(-) (limited to 'doc/book') diff --git a/doc/book/connect/apps/index.md b/doc/book/connect/apps/index.md index 737351a0..4d556ff8 100644 --- a/doc/book/connect/apps/index.md +++ b/doc/book/connect/apps/index.md @@ -13,7 +13,7 @@ In this section, we cover the following web applications: | [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` | | [Pixelfed](#pixelfed) | ❓ | Not yet tested | | [Pleroma](#pleroma) | ❓ | Not yet tested | -| [Lemmy](#lemmy) | ❓ | Not yet tested | +| [Lemmy](#lemmy) | ✅ | Supported with pict-rs | | [Funkwhale](#funkwhale) | ❓ | Not yet tested | | [Misskey](#misskey) | ❓ | Not yet tested | | [Prismo](#prismo) | ❓ | Not yet tested | @@ -484,7 +484,68 @@ And add a new line. For example, to run it every 10 minutes: ## Lemmy -Lemmy uses pict-rs that [supports S3 backends](https://git.asonix.dog/asonix/pict-rs/commit/f9f4fc63d670f357c93f24147c2ee3e1278e2d97) +Lemmy uses pict-rs that [supports S3 backends](https://git.asonix.dog/asonix/pict-rs/commit/f9f4fc63d670f357c93f24147c2ee3e1278e2d97). +This feature requires `pict-rs >= 4.0.0`. + +### Creating your bucket + +This is the usual Garage setup: + +```bash +garage key new --name pictrs-key +garage bucket create pictrs-data +garage bucket allow pictrs-data --read --write --key pictrs-key +``` + +Note the Key ID and Secret Key. + +### Migrating your data + +If your pict-rs instance holds existing data, you first need to migrate to the S3 bucket. + +Stop pict-rs, then run the migration utility from local filesystem to the bucket: + +``` +pict-rs \ + filesystem -p /path/to/existing/files \ + object-store \ + -e my-garage-instance.mydomain.tld:3900 \ + -b pictrs-data \ + -r garage \ + -a GK... \ + -s abcdef0123456789... +``` + +This is pretty slow, so hold on while migrating. + +### Running pict-rs with an S3 backend + +Pict-rs supports both a configuration file and environment variables. + +Either set the following section in your `pict-rs.toml`: + +``` +[store] +type = 'object_storage' +endpoint = 'http://my-garage-instance.mydomain.tld:3900' +bucket_name = 'pictrs-data' +region = 'garage' +access_key = 'GK...' +secret_key = 'abcdef0123456789...' +``` + +... or set these environment variables: + + +``` +PICTRS__STORE__TYPE=object_storage +PICTRS__STORE__ENDPOINT=http:/my-garage-instance.mydomain.tld:3900 +PICTRS__STORE__BUCKET_NAME=pictrs-data +PICTRS__STORE__REGION=garage +PICTRS__STORE__ACCESS_KEY=GK... +PICTRS__STORE__SECRET_KEY=abcdef0123456789... +``` + ## Funkwhale -- cgit v1.2.3 From 97bb1102193e274208ed1db7dc2a8b9445959f94 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Fri, 13 Jan 2023 14:12:02 +0000 Subject: doc: Added observability.md. --- doc/book/connect/_index.md | 7 ++--- doc/book/connect/observability.md | 57 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 doc/book/connect/observability.md (limited to 'doc/book') diff --git a/doc/book/connect/_index.md b/doc/book/connect/_index.md index ca44ac17..93a2b87e 100644 --- a/doc/book/connect/_index.md +++ b/doc/book/connect/_index.md @@ -10,11 +10,12 @@ Garage implements the Amazon S3 protocol, which makes it compatible with many ex In particular, you will find here instructions to connect it with: - - [Browsing tools](@/documentation/connect/cli.md) - [Applications](@/documentation/connect/apps/index.md) - - [Website hosting](@/documentation/connect/websites.md) - - [Software repositories](@/documentation/connect/repositories.md) + - [Browsing tools](@/documentation/connect/cli.md) - [FUSE](@/documentation/connect/fs.md) + - [Observability](@/documentation/connect/observability.md) + - [Software repositories](@/documentation/connect/repositories.md) + - [Website hosting](@/documentation/connect/websites.md) ### Generic instructions diff --git a/doc/book/connect/observability.md b/doc/book/connect/observability.md new file mode 100644 index 00000000..c5037fa4 --- /dev/null +++ b/doc/book/connect/observability.md @@ -0,0 +1,57 @@ ++++ +title = "Observability" +weight = 25 ++++ + +An object store can be used as data storage location for metrics, and logs which +can then be leveraged for systems observability. + +## Metrics + +### Prometheus + +Prometheus itself has no object store capabilities, however two projects exist +which support storing metrics in an object store: + + - [Cortex](https://cortexmetrics.io/) + - [Thanos](https://thanos.io/) + +## System logs + +### Vector + +[Vector](https://vector.dev/) natively supports S3 as a +[data sink](https://vector.dev/docs/reference/configuration/sinks/aws_s3/) +(and [source](https://vector.dev/docs/reference/configuration/sources/aws_s3/)). + +This can be configured with Garage with the following: + +```bash +garage key new --name vector-system-logs +garage bucket create system-logs +garage bucket allow system-logs --read --write --key vector-system-logs +``` + +The `vector.toml` can then be configured as follows: + +```toml +[sources.journald] +type = "journald" +current_boot_only = true + +[sinks.out] +encoding.codec = "json" +type = "aws_s3" +inputs = [ "journald" ] +bucket = "system-logs" +key_prefix = "%F/" +compression = "none" +region = "garage" +endpoint = "https://my-garage-instance.mydomain.tld" +auth.access_key_id = "" +auth.secret_access_key = "" +``` + +This is an example configuration - please refer to the Vector documentation for +all configuration and transformation possibilities. Also note that Garage +performs its own compression, so this should be disabled in Vector. -- cgit v1.2.3 From f2492107d7858882adf386f8925829659755f1e5 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Wed, 25 Jan 2023 12:00:01 +0000 Subject: cookbook/real-world.md: Added note about mesh network options. --- doc/book/cookbook/real-world.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'doc/book') diff --git a/doc/book/cookbook/real-world.md b/doc/book/cookbook/real-world.md index 5423bbab..9be1ba44 100644 --- a/doc/book/cookbook/real-world.md +++ b/doc/book/cookbook/real-world.md @@ -19,8 +19,12 @@ To run a real-world deployment, make sure the following conditions are met: - You have at least three machines with sufficient storage space available. -- Each machine has a public IP address which is reachable by other machines. - Running behind a NAT is likely to be possible but hasn't been tested for the latest version (TODO). +- Each machine has a public IP address which is reachable by other machines. It + is highly recommended that you use IPv6 for this end-to-end connectivity. If + IPv6 is not available, then using a mesh VPN such as + [Nebula](https://github.com/slackhq/nebula) or + [Yggdrasil](https://yggdrasil-network.github.io/) are approaches to consider + in addition to building out your own VPN tunneling. - This guide will assume you are using Docker containers to deploy Garage on each node. Garage can also be run independently, for instance as a [Systemd service](@/documentation/cookbook/systemd.md). -- cgit v1.2.3