aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAlex <alex@adnab.me>2024-02-09 12:11:23 +0000
committerAlex <alex@adnab.me>2024-02-09 12:11:23 +0000
commit0c7ce001c907791518f93fd9d2db01eebc517c51 (patch)
tree08336bb7610c35b7707d3c87fd3f4eac7e2b21f7
parente4a43bfd592c149af8e3eac58ab317a518f0968a (diff)
parentf7ae966ed34d3ca8fb20afd2f20874dd42df7f7e (diff)
downloadgarage-0c7ce001c907791518f93fd9d2db01eebc517c51.tar.gz
garage-0c7ce001c907791518f93fd9d2db01eebc517c51.zip
Merge pull request 'Fix & simplify CI using Woodpecker' (#706) from nix-improvements into main
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/706
-rw-r--r--.drone.yml300
-rw-r--r--.woodpecker/debug.yaml4
-rw-r--r--.woodpecker/publish.yaml23
-rw-r--r--.woodpecker/release.yaml39
-rw-r--r--doc/book/design/goals.md2
-rw-r--r--doc/book/development/devenv.md2
-rw-r--r--doc/book/development/miscellaneous-notes.md8
-rw-r--r--doc/book/development/release-process.md55
-rw-r--r--flake.nix74
-rw-r--r--nix/common.nix1
-rw-r--r--nix/kaniko.nix24
-rw-r--r--nix/manifest-tool.nix24
-rw-r--r--nix/toolchain.nix11
-rw-r--r--shell.nix130
14 files changed, 150 insertions, 547 deletions
diff --git a/.drone.yml b/.drone.yml
deleted file mode 100644
index d79a9277..00000000
--- a/.drone.yml
+++ /dev/null
@@ -1,300 +0,0 @@
----
-kind: pipeline
-name: default
-
-node:
- nix-daemon: 1
-
-steps:
- - name: check formatting
- image: nixpkgs/nix:nixos-22.05
- commands:
- - nix-shell --attr rust --run "cargo fmt -- --check"
-
- - name: build
- image: nixpkgs/nix:nixos-22.05
- commands:
- - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
-
- - name: unit + func tests
- image: nixpkgs/nix:nixos-22.05
- environment:
- GARAGE_TEST_INTEGRATION_EXE: result-bin/bin/garage
- GARAGE_TEST_INTEGRATION_PATH: tmp-garage-integration
- commands:
- - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- - nix-build --no-build-output --attr test.amd64
- - ./result/bin/garage_db-*
- - ./result/bin/garage_api-*
- - ./result/bin/garage_model-*
- - ./result/bin/garage_rpc-*
- - ./result/bin/garage_table-*
- - ./result/bin/garage_util-*
- - ./result/bin/garage_web-*
- - ./result/bin/garage-*
- - ./result/bin/integration-* || (cat tmp-garage-integration/stderr.log; false)
- - rm result
- - rm -rv tmp-garage-integration
-
- - name: integration tests
- image: nixpkgs/nix:nixos-22.05
- commands:
- - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
-
-trigger:
- event:
- - custom
- - push
- - pull_request
- - tag
- - cron
-
----
-kind: pipeline
-type: docker
-name: release-linux-amd64
-
-node:
- nix-daemon: 1
-
-steps:
- - name: build
- image: nixpkgs/nix:nixos-22.05
- commands:
- - nix-build --no-build-output --attr pkgs.amd64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
-
- - name: integration tests
- image: nixpkgs/nix:nixos-22.05
- commands:
- - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
-
- - name: upgrade tests
- image: nixpkgs/nix:nixos-22.05
- commands:
- - nix-shell --attr integration --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
-
- - name: push static binary
- image: nixpkgs/nix:nixos-22.05
- environment:
- AWS_ACCESS_KEY_ID:
- from_secret: garagehq_aws_access_key_id
- AWS_SECRET_ACCESS_KEY:
- from_secret: garagehq_aws_secret_access_key
- TARGET: "x86_64-unknown-linux-musl"
- commands:
- - nix-shell --attr release --run "to_s3"
-
- - name: docker build and publish
- image: nixpkgs/nix:nixos-22.05
- environment:
- DOCKER_AUTH:
- from_secret: docker_auth
- DOCKER_PLATFORM: "linux/amd64"
- CONTAINER_NAME: "dxflrs/amd64_garage"
- HOME: "/kaniko"
- commands:
- - mkdir -p /kaniko/.docker
- - echo $DOCKER_AUTH > /kaniko/.docker/config.json
- - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- - nix-shell --attr release --run "to_docker"
-
-
-trigger:
- event:
- - promote
- - cron
-
----
-kind: pipeline
-type: docker
-name: release-linux-i386
-
-node:
- nix-daemon: 1
-
-steps:
- - name: build
- image: nixpkgs/nix:nixos-22.05
- commands:
- - nix-build --no-build-output --attr pkgs.i386.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
-
- - name: integration tests
- image: nixpkgs/nix:nixos-22.05
- commands:
- - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
-
- - name: upgrade tests
- image: nixpkgs/nix:nixos-22.05
- commands:
- - nix-shell --attr integration --run "./script/test-upgrade.sh v0.8.4 i686-unknown-linux-musl" || (cat /tmp/garage.log; false)
-
- - name: push static binary
- image: nixpkgs/nix:nixos-22.05
- environment:
- AWS_ACCESS_KEY_ID:
- from_secret: garagehq_aws_access_key_id
- AWS_SECRET_ACCESS_KEY:
- from_secret: garagehq_aws_secret_access_key
- TARGET: "i686-unknown-linux-musl"
- commands:
- - nix-shell --attr release --run "to_s3"
-
- - name: docker build and publish
- image: nixpkgs/nix:nixos-22.05
- environment:
- DOCKER_AUTH:
- from_secret: docker_auth
- DOCKER_PLATFORM: "linux/386"
- CONTAINER_NAME: "dxflrs/386_garage"
- HOME: "/kaniko"
- commands:
- - mkdir -p /kaniko/.docker
- - echo $DOCKER_AUTH > /kaniko/.docker/config.json
- - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- - nix-shell --attr release --run "to_docker"
-
-trigger:
- event:
- - promote
- - cron
-
----
-kind: pipeline
-type: docker
-name: release-linux-arm64
-
-node:
- nix-daemon: 1
-
-steps:
- - name: build
- image: nixpkgs/nix:nixos-22.05
- commands:
- - nix-build --no-build-output --attr pkgs.arm64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
-
- - name: push static binary
- image: nixpkgs/nix:nixos-22.05
- environment:
- AWS_ACCESS_KEY_ID:
- from_secret: garagehq_aws_access_key_id
- AWS_SECRET_ACCESS_KEY:
- from_secret: garagehq_aws_secret_access_key
- TARGET: "aarch64-unknown-linux-musl"
- commands:
- - nix-shell --attr release --run "to_s3"
-
- - name: docker build and publish
- image: nixpkgs/nix:nixos-22.05
- environment:
- DOCKER_AUTH:
- from_secret: docker_auth
- DOCKER_PLATFORM: "linux/arm64"
- CONTAINER_NAME: "dxflrs/arm64_garage"
- HOME: "/kaniko"
- commands:
- - mkdir -p /kaniko/.docker
- - echo $DOCKER_AUTH > /kaniko/.docker/config.json
- - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- - nix-shell --attr release --run "to_docker"
-
-trigger:
- event:
- - promote
- - cron
-
----
-kind: pipeline
-type: docker
-name: release-linux-arm
-
-node:
- nix-daemon: 1
-
-steps:
- - name: build
- image: nixpkgs/nix:nixos-22.05
- commands:
- - nix-build --no-build-output --attr pkgs.arm.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
-
- - name: push static binary
- image: nixpkgs/nix:nixos-22.05
- environment:
- AWS_ACCESS_KEY_ID:
- from_secret: garagehq_aws_access_key_id
- AWS_SECRET_ACCESS_KEY:
- from_secret: garagehq_aws_secret_access_key
- TARGET: "armv6l-unknown-linux-musleabihf"
- commands:
- - nix-shell --attr release --run "to_s3"
-
- - name: docker build and publish
- image: nixpkgs/nix:nixos-22.05
- environment:
- DOCKER_AUTH:
- from_secret: docker_auth
- DOCKER_PLATFORM: "linux/arm"
- CONTAINER_NAME: "dxflrs/arm_garage"
- HOME: "/kaniko"
- commands:
- - mkdir -p /kaniko/.docker
- - echo $DOCKER_AUTH > /kaniko/.docker/config.json
- - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- - nix-shell --attr release --run "to_docker"
-
-trigger:
- event:
- - promote
- - cron
-
----
-kind: pipeline
-type: docker
-name: refresh-release-page
-
-node:
- nix-daemon: 1
-
-steps:
- - name: multiarch-docker
- image: nixpkgs/nix:nixos-22.05
- environment:
- DOCKER_AUTH:
- from_secret: docker_auth
- HOME: "/root"
- commands:
- - mkdir -p /root/.docker
- - echo $DOCKER_AUTH > /root/.docker/config.json
- - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- - nix-shell --attr release --run "multiarch_docker"
- - name: refresh-index
- image: nixpkgs/nix:nixos-22.05
- environment:
- AWS_ACCESS_KEY_ID:
- from_secret: garagehq_aws_access_key_id
- AWS_SECRET_ACCESS_KEY:
- from_secret: garagehq_aws_secret_access_key
- commands:
- - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
- - nix-shell --attr release --run "refresh_index"
-
-depends_on:
- - release-linux-amd64
- - release-linux-i386
- - release-linux-arm64
- - release-linux-arm
-
-trigger:
- event:
- - promote
- - cron
-
----
-kind: signature
-hmac: 0c4b57eb4b27b7c6a6ff21ab87f0767fe3eb90f5d95d5cbcdccf794e9d2a5d86
-
-...
diff --git a/.woodpecker/debug.yaml b/.woodpecker/debug.yaml
index 9486f672..59ae8c06 100644
--- a/.woodpecker/debug.yaml
+++ b/.woodpecker/debug.yaml
@@ -10,7 +10,7 @@ steps:
- name: check formatting
image: nixpkgs/nix:nixos-22.05
commands:
- - nix-shell --attr rust --run "cargo fmt -- --check"
+ - nix-shell --attr devShell --run "cargo fmt -- --check"
- name: build
image: nixpkgs/nix:nixos-22.05
@@ -41,4 +41,4 @@ steps:
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
- - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+ - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
diff --git a/.woodpecker/publish.yaml b/.woodpecker/publish.yaml
index b45d379c..739af2c4 100644
--- a/.woodpecker/publish.yaml
+++ b/.woodpecker/publish.yaml
@@ -16,17 +16,14 @@ steps:
target: AWS_SECRET_ACCESS_KEY
commands:
- mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
- - nix-shell --attr release --run "refresh_index"
+ - nix-shell --attr ci --run "refresh_index"
-# - name: multiarch-docker
-# image: nixpkgs/nix:nixos-22.05
-# environment:
-# HOME: "/root"
-# secrets:
-# - docker_auth
-# commands:
-# - mkdir -p /root/.docker
-# - echo $DOCKER_AUTH > /root/.docker/config.json
-# - sha512sum /root/.docker/config.json
-# - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
-# - nix-shell --attr release --run "multiarch_docker"
+ - name: multiarch-docker
+ image: nixpkgs/nix:nixos-22.05
+ secrets:
+ - docker_auth
+ commands:
+ - mkdir -p /root/.docker
+ - echo $DOCKER_AUTH > /root/.docker/config.json
+ - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
+ - nix-shell --attr ci --run "multiarch_docker"
diff --git a/.woodpecker/release.yaml b/.woodpecker/release.yaml
index 8580441c..c57d7272 100644
--- a/.woodpecker/release.yaml
+++ b/.woodpecker/release.yaml
@@ -19,12 +19,17 @@ steps:
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build --no-build-output --attr pkgs.${ARCH}.release --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
- - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
+
+ - name: check is static binary
+ image: nixpkgs/nix:nixos-22.05
+ commands:
+ - nix-build --no-build-output --attr pkgs.${ARCH}.release --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
+ - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
- name: integration tests
image: nixpkgs/nix:nixos-22.05
commands:
- - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+ - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
when:
- matrix:
ARCH: amd64
@@ -34,7 +39,7 @@ steps:
- name: upgrade tests
image: nixpkgs/nix:nixos-22.05
commands:
- - nix-shell --attr integration --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
+ - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
when:
- matrix:
ARCH: amd64
@@ -49,19 +54,17 @@ steps:
- source: garagehq_aws_secret_access_key
target: AWS_SECRET_ACCESS_KEY
commands:
- - nix-shell --attr release --run "to_s3_woodpecker"
+ - nix-shell --attr ci --run "to_s3"
-# - name: docker build and publish
-# image: nixpkgs/nix:nixos-22.05
-# environment:
-# DOCKER_PLATFORM: "linux/${ARCH}"
-# CONTAINER_NAME: "dxflrs/${ARCH}_garage"
-# HOME: "/kaniko"
-# secrets:
-# - docker_auth
-# commands:
-# - mkdir -p /kaniko/.docker
-# - echo $DOCKER_AUTH > /kaniko/.docker/config.json
-# - sha512sum /kaniko/.docker/config.json
-# - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
-# - nix-shell --attr release --run "to_docker"
+ - name: docker build and publish
+ image: nixpkgs/nix:nixos-22.05
+ environment:
+ DOCKER_PLATFORM: "linux/${ARCH}"
+ CONTAINER_NAME: "dxflrs/${ARCH}_garage"
+ secrets:
+ - docker_auth
+ commands:
+ - mkdir -p /root/.docker
+ - echo $DOCKER_AUTH > /root/.docker/config.json
+ - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
+ - nix-shell --attr ci --run "to_docker"
diff --git a/doc/book/design/goals.md b/doc/book/design/goals.md
index 78ac7978..4efb6349 100644
--- a/doc/book/design/goals.md
+++ b/doc/book/design/goals.md
@@ -48,7 +48,5 @@ locations. They use Garage themselves for the following tasks:
- As a backup target using `rclone` and `restic`
-- In the Drone continuous integration platform to store task logs
-
The Deuxfleurs Garage cluster is a multi-site cluster currently composed of
9 nodes in 3 physical locations.
diff --git a/doc/book/development/devenv.md b/doc/book/development/devenv.md
index dd3bdec0..88f8ba06 100644
--- a/doc/book/development/devenv.md
+++ b/doc/book/development/devenv.md
@@ -80,7 +80,7 @@ nix-build \
--git_version $(git rev-parse HEAD)
```
-*The result is located in `result/bin`. You can pass arguments to cross compile: check `.drone.yml` for examples.*
+*The result is located in `result/bin`. You can pass arguments to cross compile: check `.woodpecker/release.yml` for examples.*
If you modify a `Cargo.toml` or regenerate any `Cargo.lock`, you must run `cargo2nix`:
diff --git a/doc/book/development/miscellaneous-notes.md b/doc/book/development/miscellaneous-notes.md
index f0083ae5..a421943f 100644
--- a/doc/book/development/miscellaneous-notes.md
+++ b/doc/book/development/miscellaneous-notes.md
@@ -81,12 +81,9 @@ Our cache will be checked.
- http://www.lpenz.org/articles/nixchannel/index.html
-## Drone
+## Woodpecker
-Do not try to set a build as trusted from the interface or the CLI tool,
-your request would be ignored. Instead, directly edit the database (table `repos`, column `repo_trusted`).
-
-Drone can do parallelism both at the step and the pipeline level. At the step level, parallelism is restricted to the same runner.
+Woodpecker can do parallelism both at the step and the pipeline level. At the step level, parallelism is restricted to the same runner.
## Building Docker containers
@@ -99,3 +96,4 @@ We were:
- Unable to use the kaniko container provided by Google as we can't run arbitrary logic: we need to put our secret in .docker/config.json.
Finally we chose to build kaniko through nix and use it in a `nix-shell`.
+We then switched to using kaniko from nixpkgs when it was packaged.
diff --git a/doc/book/development/release-process.md b/doc/book/development/release-process.md
index 3fed4add..0c6701c0 100644
--- a/doc/book/development/release-process.md
+++ b/doc/book/development/release-process.md
@@ -42,7 +42,7 @@ and the docker containers on Docker Hub.
## Automation
-We automated our release process with Nix and Drone to make it more reliable.
+We automated our release process with Nix and Woodpecker to make it more reliable.
Here we describe how we have done in case you want to debug or improve it.
### Caching build steps
@@ -62,52 +62,31 @@ Sending to the cache is done through `nix copy`, for example:
nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' result
```
-*Note that you need the signing key. In our case, it is stored as a secret in Drone.*
+*The signing key possessed by the Garage maintainers is required to update the Nix cache.*
-The previous command will only send the built packet and not its dependencies.
-To send its dependency, a tool named `nix-copy-closure` has been created but it is not compatible with the S3 protocol.
-
-Instead, you can use the following commands to list all the runtime dependencies:
+The previous command will only send the built package and not its dependencies.
+In the case of our CI pipeline, we want to cache all intermediate build steps
+as well. This can be done using this quite involved command (here as an example
+for the `pkgs.amd64.relase` package):
```bash
-nix copy \
- --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
- $(nix-store -qR result/)
+nix copy -j8 \
+ --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/nix-signing-key.sec' \
+ $(nix path-info pkgs.amd64.release --file default.nix --derivation --recursive | sed 's/\.drv$/.drv^*/')
```
-*We could also write this expression with xargs but this tool is not available in our container.*
-
-But in certain cases, we want to cache compile time dependencies also.
-For example, the Nix project does not provide binaries for cross compiling to i686 and thus we need to compile gcc on our own.
-We do not want to compile gcc each time, so even if it is a compile time dependency, we want to cache it.
+This command will simultaneously build all of the required Nix paths (using at
+most 8 parallel Nix builder jobs) and send the resulting objects to the cache.
-This time, the command is a bit more involved:
-
-```bash
-nix copy --to \
- 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
- $(nix-store -qR --include-outputs \
- $(nix-instantiate))
-```
-
-This is the command we use in our CI as we expect the final binary to change, so we mainly focus on
-caching our development dependencies.
-
-*Currently there is no automatic garbage collection of the cache: we should monitor its growth.
-Hopefully, we can erase it totally without breaking any build, the next build will only be slower.*
-
-In practise, we concluded that we do not want to cache all the compilation dependencies.
-Instead, we want to cache the toolchain we use to build Garage each time we change it.
-So we removed from Drone any automatic update of the cache and instead handle them manually with:
+This can be run for all the Garage packages we build using the following command:
```
source ~/.awsrc
-nix-shell --run 'refresh_toolchain'
+nix-shell --attr cache --run 'refresh_cache'
```
-Internally, it will run `nix-build` on `nix/toolchain.nix` and send the output plus its depedencies to the cache.
-
-To erase the cache:
+We don't automate this step at each CI build, as *there is currently no automatic garbage collection of the cache.*
+This means we should also monitor the cache's size; if it ever becomes too big we can erase it with:
```
mc rm --recursive --force 'garage/nix/'
@@ -157,9 +136,9 @@ nix-shell --run refresh_index
If you want to compile for different architectures, you will need to repeat all these commands for each architecture.
-**In practise, and except for debugging, you will never directly run these commands. Release is handled by drone**
+**In practice, and except for debugging, you will never directly run these commands. Release is handled by Woodpecker.**
-### Drone
+### Drone (obsolete)
Our instance is available at [https://drone.deuxfleurs.fr](https://drone.deuxfleurs.fr).
You need an account on [https://git.deuxfleurs.fr](https://git.deuxfleurs.fr) to use it.
diff --git a/flake.nix b/flake.nix
index 236b2310..06c74220 100644
--- a/flake.nix
+++ b/flake.nix
@@ -33,27 +33,57 @@
compile = import ./nix/compile.nix;
in
flake-utils.lib.eachDefaultSystem (system:
- let pkgs = nixpkgs.legacyPackages.${system};
- in {
- packages = {
- default = (compile {
- inherit system git_version;
- pkgsSrc = nixpkgs;
- cargo2nixOverlay = cargo2nix.overlays.default;
- release = true;
- }).workspace.garage { compileMode = "build"; };
- };
- devShell = (compile {
- inherit system git_version;
- pkgsSrc = nixpkgs;
- cargo2nixOverlay = cargo2nix.overlays.default;
- release = false;
- }).workspaceShell { packages = with pkgs; [
- cargo-audit
- cargo-outdated
- rustfmt
- clang
- mold
- ]; };
+ let
+ pkgs = nixpkgs.legacyPackages.${system};
+ in
+ {
+ packages =
+ let
+ packageFor = target: (compile {
+ inherit system git_version target;
+ pkgsSrc = nixpkgs;
+ cargo2nixOverlay = cargo2nix.overlays.default;
+ release = true;
+ }).workspace.garage { compileMode = "build"; };
+ in
+ {
+ # default = native release build
+ default = packageFor null;
+ # other = cross-compiled, statically-linked builds
+ amd64 = packageFor "x86_64-unknown-linux-musl";
+ i386 = packageFor "i686-unknown-linux-musl";
+ arm64 = packageFor "aarch64-unknown-linux-musl";
+ arm = packageFor "armv6l-unknown-linux-musl";
+ };
+
+ # ---- developpment shell, for making native builds only ----
+ devShells =
+ let
+ shellWithPackages = (packages: (compile {
+ inherit system git_version;
+ pkgsSrc = nixpkgs;
+ cargo2nixOverlay = cargo2nix.overlays.default;
+ }).workspaceShell { inherit packages; });
+ in
+ {
+ default = shellWithPackages
+ (with pkgs; [
+ rustfmt
+ clang
+ mold
+ ]);
+
+ # import the full shell using `nix develop .#full`
+ full = shellWithPackages (with pkgs; [
+ rustfmt
+ clang
+ mold
+ # ---- extra packages for dev tasks ----
+ cargo-audit
+ cargo-outdated
+ cargo-machete
+ nixpkgs-fmt
+ ]);
+ };
});
}
diff --git a/nix/common.nix b/nix/common.nix
index 323c3882..1ad809bb 100644
--- a/nix/common.nix
+++ b/nix/common.nix
@@ -14,4 +14,5 @@ rec {
pkgsSrc = flake.defaultNix.inputs.nixpkgs;
cargo2nix = flake.defaultNix.inputs.cargo2nix;
cargo2nixOverlay = cargo2nix.overlays.default;
+ devShells = builtins.getAttr builtins.currentSystem flake.defaultNix.devShells;
}
diff --git a/nix/kaniko.nix b/nix/kaniko.nix
deleted file mode 100644
index 8380fa2f..00000000
--- a/nix/kaniko.nix
+++ /dev/null
@@ -1,24 +0,0 @@
-pkgs:
-pkgs.buildGoModule rec {
- pname = "kaniko";
- version = "1.9.2";
-
- src = pkgs.fetchFromGitHub {
- owner = "GoogleContainerTools";
- repo = "kaniko";
- rev = "v${version}";
- sha256 = "dXQ0/o1qISv+sjNVIpfF85bkbM9sGOGwqVbWZpMWfMY=";
- };
-
- vendorSha256 = null;
-
- checkPhase = "true";
-
- meta = with pkgs.lib; {
- description =
- "kaniko is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster.";
- homepage = "https://github.com/GoogleContainerTools/kaniko";
- license = licenses.asl20;
- platforms = platforms.linux;
- };
-}
diff --git a/nix/manifest-tool.nix b/nix/manifest-tool.nix
deleted file mode 100644
index 1090a6ef..00000000
--- a/nix/manifest-tool.nix
+++ /dev/null
@@ -1,24 +0,0 @@
-pkgs:
-pkgs.buildGoModule rec {
- pname = "manifest-tool";
- version = "2.0.5";
-
- src = pkgs.fetchFromGitHub {
- owner = "estesp";
- repo = "manifest-tool";
- rev = "v${version}";
- sha256 = "hjCGKnE0yrlnF/VIzOwcDzmQX3Wft+21KCny/opqdLg=";
- } + "/v2";
-
- vendorSha256 = null;
-
- checkPhase = "true";
-
- meta = with pkgs.lib; {
- description =
- "Command line tool to create and query container image manifest list/indexes";
- homepage = "https://github.com/estesp/manifest-tool";
- license = licenses.asl20;
- platforms = platforms.linux;
- };
-}
diff --git a/nix/toolchain.nix b/nix/toolchain.nix
deleted file mode 100644
index c4fef9ab..00000000
--- a/nix/toolchain.nix
+++ /dev/null
@@ -1,11 +0,0 @@
-{ system ? builtins.currentSystem, }:
-
-with import ./common.nix;
-
-let
- pkgsHost = import pkgsSrc { };
- kaniko = (import ./kaniko.nix) pkgsHost;
- winscp = (import ./winscp.nix) pkgsHost;
- manifestTool = (import ./manifest-tool.nix) pkgsHost;
-in [ kaniko winscp manifestTool ]
-
diff --git a/shell.nix b/shell.nix
index 6d6cc68f..0fa02b66 100644
--- a/shell.nix
+++ b/shell.nix
@@ -5,110 +5,48 @@ with import ./nix/common.nix;
let
pkgs = import pkgsSrc {
inherit system;
- overlays = [ cargo2nixOverlay ];
};
- kaniko = (import ./nix/kaniko.nix) pkgs;
- manifest-tool = (import ./nix/manifest-tool.nix) pkgs;
winscp = (import ./nix/winscp.nix) pkgs;
-
-in {
- # --- Rust Shell ---
- # Use it to compile Garage
- rust = pkgs.mkShell {
+in
+{
+ # --- Dev shell inherited from flake.nix ---
+ devShell = devShells.default;
+
+ # --- Continuous integration shell ---
+ # The shell used for all CI jobs (along with devShell)
+ ci = pkgs.mkShell {
nativeBuildInputs = with pkgs; [
- #rustPlatform.rust.rustc
- rustPlatform.rust.cargo
- clang
- mold
- #clippy
- rustfmt
- #perl
- #protobuf
- #pkg-config
- #openssl
- file
- #cargo2nix.packages.x86_64-linux.cargo2nix
- ];
- };
-
- # --- Integration shell ---
- # Use it to test Garage with common S3 clients
- integration = pkgs.mkShell {
- nativeBuildInputs = [
winscp
- pkgs.s3cmd
- pkgs.awscli2
- pkgs.minio-client
- pkgs.rclone
- pkgs.socat
- pkgs.psmisc
- pkgs.which
- pkgs.openssl
- pkgs.curl
- pkgs.jq
- ];
- };
- # --- Release shell ---
- # A shell built to make releasing easier
- release = pkgs.mkShell {
+ kaniko
+ manifest-tool
+ awscli2
+ file
+ s3cmd
+ minio-client
+ rclone
+ socat
+ psmisc
+ which
+ openssl
+ curl
+ jq
+ ];
shellHook = ''
- function refresh_toolchain {
- pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
- nix copy \
- --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
- $(nix-store -qR \
- $(nix-build --no-build-output --no-out-link nix/toolchain.nix))
- rm /tmp/nix-signing-key.sec
- }
-
- function refresh_cache {
- pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
- for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do
- echo "Updating cache for ''${attr}"
- derivation=$(nix-instantiate --attr ''${attr})
- nix copy -j8 \
- --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
- $(nix-store -qR ''${derivation%\!bin})
- done
- rm /tmp/nix-signing-key.sec
- }
-
- function refresh_flake_cache {
- pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
- for attr in packages.x86_64-linux.default devShell.x86_64-linux; do
- echo "Updating cache for ''${attr}"
- derivation=$(nix path-info --derivation ".#''${attr}")
- nix copy -j8 \
- --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
- $(nix-store -qR ''${derivation})
- done
- rm /tmp/nix-signing-key.sec
- }
-
function to_s3 {
aws \
--endpoint-url https://garage.deuxfleurs.fr \
--region garage \
s3 cp \
./result-bin/bin/garage \
- s3://garagehq.deuxfleurs.fr/_releases/''${DRONE_TAG:-$DRONE_COMMIT}/''${TARGET}/garage
- }
-
- function to_s3_woodpecker {
- aws \
- --endpoint-url https://garage.deuxfleurs.fr \
- --region garage \
- s3 cp \
- ./result-bin/bin/garage \
s3://garagehq.deuxfleurs.fr/_releases/''${CI_COMMIT_TAG:-$CI_COMMIT_SHA}/''${TARGET}/garage
}
function to_docker {
executor \
--force \
- --customPlatform="''${DOCKER_PLATFORM}" \
- --destination "''${CONTAINER_NAME}:''${CONTAINER_TAG}" \
+ --customPlatform="$(echo "''${DOCKER_PLATFORM}" | sed 's/i386/386/')" \
+ --destination "$(echo "''${CONTAINER_NAME}" | sed 's/i386/386/'):''${CONTAINER_TAG}" \
--context dir://`pwd` \
--verbosity=debug
}
@@ -167,7 +105,25 @@ in {
s3://garagehq.deuxfleurs.fr/
}
'';
- nativeBuildInputs = [ pkgs.awscli2 kaniko manifest-tool ];
+
+ };
+
+ # --- Cache shell ---
+ # A shell for refreshing caches
+ cache = pkgs.mkShell {
+ shellHook = ''
+ function refresh_cache {
+ pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
+ for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.release; do
+ echo "Updating cache for ''${attr}"
+ nix copy -j8 \
+ --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
+ $(nix path-info ''${attr} --file default.nix --derivation --recursive | sed 's/\.drv$/.drv^*/')
+
+ done
+ rm /tmp/nix-signing-key.sec
+ }
+ '';
};
}