1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
(ns jepsen.garage.nemesis
(:require [clojure.tools.logging :refer :all]
[jepsen [control :as c]
[core :as jepsen]
[generator :as gen]
[nemesis :as nemesis]]
[jepsen.garage.daemon :as grg]
[jepsen.control.util :as cu]))
; ---- reconfiguration nemesis ----
(defn configure-present!
"Configure node to be active in new cluster layout"
[test node]
(info "configure-present!" node)
(let [node-id (c/on node (c/exec grg/binary :node :id :-q))]
(c/on
(jepsen/primary test)
(c/exec grg/binary :layout :assign (subs node-id 0 16) :-c :1G))))
(defn configure-absent!
"Configure node to be active in new cluster layout"
[test node]
(info "configure-absent!" node)
(let [node-id (c/on node (c/exec grg/binary :node :id :-q))]
(c/on
(jepsen/primary test)
(c/exec grg/binary :layout :assign (subs node-id 0 16) :-g))))
(defn finalize-config!
"Apply the proposed cluster layout"
[test]
(let [layout-show (c/on (jepsen/primary test) (c/exec grg/binary :layout :show))
[_ layout-next-version] (re-find #"apply --version (\d+)\n" layout-show)]
(info "layout show: " layout-show "; next-version: " layout-next-version)
(c/on (jepsen/primary test)
(c/exec grg/binary :layout :apply :--version layout-next-version))))
(defn reconfigure-subset
"Reconfigure cluster with only a subset of nodes"
[cnt]
(reify nemesis/Nemesis
(setup! [this test] this)
(invoke! [this test op] op
(case (:f op)
:start
(let [[keep-nodes remove-nodes]
(->> (:nodes test)
shuffle
(split-at cnt))]
(info "layout split: keep " keep-nodes ", remove " remove-nodes)
(run! #(configure-present! test %) keep-nodes)
(run! #(configure-absent! test %) remove-nodes)
(finalize-config! test)
(assoc op :value keep-nodes))
:stop
(do
(info "layout un-split: all nodes=" (:nodes test))
(run! #(configure-present! test %) (:nodes test))
(finalize-config! test)
(assoc op :value (:nodes test)))))
(teardown! [this test] this)))
; ---- nemesis scenari ----
(defn scenario-c
"Clock scramble scenario"
[opts]
{:generator (cycle [(gen/sleep 5)
{:type :info, :f :clock-scramble}])
:nemesis (nemesis/compose
{{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})})
(defn scenario-cp
"Clock scramble + partition scenario"
[opts]
{:generator (cycle [(gen/sleep 5)
{:type :info, :f :partition-start}
(gen/sleep 5)
{:type :info, :f :clock-scramble}
(gen/sleep 5)
{:type :info, :f :partition-stop}
(gen/sleep 5)
{:type :info, :f :clock-scramble}])
:final-generator (gen/once {:type :info, :f :partition-stop})
:nemesis (nemesis/compose
{{:partition-start :start
:partition-stop :stop} (nemesis/partition-random-halves)
{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})})
(defn scenario-r
"Cluster reconfiguration scenario"
[opts]
{:generator (cycle [(gen/sleep 5)
{:type :info, :f :reconfigure-start}
(gen/sleep 5)
{:type :info, :f :reconfigure-start}
(gen/sleep 5)
{:type :info, :f :reconfigure-stop}])
:final-generator (gen/once {:type :info, :f :reconfigure-stop})
:nemesis (nemesis/compose
{{:reconfigure-start :start
:reconfigure-stop :stop} (reconfigure-subset 3)})})
(defn scenario-pr
"Partition + cluster reconfiguration scenario"
[opts]
{:generator (cycle [(gen/sleep 3)
{:type :info, :f :reconfigure-start}
(gen/sleep 3)
{:type :info, :f :partition-start}
(gen/sleep 3)
{:type :info, :f :reconfigure-start}
(gen/sleep 3)
{:type :info, :f :partition-stop}
(gen/sleep 3)
{:type :info, :f :reconfigure-stop}])
:final-generator (gen/once {:type :info, :f :partition-stop})
:nemesis (nemesis/compose
{{:partition-start :start
:partition-stop :stop} (nemesis/partition-random-halves)
{:reconfigure-start :start
:reconfigure-stop :stop} (reconfigure-subset 3)})})
|
