diff options
Diffstat (limited to 'script')
| -rwxr-xr-x | script/dev-bucket.sh | 17 | ||||
| -rwxr-xr-x | script/dev-cluster.sh | 20 | ||||
| -rwxr-xr-x | script/dev-configure.sh | 33 | ||||
| -rw-r--r-- | script/test-renumbering.sh | 138 | ||||
| -rw-r--r-- | script/test-skip-part.sh | 103 | ||||
| -rwxr-xr-x | script/test-smoke.sh | 36 | ||||
| -rwxr-xr-x | script/test-upgrade.sh | 75 |
7 files changed, 403 insertions, 19 deletions
diff --git a/script/dev-bucket.sh b/script/dev-bucket.sh index 9bcf5bda..708c2c43 100755 --- a/script/dev-bucket.sh +++ b/script/dev-bucket.sh @@ -9,11 +9,22 @@ GARAGE_RELEASE="${REPO_FOLDER}/target/release/" NIX_RELEASE="${REPO_FOLDER}/result/bin/" PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH" -garage -c /tmp/config.1.toml bucket create eprouvette -KEY_INFO=$(garage -c /tmp/config.1.toml key new --name opérateur) +if [ -z "$GARAGE_BIN" ]; then + GARAGE_BIN=$(which garage || exit 1) + echo -en "Found garage at: ${GARAGE_BIN}\n" +else + echo -en "Using garage binary at: ${GARAGE_BIN}\n" +fi + +$GARAGE_BIN -c /tmp/config.1.toml bucket create eprouvette +if [ "$GARAGE_08" = "1" ]; then + KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml key new --name opérateur) +else + KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml key create opérateur) +fi ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'` SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'` -garage -c /tmp/config.1.toml bucket allow eprouvette --read --write --owner --key $ACCESS_KEY +$GARAGE_BIN -c /tmp/config.1.toml bucket allow eprouvette --read --write --owner --key $ACCESS_KEY echo "$ACCESS_KEY $SECRET_KEY" > /tmp/garage.s3 echo "Bucket s3://eprouvette created. Credentials stored in /tmp/garage.s3." diff --git a/script/dev-cluster.sh b/script/dev-cluster.sh index c7fbe08d..6b39255a 100755 --- a/script/dev-cluster.sh +++ b/script/dev-cluster.sh @@ -11,11 +11,16 @@ PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH" FANCYCOLORS=("41m" "42m" "44m" "45m" "100m" "104m") export RUST_BACKTRACE=1 -export RUST_LOG=garage=info,garage_api=debug,netapp=trace +export RUST_LOG=garage=info,garage_api=debug MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m" -WHICH_GARAGE=$(which garage || exit 1) -echo -en "${MAIN_LABEL} Found garage at: ${WHICH_GARAGE}\n" +if [ -z "$GARAGE_BIN" ]; then + GARAGE_BIN=$(which garage || exit 1) + echo -en "${MAIN_LABEL} Found garage at: ${GARAGE_BIN}\n" +else + echo -en "${MAIN_LABEL} Using garage binary at: ${GARAGE_BIN}\n" +fi +$GARAGE_BIN --version NETWORK_SECRET="$(openssl rand -hex 32)" @@ -28,6 +33,7 @@ LABEL="\e[${FANCYCOLORS[$count]}[$count]\e[49m" cat > $CONF_PATH <<EOF block_size = 1048576 # objects are split in blocks of maximum this number of bytes metadata_dir = "/tmp/garage-meta-$count" +db_engine = "lmdb" data_dir = "/tmp/garage-data-$count" rpc_bind_addr = "0.0.0.0:$((3900+$count))" # the port other Garage nodes will use to talk to this node rpc_public_addr = "127.0.0.1:$((3900+$count))" @@ -51,7 +57,7 @@ EOF echo -en "$LABEL configuration written to $CONF_PATH\n" -(garage -c /tmp/config.$count.toml server 2>&1|while read r; do echo -en "$LABEL $r\n"; done) & +($GARAGE_BIN -c /tmp/config.$count.toml server 2>&1|while read r; do echo -en "$LABEL $r\n"; done) & done # >>>>>>>>>>>>>>>> END FOR LOOP ON NODES @@ -73,14 +79,14 @@ fi sleep 3 # Establish connections between nodes for count in $(seq 1 3); do - NODE=$(garage -c /tmp/config.$count.toml node id -q) + NODE=$($GARAGE_BIN -c /tmp/config.$count.toml node id -q) for count2 in $(seq 1 3); do - garage -c /tmp/config.$count2.toml node connect $NODE + $GARAGE_BIN -c /tmp/config.$count2.toml node connect $NODE done done RETRY=120 -until garage -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do +until $GARAGE_BIN -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do (( RETRY-- )) if (( RETRY <= 0 )); then echo -en "${MAIN_LABEL} Garage did not start" diff --git a/script/dev-configure.sh b/script/dev-configure.sh index f0a7843d..0649cdbe 100755 --- a/script/dev-configure.sh +++ b/script/dev-configure.sh @@ -9,9 +9,17 @@ GARAGE_RELEASE="${REPO_FOLDER}/target/release/" NIX_RELEASE="${REPO_FOLDER}/result/bin/" PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH" +if [ -z "$GARAGE_BIN" ]; then + GARAGE_BIN=$(which garage || exit 1) + echo -en "Found garage at: ${GARAGE_BIN}\n" +else + echo -en "Using garage binary at: ${GARAGE_BIN}\n" +fi +$GARAGE_BIN --version + sleep 5 RETRY=120 -until garage -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do +until $GARAGE_BIN -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do (( RETRY-- )) if (( RETRY <= 0 )); then echo "garage did not start in time, failing." @@ -21,11 +29,20 @@ until garage -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do sleep 1 done -garage -c /tmp/config.1.toml status \ - | grep 'NO ROLE' \ - | grep -Po '^[0-9a-f]+' \ - | while read id; do - garage -c /tmp/config.1.toml layout assign $id -z dc1 -c 1 - done +if [ "$GARAGE_08" = "1" ]; then + $GARAGE_BIN -c /tmp/config.1.toml status \ + | grep 'NO ROLE' \ + | grep -Po '^[0-9a-f]+' \ + | while read id; do + $GARAGE_BIN -c /tmp/config.1.toml layout assign $id -z dc1 -c 1 + done +else + $GARAGE_BIN -c /tmp/config.1.toml status \ + | grep 'NO ROLE' \ + | grep -Po '^[0-9a-f]+' \ + | while read id; do + $GARAGE_BIN -c /tmp/config.1.toml layout assign $id -z dc1 -c 1G + done +fi -garage -c /tmp/config.1.toml layout apply --version 1 +$GARAGE_BIN -c /tmp/config.1.toml layout apply --version 1 diff --git a/script/test-renumbering.sh b/script/test-renumbering.sh new file mode 100644 index 00000000..0b1bd320 --- /dev/null +++ b/script/test-renumbering.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +: ' + This script tests part renumbering on an S3 remote (here configured for Minio). + + On Minio: + + The results confirm that if I upload parts with number 1, 4, 5 and 6, + they are renumbered to 1, 2, 3 and 4 after CompleteMultipartUpload. + Thus, specifying partNumber=4 on a GetObject/HeadObject should return + information on the part I originally uploaded with part number + + On S3: not tested + + Sample output (on Minio): + + f07e1404cc527d494242824ded3a616b part1 + 78974cd4d0f622eb3426ea7cd22f5a1c part4 + f9cc379f8baa61645558d9ba7e6351fa part5 + 1bd2383eebbac1f8e7143575ba5b1f4a part6 + Upload ID: 6838b813-d0ca-400b-9d28-ec8b2b5cd004 + PART 1 ETag: "f07e1404cc527d494242824ded3a616b" + PART 4 ETag: "78974cd4d0f622eb3426ea7cd22f5a1c" + PART 5 ETag: "f9cc379f8baa61645558d9ba7e6351fa" + PART 6 ETag: "1bd2383eebbac1f8e7143575ba5b1f4a" + ======================================== LIST ==== + { + "Parts": [ + { + "PartNumber": 1, + "LastModified": "2023-04-25T10:21:54.350000+00:00", + "ETag": "\"f07e1404cc527d494242824ded3a616b\"", + "Size": 20971520 + }, + { + "PartNumber": 4, + "LastModified": "2023-04-25T10:21:54.350000+00:00", + "ETag": "\"78974cd4d0f622eb3426ea7cd22f5a1c\"", + "Size": 20971520 + }, + { + "PartNumber": 5, + "LastModified": "2023-04-25T10:21:54.350000+00:00", + "ETag": "\"f9cc379f8baa61645558d9ba7e6351fa\"", + "Size": 20971520 + }, + { + "PartNumber": 6, + "LastModified": "2023-04-25T10:21:54.350000+00:00", + "ETag": "\"1bd2383eebbac1f8e7143575ba5b1f4a\"", + "Size": 20971520 + } + ], + "ChecksumAlgorithm": "", + "Initiator": { + "ID": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4", + "DisplayName": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4" + }, + "Owner": { + "DisplayName": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4", + "ID": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4" + }, + "StorageClass": "STANDARD" + } + ======================================== COMPLETE ==== + { + "Location": "http://localhost:9000/test/upload", + "Bucket": "test", + "Key": "upload", + "ETag": "\"8e817c8ccd442f9a79c77b58fe808c43-4\"" + } + ======================================== LIST ==== + + An error occurred (NoSuchUpload) when calling the ListParts operation: The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed. + ======================================== GET PART 4 ==== + { + "AcceptRanges": "bytes", + "LastModified": "2023-04-25T10:21:59+00:00", + "ContentLength": 20971520, + "ETag": "\"8e817c8ccd442f9a79c77b58fe808c43-4\"", + "ContentRange": "bytes 62914560-83886079/83886080", + "ContentType": "binary/octet-stream", + "Metadata": {}, + "PartsCount": 4 + } + 1bd2383eebbac1f8e7143575ba5b1f4a get-part4 + + + Conclusions: + + - Parts are indeed renumbered with consecutive numbers + - ListParts only applies to multipart uploads in progress, + it cannot be used once the multipart upload has been completed +' + +export AWS_ACCESS_KEY_ID=1D8Pk2k4oQSoh1BU +export AWS_SECRET_ACCESS_KEY=4B46SR8U7FUgY0raB8Zuxg1NLyLTvbNV + +function aws { command aws --endpoint-url http://localhost:9000 $@ ; } + +aws --version + +aws s3 mb s3://test + +for NUM in 1 4 5 6; do + dd if=/dev/urandom of=part$NUM bs=1M count=10 +done +md5sum part* + +UPLOAD=$(aws s3api create-multipart-upload --bucket test --key 'upload' | jq -r ".UploadId") +echo "Upload ID: $UPLOAD" + +PARTS="" + +for NUM in 1 4 5 6; do + ETAG=$(aws s3api upload-part --bucket test --key 'upload' --part-number $NUM \ + --body "part$NUM" --upload-id "$UPLOAD" | jq -r ".ETag") + echo "PART $NUM ETag: $ETAG" + if [ -n "$PARTS" ]; then + PARTS="$PARTS," + fi + PARTS="$PARTS {\"ETag\":$ETAG,\"PartNumber\":$NUM}" +done + +echo "======================================== LIST ====" +aws s3api list-parts --bucket test --key upload --upload-id "$UPLOAD" | jq + +echo "======================================== COMPLETE ====" +echo "{\"Parts\":[$PARTS]}" > mpu +aws s3api complete-multipart-upload --multipart-upload file://mpu \ + --bucket test --key 'upload' --upload-id "$UPLOAD" + +echo "======================================== LIST ====" +aws s3api list-parts --bucket test --key upload --upload-id "$UPLOAD" | jq + +echo "======================================== GET PART 4 ====" +aws s3api get-object --bucket test --key upload --part-number 4 get-part4 +md5sum get-part4 diff --git a/script/test-skip-part.sh b/script/test-skip-part.sh new file mode 100644 index 00000000..20ae017d --- /dev/null +++ b/script/test-skip-part.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +: ' + This script tests whether uploaded parts can be skipped in a + CompleteMultipartUpoad + + On Minio: yes, parts can be skipped + + On S3: not tested + + Sample output (on Minio): + + f23911bcd1230f5ebe8887cbf5bc396e part1 + a2657143167eaf647c40473e78a091dc part4 + 72f72c02c5163bc81024b28ac818c5e0 part5 + e29cf500d20498218904b8df8806caa2 part6 + Upload ID: e8fe7b83-9800-46fb-ae90-9d7ccd42fe76 + PART 1 ETag: "f23911bcd1230f5ebe8887cbf5bc396e" + PART 4 ETag: "a2657143167eaf647c40473e78a091dc" + PART 5 ETag: "72f72c02c5163bc81024b28ac818c5e0" + PART 6 ETag: "e29cf500d20498218904b8df8806caa2" + ======================================== COMPLETE ==== + { + "Location": "http://localhost:9000/test/upload", + "Bucket": "test", + "Key": "upload", + "ETag": "\"48246e44d4b38bdc2f3c10ee25b1af17-3\"" + } + ======================================== GET FULL ==== + { + "AcceptRanges": "bytes", + "LastModified": "2023-04-25T10:54:35+00:00", + "ContentLength": 31457280, + "ETag": "\"48246e44d4b38bdc2f3c10ee25b1af17-3\"", + "ContentType": "binary/octet-stream", + "Metadata": {} + } + 97fb904da7ad310699a6afab0eb6e061 get-full + 97fb904da7ad310699a6afab0eb6e061 - + ======================================== GET PART 3 ==== + { + "AcceptRanges": "bytes", + "LastModified": "2023-04-25T10:54:35+00:00", + "ContentLength": 10485760, + "ETag": "\"48246e44d4b38bdc2f3c10ee25b1af17-3\"", + "ContentRange": "bytes 20971520-31457279/31457280", + "ContentType": "binary/octet-stream", + "Metadata": {}, + "PartsCount": 3 + } + e29cf500d20498218904b8df8806caa2 get-part3 + + Conclusions: + + - Skipping a part in a CompleteMultipartUpoad call is OK + - The part is simply not included in the stored object + - Sequential part renumbering counts only non-skipped parts +' + +export AWS_ACCESS_KEY_ID=1D8Pk2k4oQSoh1BU +export AWS_SECRET_ACCESS_KEY=4B46SR8U7FUgY0raB8Zuxg1NLyLTvbNV + +function aws { command aws --endpoint-url http://localhost:9000 $@ ; } + +aws --version + +aws s3 mb s3://test + +for NUM in 1 4 5 6; do + dd if=/dev/urandom of=part$NUM bs=1M count=10 +done +md5sum part* + +UPLOAD=$(aws s3api create-multipart-upload --bucket test --key 'upload' | jq -r ".UploadId") +echo "Upload ID: $UPLOAD" + +PARTS="" + +for NUM in 1 4 5 6; do + ETAG=$(aws s3api upload-part --bucket test --key 'upload' --part-number $NUM \ + --body "part$NUM" --upload-id "$UPLOAD" | jq -r ".ETag") + echo "PART $NUM ETag: $ETAG" + if [ "$NUM" != "5" ]; then + if [ -n "$PARTS" ]; then + PARTS="$PARTS," + fi + PARTS="$PARTS {\"ETag\":$ETAG,\"PartNumber\":$NUM}" + fi +done + +echo "======================================== COMPLETE ====" +echo "{\"Parts\":[$PARTS]}" > mpu +aws s3api complete-multipart-upload --multipart-upload file://mpu \ + --bucket test --key 'upload' --upload-id "$UPLOAD" + +echo "======================================== GET FULL ====" +aws s3api get-object --bucket test --key upload get-full +md5sum get-full +cat part1 part4 part6 | md5sum + +echo "======================================== GET PART 3 ====" +aws s3api get-object --bucket test --key upload --part-number 3 get-part3 +md5sum get-part3 diff --git a/script/test-smoke.sh b/script/test-smoke.sh index eababf38..6965c0f3 100755 --- a/script/test-smoke.sh +++ b/script/test-smoke.sh @@ -31,6 +31,11 @@ dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # No multipart, inline sto dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5 # No multipart but file will be chunked dd if=/dev/urandom of=/tmp/garage.3.rnd bs=1M count=10 # by default, AWS starts using multipart at 8MB +dd if=/dev/urandom of=/tmp/garage.part1.rnd bs=1M count=5 +dd if=/dev/urandom of=/tmp/garage.part2.rnd bs=1M count=5 +dd if=/dev/urandom of=/tmp/garage.part3.rnd bs=1M count=5 +dd if=/dev/urandom of=/tmp/garage.part4.rnd bs=1M count=5 + # data of lower entropy, to test compression dd if=/dev/urandom bs=1k count=2 | base64 -w0 > /tmp/garage.1.b64 dd if=/dev/urandom bs=1M count=5 | base64 -w0 > /tmp/garage.2.b64 @@ -40,7 +45,7 @@ echo "🧪 S3 API testing..." # AWS if [ -z "$SKIP_AWS" ]; then - echo "🛠️ Testing with awscli" + echo "🛠️ Testing with awscli (aws s3)" source ${SCRIPT_FOLDER}/dev-env-aws.sh aws s3 ls for idx in {1..3}.{rnd,b64}; do @@ -51,8 +56,36 @@ if [ -z "$SKIP_AWS" ]; then rm /tmp/garage.$idx.dl aws s3 rm "s3://eprouvette/&+-é\"/garage.$idx.aws" done + + echo "🛠️ Testing multipart uploads with awscli (aws s3api)" + UPLOAD=$(aws s3api create-multipart-upload --bucket eprouvette --key 'upload' | jq -r ".UploadId") + echo "Upload ID: $UPLOAD" + ETAG3=$(aws s3api upload-part --bucket eprouvette --key 'upload' \ + --part-number 3 --body "/tmp/garage.part1.rnd" --upload-id "$UPLOAD" \ + | jq -r ".ETag") + ETAG2=$(aws s3api upload-part --bucket eprouvette --key 'upload' \ + --part-number 2 --body "/tmp/garage.part2.rnd" --upload-id "$UPLOAD" \ + | jq -r ".ETag") + ETAG3=$(aws s3api upload-part --bucket eprouvette --key 'upload' \ + --part-number 3 --body "/tmp/garage.part3.rnd" --upload-id "$UPLOAD" \ + | jq -r ".ETag") + ETAG6=$(aws s3api upload-part --bucket eprouvette --key 'upload' \ + --part-number 6 --body "/tmp/garage.part4.rnd" --upload-id "$UPLOAD" \ + | jq -r ".ETag") + MPU="{\"Parts\":[{\"PartNumber\":2,\"ETag\":$ETAG2}, {\"PartNumber\":3,\"ETag\":$ETAG3}, {\"PartNumber\":6,\"ETag\":$ETAG6}]}" + echo $MPU > /tmp/garage.mpu.json + aws s3api complete-multipart-upload --multipart-upload file:///tmp/garage.mpu.json \ + --bucket eprouvette --key 'upload' --upload-id "$UPLOAD" + aws s3api get-object --bucket eprouvette --key upload /tmp/garage.mpu.get + if [ "$(md5sum /tmp/garage.mpu.get | cut -d ' ' -f 1)" != "$(cat /tmp/garage.part{2,3,4}.rnd | md5sum | cut -d ' ' -f 1)" ]; then + echo "Invalid multipart upload" + exit 1 + fi fi +echo "OK!!" +exit 0 + # S3CMD if [ -z "$SKIP_S3CMD" ]; then echo "🛠️ Testing with s3cmd" @@ -141,6 +174,7 @@ rm eprouvette/winscp EOF fi +rm /tmp/garage.part{1..4}.rnd rm /tmp/garage.{1..3}.{rnd,b64} echo "🏁 Teardown" diff --git a/script/test-upgrade.sh b/script/test-upgrade.sh new file mode 100755 index 00000000..dc25e7c6 --- /dev/null +++ b/script/test-upgrade.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +set -ex + +export LC_ALL=C.UTF-8 +export LANG=C.UTF-8 +SCRIPT_FOLDER="`dirname \"$0\"`" +REPO_FOLDER="${SCRIPT_FOLDER}/../" +GARAGE_DEBUG="${REPO_FOLDER}/target/debug/" +GARAGE_RELEASE="${REPO_FOLDER}/target/release/" +NIX_RELEASE="${REPO_FOLDER}/result/bin/:${REPO_FOLDER}/result-bin/bin/" +PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH" + +OLD_VERSION="$1" +ARCH="$2" + + +echo "Downloading old garage binary..." +curl https://garagehq.deuxfleurs.fr/_releases/$OLD_VERSION/$ARCH/garage > /tmp/old_garage +chmod +x /tmp/old_garage + +echo "============= insert data into old version cluster =================" + +export GARAGE_BIN=/tmp/old_garage +if echo $OLD_VERSION | grep 'v0\.8\.'; then + echo "Detected Garage v0.8.x" + export GARAGE_08=1 +fi + +echo "⏳ Setup cluster using old version" +$GARAGE_BIN --version +${SCRIPT_FOLDER}/dev-clean.sh +${SCRIPT_FOLDER}/dev-cluster.sh > /tmp/garage.log 2>&1 & +sleep 6 +${SCRIPT_FOLDER}/dev-configure.sh +${SCRIPT_FOLDER}/dev-bucket.sh + +echo "🛠️ Inserting data in old cluster" +source ${SCRIPT_FOLDER}/dev-env-rclone.sh +rclone copy "${SCRIPT_FOLDER}/../.git/" garage:eprouvette/test_dotgit --stats=1s --stats-log-level=NOTICE --stats-one-line + +echo "🏁 Stopping old cluster" +killall -INT old_garage +sleep 2 +killall -9 old_garage || true + +echo "🏁 Removing old garage version" +rm -rv $GARAGE_BIN +export -n GARAGE_BIN +export -n GARAGE_08 + +echo "================ read data from new cluster ===================" + +echo "⏳ Setup cluster using new version" +pwd +ls +export GARAGE_BIN=$(which garage) +$GARAGE_BIN --version +${SCRIPT_FOLDER}/dev-cluster.sh >> /tmp/garage.log 2>&1 & +sleep 3 + +echo "🛠️ Retrieving data from old cluster" +rclone copy garage:eprouvette/test_dotgit /tmp/test_dotgit --stats=1s --stats-log-level=NOTICE --stats-one-line --fast-list + +if ! diff <(find "${SCRIPT_FOLDER}/../.git" -type f | xargs md5sum | cut -d ' ' -f 1 | sort) <(find /tmp/test_dotgit -type f | xargs md5sum | cut -d ' ' -f 1 | sort); then + echo "TEST FAILURE: directories are different" + exit 1 +fi +rm -r /tmp/test_dotgit + +echo "🏁 Teardown" +rm -rf /tmp/garage-{data,meta}-* +rm -rf /tmp/config.*.toml + +echo "✅ Success" |