1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
|
#!/usr/bin/env bash
: '
This script tests part renumbering on an S3 remote (here configured for Minio).
On Minio:
The results confirm that if I upload parts with number 1, 4, 5 and 6,
they are renumbered to 1, 2, 3 and 4 after CompleteMultipartUpload.
Thus, specifying partNumber=4 on a GetObject/HeadObject should return
information on the part I originally uploaded with part number
On S3: not tested
Sample output (on Minio):
f07e1404cc527d494242824ded3a616b part1
78974cd4d0f622eb3426ea7cd22f5a1c part4
f9cc379f8baa61645558d9ba7e6351fa part5
1bd2383eebbac1f8e7143575ba5b1f4a part6
Upload ID: 6838b813-d0ca-400b-9d28-ec8b2b5cd004
PART 1 ETag: "f07e1404cc527d494242824ded3a616b"
PART 4 ETag: "78974cd4d0f622eb3426ea7cd22f5a1c"
PART 5 ETag: "f9cc379f8baa61645558d9ba7e6351fa"
PART 6 ETag: "1bd2383eebbac1f8e7143575ba5b1f4a"
======================================== LIST ====
{
"Parts": [
{
"PartNumber": 1,
"LastModified": "2023-04-25T10:21:54.350000+00:00",
"ETag": "\"f07e1404cc527d494242824ded3a616b\"",
"Size": 20971520
},
{
"PartNumber": 4,
"LastModified": "2023-04-25T10:21:54.350000+00:00",
"ETag": "\"78974cd4d0f622eb3426ea7cd22f5a1c\"",
"Size": 20971520
},
{
"PartNumber": 5,
"LastModified": "2023-04-25T10:21:54.350000+00:00",
"ETag": "\"f9cc379f8baa61645558d9ba7e6351fa\"",
"Size": 20971520
},
{
"PartNumber": 6,
"LastModified": "2023-04-25T10:21:54.350000+00:00",
"ETag": "\"1bd2383eebbac1f8e7143575ba5b1f4a\"",
"Size": 20971520
}
],
"ChecksumAlgorithm": "",
"Initiator": {
"ID": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4",
"DisplayName": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4"
},
"Owner": {
"DisplayName": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4",
"ID": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4"
},
"StorageClass": "STANDARD"
}
======================================== COMPLETE ====
{
"Location": "http://localhost:9000/test/upload",
"Bucket": "test",
"Key": "upload",
"ETag": "\"8e817c8ccd442f9a79c77b58fe808c43-4\""
}
======================================== LIST ====
An error occurred (NoSuchUpload) when calling the ListParts operation: The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.
======================================== GET PART 4 ====
{
"AcceptRanges": "bytes",
"LastModified": "2023-04-25T10:21:59+00:00",
"ContentLength": 20971520,
"ETag": "\"8e817c8ccd442f9a79c77b58fe808c43-4\"",
"ContentRange": "bytes 62914560-83886079/83886080",
"ContentType": "binary/octet-stream",
"Metadata": {},
"PartsCount": 4
}
1bd2383eebbac1f8e7143575ba5b1f4a get-part4
Conclusions:
- Parts are indeed renumbered with consecutive numbers
- ListParts only applies to multipart uploads in progress,
it cannot be used once the multipart upload has been completed
'
export AWS_ACCESS_KEY_ID=1D8Pk2k4oQSoh1BU
export AWS_SECRET_ACCESS_KEY=4B46SR8U7FUgY0raB8Zuxg1NLyLTvbNV
function aws { command aws --endpoint-url http://localhost:9000 $@ ; }
aws --version
aws s3 mb s3://test
for NUM in 1 4 5 6; do
dd if=/dev/urandom of=part$NUM bs=1M count=10
done
md5sum part*
UPLOAD=$(aws s3api create-multipart-upload --bucket test --key 'upload' | jq -r ".UploadId")
echo "Upload ID: $UPLOAD"
PARTS=""
for NUM in 1 4 5 6; do
ETAG=$(aws s3api upload-part --bucket test --key 'upload' --part-number $NUM \
--body "part$NUM" --upload-id "$UPLOAD" | jq -r ".ETag")
echo "PART $NUM ETag: $ETAG"
if [ -n "$PARTS" ]; then
PARTS="$PARTS,"
fi
PARTS="$PARTS {\"ETag\":$ETAG,\"PartNumber\":$NUM}"
done
echo "======================================== LIST ===="
aws s3api list-parts --bucket test --key upload --upload-id "$UPLOAD" | jq
echo "======================================== COMPLETE ===="
echo "{\"Parts\":[$PARTS]}" > mpu
aws s3api complete-multipart-upload --multipart-upload file://mpu \
--bucket test --key 'upload' --upload-id "$UPLOAD"
echo "======================================== LIST ===="
aws s3api list-parts --bucket test --key upload --upload-id "$UPLOAD" | jq
echo "======================================== GET PART 4 ===="
aws s3api get-object --bucket test --key upload --part-number 4 get-part4
md5sum get-part4
|
