diff options
| -rw-r--r-- | main.go | 329 |
1 files changed, 238 insertions, 91 deletions
@@ -4,8 +4,10 @@ import ( "context" "errors" "fmt" + "io" "io/fs" "log" + "mime" "net/http" "os" "path" @@ -49,11 +51,13 @@ func main() { FileSystem: NewGarageFS(), LockSystem: webdav.NewMemLS(), Logger: func(r *http.Request, err error) { - log.Printf("WEBDAV: %#s, ERROR: %v", r, err) + log.Printf("INFO: %s %s %s\n", r.RemoteAddr, r.Method, r.URL) + if err != nil { + log.Printf("ERR: %v", err) + } }, } - //http.Handle("/", srv) http.HandleFunc(pathPrefix+"/", func(w http.ResponseWriter, r *http.Request) { username, password, ok := r.BasicAuth() @@ -167,6 +171,7 @@ func (s *GarageFS) Mkdir(ctx context.Context, name string, perm os.FileMode) err } func (s *GarageFS) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) { + NewGarageStatFromFile(ctx, name) return NewGarageFile(ctx, name) } @@ -179,71 +184,112 @@ func (s *GarageFS) Rename(ctx context.Context, oldName, newName string) error { } func (s *GarageFS) Stat(ctx context.Context, name string) (os.FileInfo, error) { + log.Println("Stat from GarageFS") return NewGarageStat(ctx, name) } type GarageFile struct { - ctx context.Context - mc *minio.Client - obj *minio.Object - stat *GarageStat - path string + ctx context.Context + mc *minio.Client + obj *minio.Object + objw *io.PipeWriter + donew chan error + pos int64 + path S3Path } func NewGarageFile(ctx context.Context, path string) (webdav.File, error) { gf := new(GarageFile) gf.ctx = ctx + gf.pos = 0 gf.mc = ctx.Value(garageEntry).(garageCtx).MC - gf.path = path - stat, err := NewGarageStat(ctx, path) - if err != nil { - return nil, err - } - gf.stat = stat + gf.path = NewS3Path(path) return gf, nil } func (gf *GarageFile) Close() error { - if gf.obj == nil { - return nil - } - err := gf.obj.Close() - gf.obj = nil - return err + err := make([]error, 0) + + if gf.obj != nil { + err = append(err, gf.obj.Close()) + gf.obj = nil + } + + if gf.objw != nil { + // wait that minio completes its transfers in background + err = append(err, gf.objw.Close()) + err = append(err, <-gf.donew) + gf.donew = nil + gf.objw = nil + } + + count := 0 + for _, e := range err { + if e != nil { + count++ + log.Println(e) + } + } + if count > 0 { + return errors.New(fmt.Sprintf("%d errors when closing this WebDAV File. Read previous logs to know more.", count)) + } + return nil } func (gf *GarageFile) loadObject() error { - if gf.obj == nil { - log.Println("Called GetObject on", gf.path) - obj, err := gf.mc.GetObject(gf.ctx, gf.stat.bucket, gf.stat.obj.Key, minio.GetObjectOptions{}) - if err != nil { - return err - } - gf.obj = obj - } - return nil + if gf.obj == nil { + obj, err := gf.mc.GetObject(gf.ctx, gf.path.bucket, gf.path.key, minio.GetObjectOptions{}) + if err != nil { + return err + } + gf.obj = obj + } + return nil } func (gf *GarageFile) Read(p []byte) (n int, err error) { - if gf.stat.Mode().IsDir() { - return 0, os.ErrInvalid + //if gf.Stat() & OBJECT == 0 { /* @FIXME Ideally we would check against OBJECT but we need a non OPAQUE_KEY */ + // return 0, os.ErrInvalid + //} + if err := gf.loadObject(); err != nil { + return 0, err } - if err := gf.loadObject(); err != nil { - return 0, err - } - return gf.obj.Read(p) + return gf.obj.Read(p) } func (gf *GarageFile) Write(p []byte) (n int, err error) { - return 0, errors.New("not implemented Write") + /*if gf.path.class != OBJECT { + return 0, os.ErrInvalid + }*/ + + if gf.objw == nil { + if gf.pos != 0 { + return 0, errors.New("writing with an offset is not implemented") + } + + r, w := io.Pipe() + gf.donew = make(chan error, 1) + gf.objw = w + + contentType := mime.TypeByExtension(path.Ext(gf.path.key)) + go func() { + _, err := gf.mc.PutObject(context.Background(), gf.path.bucket, gf.path.key, r, -1, minio.PutObjectOptions{ContentType: contentType}) + gf.donew <- err + }() + } + + return gf.objw.Write(p) } func (gf *GarageFile) Seek(offset int64, whence int) (int64, error) { - if err := gf.loadObject(); err != nil { - return 0, err - } - return gf.obj.Seek(offset, whence) + if err := gf.loadObject(); err != nil { + return 0, err + } + + pos, err := gf.obj.Seek(offset, whence) + gf.pos += pos + return pos, err } /* @@ -254,13 +300,14 @@ If n > 0, ReadDir returns at most n DirEntry records. In this case, if ReadDir r If n <= 0, ReadDir returns all the DirEntry records remaining in the directory. When it succeeds, it returns a nil error (not io.EOF). */ func (gf *GarageFile) Readdir(count int) ([]fs.FileInfo, error) { - log.Println("Call Readdir with count", count) + if count > 0 { + return nil, errors.New("returning a limited number of directory entry is not supported in readdir") + } - if gf.path == "/" { + if gf.path.class == ROOT { return gf.readDirRoot(count) } else { - exploded_path := strings.SplitN(gf.path, "/", 3) - return gf.readDirChild(count, exploded_path[1], exploded_path[2]) + return gf.readDirChild(count) } } @@ -282,10 +329,9 @@ func (gf *GarageFile) readDirRoot(count int) ([]fs.FileInfo, error) { return entries, nil } -func (gf *GarageFile) readDirChild(count int, bucket, prefix string) ([]fs.FileInfo, error) { - log.Println("call ListObjects with", bucket, prefix) - objs_info := gf.mc.ListObjects(gf.ctx, bucket, minio.ListObjectsOptions{ - Prefix: prefix, +func (gf *GarageFile) readDirChild(count int) ([]fs.FileInfo, error) { + objs_info := gf.mc.ListObjects(gf.ctx, gf.path.bucket, minio.ListObjectsOptions{ + Prefix: gf.path.key, Recursive: false, }) @@ -294,7 +340,7 @@ func (gf *GarageFile) readDirChild(count int, bucket, prefix string) ([]fs.FileI if object.Err != nil { return nil, object.Err } - ngf, err := NewGarageStatFromObjectInfo(gf.ctx, bucket, object) + ngf, err := NewGarageStatFromObjectInfo(gf.ctx, gf.path.bucket, object) if err != nil { return nil, err } @@ -305,24 +351,40 @@ func (gf *GarageFile) readDirChild(count int, bucket, prefix string) ([]fs.FileI } func (gf *GarageFile) Stat() (fs.FileInfo, error) { - return NewGarageStat(gf.ctx, gf.path) + return NewGarageStatFromFile(gf.ctx, gf.path.path) } /* Implements */ // StatObject??? type GarageStat struct { - obj minio.ObjectInfo - bucket string + obj minio.ObjectInfo + ctx context.Context + path S3Path } -func NewGarageStat(ctx context.Context, path string) (*GarageStat, error) { +/* + * Stat a path + */ +func NewGarageStatFromFile(ctx context.Context, path string) (*GarageStat, error) { cache := ctx.Value(garageEntry).(garageCtx).StatCache + + // Maybe this file is already in our cache? if entry, ok := cache[path]; ok { return entry, nil } - gs, err := newGarageStatFresh(ctx, path) - if err != nil { + // Create a placeholder in case we are creating the object + gs := new(GarageStat) + gs.ctx = ctx + gs.path = NewS3Path(path) + gs.path.class = OBJECT // known because called from GarageFile + gs.obj.Key = gs.path.key + gs.obj.LastModified = time.Now() + + // Maybe this file exists in garage? + err := gs.Refresh() + if err != nil && !os.IsNotExist(err) { + // There is an error and this is not a 404, report it. return nil, err } @@ -330,56 +392,92 @@ func NewGarageStat(ctx context.Context, path string) (*GarageStat, error) { return gs, nil } -func newGarageStatFresh(ctx context.Context, path string) (*GarageStat, error) { - mc := ctx.Value(garageEntry).(garageCtx).MC +/* + * Stat a path knowing its ObjectInfo + */ +func NewGarageStatFromObjectInfo(ctx context.Context, bucket string, obj minio.ObjectInfo) (*GarageStat, error) { gs := new(GarageStat) - gs.bucket = "/" - gs.obj = minio.ObjectInfo{} + gs.path = NewTrustedS3Path(bucket, obj) + gs.obj = obj - exploded_path := strings.SplitN(path, "/", 3) + cache := ctx.Value(garageEntry).(garageCtx).StatCache + cache[gs.path.path] = gs + return gs, nil +} - // Check if we can extract the bucket name - if len(exploded_path) < 2 { - return gs, nil +/* + * Stat a path without additional information + */ +func NewGarageStat(ctx context.Context, path string) (*GarageStat, error) { + log.Println("Probe file", path) + cache := ctx.Value(garageEntry).(garageCtx).StatCache + if entry, ok := cache[path]; ok { + return entry, nil } - gs.bucket = exploded_path[1] - // Check if we can extract the prefix - if len(exploded_path) < 3 || exploded_path[2] == "" { - return gs, nil + gs := new(GarageStat) + gs.ctx = ctx + gs.path = NewS3Path(path) + if err := gs.Refresh(); err != nil { + return nil, err } - gs.obj.Key = exploded_path[2] - // Check if this is a file or a folder - log.Println("call StatObject with", gs.bucket, gs.obj.Key) - obj, err := mc.StatObject(ctx, gs.bucket, gs.obj.Key, minio.StatObjectOptions{}) - if e, ok := err.(minio.ErrorResponse); ok && e.Code == "NoSuchKey" { - return gs, nil - } - if err != nil { - return nil, err + if gs.path.class&OPAQUE_KEY != 0 { + return nil, errors.New("Failed to precisely determine the key type, this a logic error.") } - // If it is a file, assign its data - gs.obj = obj + cache[path] = gs + cache[gs.path.path] = gs return gs, nil } -func NewGarageStatFromObjectInfo(ctx context.Context, bucket string, obj minio.ObjectInfo) (*GarageStat, error) { - gs := new(GarageStat) - gs.bucket = bucket - gs.obj = obj +func (gs *GarageStat) Refresh() error { + if gs.path.class == ROOT || gs.path.class == BUCKET { + return nil + } - cache := ctx.Value(garageEntry).(garageCtx).StatCache - cache[path.Join("/", bucket, obj.Key)] = gs - return gs, nil + mc := gs.ctx.Value(garageEntry).(garageCtx).MC + + // Compute the prefix to have the desired behaviour for our stat logic + prefix := gs.path.key + if prefix[len(prefix)-1:] == "/" { + prefix = prefix[:len(prefix)-1] + } + + // Get info and check if the key exists + objs_info := mc.ListObjects(gs.ctx, gs.path.bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: false, + }) + + found := false + for object := range objs_info { + if object.Err != nil { + return object.Err + } + + if object.Key == prefix || object.Key == prefix+"/" { + gs.obj = object + gs.path = NewTrustedS3Path(gs.path.bucket, object) + found = true + break + } + } + + if !found { + return fs.ErrNotExist + } + + return nil } func (gs *GarageStat) Name() string { - if gs.obj.Key != "" { - return path.Base(gs.obj.Key) + if gs.path.class == ROOT { + return "/" + } else if gs.path.class == BUCKET { + return gs.path.bucket } else { - return gs.bucket + return path.Base(gs.path.key) } } @@ -388,10 +486,10 @@ func (gs *GarageStat) Size() int64 { } func (gs *GarageStat) Mode() fs.FileMode { - if gs.obj.ETag == "" { - return fs.ModeDir | fs.ModePerm - } else { + if gs.path.class == OBJECT { return fs.ModePerm + } else { + return fs.ModeDir | fs.ModePerm } } @@ -400,9 +498,58 @@ func (gs *GarageStat) ModTime() time.Time { } func (gs *GarageStat) IsDir() bool { - return gs.Mode().IsDir() + return gs.path.class != OBJECT } func (gs *GarageStat) Sys() interface{} { return nil } + +type S3Class int + +const ( + ROOT S3Class = 1 << iota + BUCKET + COMMON_PREFIX + OBJECT + OPAQUE_KEY + + KEY = COMMON_PREFIX | OBJECT | OPAQUE_KEY +) + +type S3Path struct { + path string + class S3Class + bucket string + key string +} + +func NewS3Path(path string) S3Path { + exploded_path := strings.SplitN(path, "/", 3) + + // If there is no bucket name (eg. "/") + if len(exploded_path) < 2 || exploded_path[1] == "" { + return S3Path{path, ROOT, "", ""} + } + + // If there is no key + if len(exploded_path) < 3 || exploded_path[2] == "" { + return S3Path{path, BUCKET, exploded_path[1], ""} + } + + return S3Path{path, OPAQUE_KEY, exploded_path[1], exploded_path[2]} +} + +func NewTrustedS3Path(bucket string, obj minio.ObjectInfo) S3Path { + cl := OBJECT + if obj.Key[len(obj.Key)-1:] == "/" { + cl = COMMON_PREFIX + } + + return S3Path{ + path: path.Join("/", bucket, obj.Key), + bucket: bucket, + key: obj.Key, + class: cl, + } +} |