diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | auth_basic.go | 28 | ||||
| -rw-r--r-- | auth_ldap.go | 115 | ||||
| -rw-r--r-- | auth_s3.go | 29 | ||||
| -rw-r--r-- | config.go | 77 | ||||
| -rw-r--r-- | error.go | 24 | ||||
| -rw-r--r-- | main.go | 574 | ||||
| -rw-r--r-- | middleware.go | 17 | ||||
| -rw-r--r-- | s3_file.go | 186 | ||||
| -rw-r--r-- | s3_fs.go | 66 | ||||
| -rw-r--r-- | s3_path.go | 57 | ||||
| -rw-r--r-- | s3_stat.go | 128 | ||||
| -rw-r--r-- | webdav.go | 28 |
13 files changed, 777 insertions, 553 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9b6940b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +bagage diff --git a/auth_basic.go b/auth_basic.go new file mode 100644 index 0000000..f0cfaae --- /dev/null +++ b/auth_basic.go @@ -0,0 +1,28 @@ +package main + +import ( + "errors" + "net/http" +) + +/* + * We extract the credentials from the Basic Auth headers + * (We may think to other ways to pass credentials such as a JWT) + */ +type BasicAuthExtract struct { + OnNotFound ErrorHandler + OnCreds CredsHandler +} + +func (b BasicAuthExtract) ServeHTTP(w http.ResponseWriter, r *http.Request) { + username, password, ok := r.BasicAuth() + if !ok { + b.OnNotFound.WithError(errors.New("LDAP. Missing Authentication Header")).ServeHTTP(w, r) + return + } + if username == "" || password == "" { + b.OnNotFound.WithError(errors.New("LDAP. Username or password cannot be empty")).ServeHTTP(w, r) + return + } + b.OnCreds.WithCreds(username, password).ServeHTTP(w, r) +} diff --git a/auth_ldap.go b/auth_ldap.go new file mode 100644 index 0000000..9164e93 --- /dev/null +++ b/auth_ldap.go @@ -0,0 +1,115 @@ +package main + +import ( + "errors" + "fmt" + "net/http" + + "github.com/go-ldap/ldap/v3" +) + +/* Check credentials against LDAP */ +type LdapPreAuth struct { + WithConfig *Config + OnWrongPassword ErrorHandler + OnFailure ErrorHandler + OnCreds CredsHandler +} + +func (l LdapPreAuth) WithCreds(username, password string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + // 1. Connect to the server + conn, err := ldapConnect(l.WithConfig) + if err != nil { + l.OnFailure.WithError(err).ServeHTTP(w, r) + return + } + defer conn.Close() + + // 2. Authenticate with provided credentials + // @FIXME we should better check the error, it could also be due to an LDAP error + err = conn.auth(username, password) + if err != nil { + l.OnWrongPassword.WithError(err).ServeHTTP(w, r) + return + } + + // 3. Fetch user's profile + profile, err := conn.profile() + if err != nil { + l.OnFailure.WithError(err).ServeHTTP(w, r) + return + } + + // 4. Basic checks upon users' attributes + access_key := profile.GetAttributeValue("garage_s3_access_key") + secret_key := profile.GetAttributeValue("garage_s3_secret_key") + if access_key == "" || secret_key == "" { + err = errors.New(fmt.Sprintf("Either access key or secret key is missing in LDAP for %s", conn.userDn)) + l.OnFailure.WithError(err).ServeHTTP(w, r) + return + } + + // 5. Send fetched credentials to the next middleware + l.OnCreds.WithCreds(access_key, secret_key).ServeHTTP(w, r) + }) +} + +/** + * Private logic + */ + +type ldapConnector struct { + conn *ldap.Conn + config *Config + userDn string +} + +func ldapConnect(c *Config) (ldapConnector, error) { + ldapSock, err := ldap.Dial("tcp", c.LdapServer) + if err != nil { + return ldapConnector{}, err + } + + return ldapConnector{ + conn: ldapSock, + config: c, + }, nil +} + +func (l *ldapConnector) auth(username, password string) error { + l.userDn = fmt.Sprintf("%s=%s,%s", l.config.UserNameAttr, username, l.config.UserBaseDN) + return l.conn.Bind(l.userDn, password) +} + +func (l *ldapConnector) profile() (*ldap.Entry, error) { + searchRequest := ldap.NewSearchRequest( + l.userDn, + ldap.ScopeBaseObject, + ldap.NeverDerefAliases, + 0, + 0, + false, + "(objectClass=*)", + []string{"garage_s3_access_key", "garage_s3_secret_key"}, + nil) + + sr, err := l.conn.Search(searchRequest) + if err != nil { + return nil, err + } + + if len(sr.Entries) != 1 { + return nil, errors.New(fmt.Sprintf("Wrong number of LDAP entries, expected 1, got", len(sr.Entries))) + } + + return sr.Entries[0], nil +} + +func (l *ldapConnector) Close() { + if l.conn != nil { + l.conn.Close() + l.conn = nil + } +} diff --git a/auth_s3.go b/auth_s3.go new file mode 100644 index 0000000..4bbfe5e --- /dev/null +++ b/auth_s3.go @@ -0,0 +1,29 @@ +package main + +import ( + "net/http" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +/* Check credentials against Minio */ +type S3Auth struct { + WithConfig *Config + OnMinioClient MinioClientHandler + OnFailure ErrorHandler +} + +func (s S3Auth) WithCreds(access_key, secret_key string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mc, err := minio.New(s.WithConfig.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(access_key, secret_key, ""), + Secure: s.WithConfig.UseSSL, + }) + if err != nil { + s.OnFailure.WithError(err).ServeHTTP(w, r) + return + } + s.OnMinioClient.WithMC(mc).ServeHTTP(w, r) + }) +} diff --git a/config.go b/config.go new file mode 100644 index 0000000..002f317 --- /dev/null +++ b/config.go @@ -0,0 +1,77 @@ +package main + +import ( + "fmt" + "os" + "reflect" +) + +type Config struct { + HttpListen string `env:"BAGAGE_HTTP_LISTEN" default:":8080"` + DavPath string `env:"BAGAGE_WEBDAV_PREFIX" default:"/webdav"` + LdapServer string `env:"BAGAGE_LDAP_ENDPOINT" default:"127.0.0.1:1389"` + UserBaseDN string `env:"BAGAGE_LDAP_USER_BASE_DN" default:"ou=users,dc=deuxfleurs,dc=fr"` + UserNameAttr string `env:"BAGAGE_LDAP_USERNAME_ATTR" default:"cn"` + Endpoint string `env:"BAGAGE_S3_ENDPOINT" default:"garage.deuxfleurs.fr"` + UseSSL bool `env:"BAGAGE_S3_SSL" default:"true"` +} + +func (c *Config) LoadWithDefault() *Config { + c.iter(func(t reflect.StructField, v reflect.Value) { + tag := t.Tag.Get("default") + if tag == "" { + return + } else { + setKey(v, tag) + } + }) + + return c +} + +func (c *Config) LoadWithEnv() *Config { + c.iter(func(t reflect.StructField, v reflect.Value) { + tag := t.Tag.Get("env") + if tag == "" { + return + } else if val, ok := os.LookupEnv(tag); ok { + setKey(v, val) + } + }) + + return c +} + +func (c *Config) String() (rep string) { + rep = "Configuration:\n" + + c.iter(func(t reflect.StructField, v reflect.Value) { + rep += "\t" + t.Name + ": " + if t.Type.Kind() == reflect.Bool { + rep += fmt.Sprintf("%v", v.Bool()) + "\n" + } else { + rep += "\"" + v.String() + "\"\n" + } + }) + + return +} + +func (c *Config) iter(cb func(t reflect.StructField, v reflect.Value)) { + t := reflect.ValueOf(c).Elem() + for i := 0; i < t.Type().NumField(); i++ { + field := t.Field(i) + typeField := t.Type().Field(i) + cb(typeField, field) + } +} + +func setKey(v reflect.Value, e string) { + if v.Type().Kind() == reflect.String { + v.SetString(e) + } else if v.Type().Kind() == reflect.Bool { + v.SetBool(e == "true") + } else { + panic("Unsupported type") + } +} diff --git a/error.go b/error.go new file mode 100644 index 0000000..114bbf1 --- /dev/null +++ b/error.go @@ -0,0 +1,24 @@ +package main + +import ( + "net/http" +) + +type NotAuthorized struct{} + +func (n NotAuthorized) WithError(err error) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("WWW-Authenticate", `Basic realm="Pour accéder à Bagage, veuillez entrer vos identifiants Deuxfleurs"`) + w.WriteHeader(401) + w.Write([]byte("401 Unauthorized\n")) + }) +} + +type InternalError struct{} + +func (i InternalError) WithError(err error) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + w.Write([]byte("500 Internal Server Error\n")) + }) +} @@ -1,567 +1,35 @@ package main import ( - "context" - "errors" - "fmt" - "io" - "io/fs" "log" - "mime" "net/http" - "os" - "path" - "strings" - "time" - - "golang.org/x/net/webdav" - - "github.com/go-ldap/ldap/v3" - - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" ) -type bagageCtxKey string - -const garageEntry = bagageCtxKey("garage") - -type garageCtx struct { - MC *minio.Client - StatCache map[string]*GarageStat -} - -func EnvOrDefault(key, def string) string { - if val, ok := os.LookupEnv(key); ok { - return val - } - return def -} - func main() { log.Println("=== Starting Bagage ===") - HttpListen := EnvOrDefault("BAGAGE_HTTP_LISTEN", ":8080") - pathPrefix := EnvOrDefault("BAGAGE_WEBDAV_PREFIX", "/webdav") - LdapServer := EnvOrDefault("BAGAGE_LDAP_ENDPOINT", "127.0.0.1:1389") - UserBaseDN := EnvOrDefault("BAGAGE_LDAP_USER_BASE_DN", "ou=users,dc=deuxfleurs,dc=fr") - UserNameAttr := EnvOrDefault("BAGAGE_LDAP_USERNAME_ATTR", "cn") - Endpoint := EnvOrDefault("BAGAGE_S3_ENDPOINT", "garage.deuxfleurs.fr") - UseSSL := EnvOrDefault("BAGAGE_S3_SSL", "true") == "true" - - srv := &webdav.Handler{ - Prefix: pathPrefix, - FileSystem: NewGarageFS(), - LockSystem: webdav.NewMemLS(), - Logger: func(r *http.Request, err error) { - log.Printf("INFO: %s %s %s\n", r.RemoteAddr, r.Method, r.URL) - if err != nil { - log.Printf("ERR: %v", err) - } - }, - } - - http.HandleFunc(pathPrefix+"/", func(w http.ResponseWriter, r *http.Request) { - username, password, ok := r.BasicAuth() - - if !ok { - NotAuthorized(w, r) - return - } - - ldapSock, err := ldap.Dial("tcp", LdapServer) - if err != nil { - log.Println(err) - InternalError(w, r) - return - } - defer ldapSock.Close() - - // Check credential - userDn := fmt.Sprintf("%s=%s,%s", UserNameAttr, username, UserBaseDN) - err = ldapSock.Bind(userDn, password) - if err != nil { - log.Println(err) - NotAuthorized(w, r) - return - } - - // Get S3 creds garage_s3_access_key garage_s3_secret_key - searchRequest := ldap.NewSearchRequest( - userDn, - ldap.ScopeBaseObject, - ldap.NeverDerefAliases, - 0, - 0, - false, - "(objectClass=*)", - []string{"garage_s3_access_key", "garage_s3_secret_key"}, - nil) - - sr, err := ldapSock.Search(searchRequest) - if err != nil { - log.Println(err) - InternalError(w, r) - return - } - - if len(sr.Entries) != 1 { - log.Println("Wrong number of LDAP entries, expected 1, got", len(sr.Entries)) - InternalError(w, r) - return - } - - access_key := sr.Entries[0].GetAttributeValue("garage_s3_access_key") - secret_key := sr.Entries[0].GetAttributeValue("garage_s3_secret_key") - - if access_key == "" || secret_key == "" { - log.Println("Either access key or secret key is missing in LDAP for ", userDn) - InternalError(w, r) - return - } - - mc, err := minio.New(Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(access_key, secret_key, ""), - Secure: UseSSL, + config := (&Config{}).LoadWithDefault().LoadWithEnv() + + log.Println(config) + + // Assemble components to handle WebDAV requests + http.Handle(config.DavPath+"/", + BasicAuthExtract{ + OnNotFound: NotAuthorized{}, + OnCreds: LdapPreAuth{ + WithConfig: config, + OnWrongPassword: NotAuthorized{}, + OnFailure: InternalError{}, + OnCreds: S3Auth{ + WithConfig: config, + OnFailure: InternalError{}, + OnMinioClient: WebDav{ + WithConfig: config, + }, + }, + }, }) - if err != nil { - log.Println(err) - InternalError(w, r) - return - } - - nctx := context.WithValue(r.Context(), garageEntry, garageCtx{MC: mc, StatCache: make(map[string]*GarageStat)}) - srv.ServeHTTP(w, r.WithContext(nctx)) - return - }) - if err := http.ListenAndServe(HttpListen, nil); err != nil { + if err := http.ListenAndServe(config.HttpListen, nil); err != nil { log.Fatalf("Error with WebDAV server: %v", err) } } - -func NotAuthorized(w http.ResponseWriter, r *http.Request) { - w.Header().Set("WWW-Authenticate", `Basic realm="Pour accéder à Bagage, veuillez entrer vos identifiants Deuxfleurs"`) - w.WriteHeader(401) - w.Write([]byte("401 Unauthorized\n")) -} - -func InternalError(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(500) - w.Write([]byte("500 Internal Server Error\n")) -} - -/* - /////// Select Action - If no slash or one trailing slash - return ListBuckets - Else - obj := ListObjects - If obj.Length == 1 - return GetObject - Else - return obj -*/ -type GarageFS struct{} - -func NewGarageFS() *GarageFS { - grg := new(GarageFS) - return grg -} - -func (s *GarageFS) Mkdir(ctx context.Context, name string, perm os.FileMode) error { - return errors.New("Not implemented Mkdir") -} - -func (s *GarageFS) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) { - //log.Println("Stat from GarageFS.OpenFile()", name) - NewGarageStatFromFile(ctx, name) - return NewGarageFile(ctx, name) -} - -func (s *GarageFS) RemoveAll(ctx context.Context, name string) error { - return errors.New("Not implemented RemoveAll") -} - -func (s *GarageFS) Rename(ctx context.Context, oldName, newName string) error { - return errors.New("Not implemented Rename") -} - -func (s *GarageFS) Stat(ctx context.Context, name string) (os.FileInfo, error) { - //log.Println("Stat from GarageFS.Stat()", name) - return NewGarageStat(ctx, name) -} - -type GarageFile struct { - ctx context.Context - mc *minio.Client - obj *minio.Object - objw *io.PipeWriter - donew chan error - pos int64 - path S3Path -} - -func NewGarageFile(ctx context.Context, path string) (webdav.File, error) { - gf := new(GarageFile) - gf.ctx = ctx - gf.pos = 0 - gf.mc = ctx.Value(garageEntry).(garageCtx).MC - gf.path = NewS3Path(path) - return gf, nil -} - -func (gf *GarageFile) Close() error { - err := make([]error, 0) - - if gf.obj != nil { - err = append(err, gf.obj.Close()) - gf.obj = nil - } - - if gf.objw != nil { - // wait that minio completes its transfers in background - err = append(err, gf.objw.Close()) - err = append(err, <-gf.donew) - gf.donew = nil - gf.objw = nil - } - - count := 0 - for _, e := range err { - if e != nil { - count++ - log.Println(e) - } - } - if count > 0 { - return errors.New(fmt.Sprintf("%d errors when closing this WebDAV File. Read previous logs to know more.", count)) - } - return nil -} - -func (gf *GarageFile) loadObject() error { - if gf.obj == nil { - obj, err := gf.mc.GetObject(gf.ctx, gf.path.bucket, gf.path.key, minio.GetObjectOptions{}) - if err != nil { - return err - } - gf.obj = obj - } - return nil -} - -func (gf *GarageFile) Read(p []byte) (n int, err error) { - //if gf.Stat() & OBJECT == 0 { /* @FIXME Ideally we would check against OBJECT but we need a non OPAQUE_KEY */ - // return 0, os.ErrInvalid - //} - if err := gf.loadObject(); err != nil { - return 0, err - } - - return gf.obj.Read(p) -} - -func (gf *GarageFile) Write(p []byte) (n int, err error) { - /*if gf.path.class != OBJECT { - return 0, os.ErrInvalid - }*/ - - if gf.objw == nil { - if gf.pos != 0 { - return 0, errors.New("writing with an offset is not implemented") - } - - r, w := io.Pipe() - gf.donew = make(chan error, 1) - gf.objw = w - - contentType := mime.TypeByExtension(path.Ext(gf.path.key)) - go func() { - _, err := gf.mc.PutObject(context.Background(), gf.path.bucket, gf.path.key, r, -1, minio.PutObjectOptions{ContentType: contentType}) - gf.donew <- err - }() - } - - return gf.objw.Write(p) -} - -func (gf *GarageFile) Seek(offset int64, whence int) (int64, error) { - if err := gf.loadObject(); err != nil { - return 0, err - } - - pos, err := gf.obj.Seek(offset, whence) - gf.pos += pos - return pos, err -} - -/* -ReadDir reads the contents of the directory associated with the file f and returns a slice of DirEntry values in directory order. Subsequent calls on the same file will yield later DirEntry records in the directory. - -If n > 0, ReadDir returns at most n DirEntry records. In this case, if ReadDir returns an empty slice, it will return an error explaining why. At the end of a directory, the error is io.EOF. - -If n <= 0, ReadDir returns all the DirEntry records remaining in the directory. When it succeeds, it returns a nil error (not io.EOF). -*/ -func (gf *GarageFile) Readdir(count int) ([]fs.FileInfo, error) { - if count > 0 { - return nil, errors.New("returning a limited number of directory entry is not supported in readdir") - } - - if gf.path.class == ROOT { - return gf.readDirRoot(count) - } else { - return gf.readDirChild(count) - } -} - -func (gf *GarageFile) readDirRoot(count int) ([]fs.FileInfo, error) { - buckets, err := gf.mc.ListBuckets(gf.ctx) - if err != nil { - return nil, err - } - - entries := make([]fs.FileInfo, 0, len(buckets)) - for _, bucket := range buckets { - //log.Println("Stat from GarageFile.readDirRoot()", "/"+bucket.Name) - ngf, err := NewGarageStat(gf.ctx, "/"+bucket.Name) - if err != nil { - return nil, err - } - entries = append(entries, ngf) - } - - return entries, nil -} - -func (gf *GarageFile) readDirChild(count int) ([]fs.FileInfo, error) { - prefix := gf.path.key - if len(prefix) > 0 && prefix[len(prefix)-1:] != "/" { - prefix = prefix + "/" - } - - objs_info := gf.mc.ListObjects(gf.ctx, gf.path.bucket, minio.ListObjectsOptions{ - Prefix: prefix, - Recursive: false, - }) - - entries := make([]fs.FileInfo, 0) - for object := range objs_info { - if object.Err != nil { - return nil, object.Err - } - //log.Println("Stat from GarageFile.readDirChild()", path.Join("/", gf.path.bucket, object.Key)) - ngf, err := NewGarageStatFromObjectInfo(gf.ctx, gf.path.bucket, object) - if err != nil { - return nil, err - } - entries = append(entries, ngf) - } - - return entries, nil -} - -func (gf *GarageFile) Stat() (fs.FileInfo, error) { - //log.Println("Stat from GarageFile.Stat()", gf.path.path) - return NewGarageStatFromFile(gf.ctx, gf.path.path) -} - -/* Implements */ -// StatObject??? -type GarageStat struct { - obj minio.ObjectInfo - ctx context.Context - path S3Path -} - -/* - * Stat a path - */ -func NewGarageStatFromFile(ctx context.Context, path string) (*GarageStat, error) { - cache := ctx.Value(garageEntry).(garageCtx).StatCache - - // Maybe this file is already in our cache? - if entry, ok := cache[path]; ok { - return entry, nil - } - - // Create a placeholder in case we are creating the object - gs := new(GarageStat) - gs.ctx = ctx - gs.path = NewS3Path(path) - if gs.path.class == OPAQUE_KEY { - gs.path.class = OBJECT // known because called from GarageFile - } - gs.obj.Key = gs.path.key - gs.obj.LastModified = time.Now() - - // Maybe this file exists in garage? - err := gs.Refresh() - if err != nil && !os.IsNotExist(err) { - // There is an error and this is not a 404, report it. - return nil, err - } - - cache[path] = gs - return gs, nil -} - -/* - * Stat a path knowing its ObjectInfo - */ -func NewGarageStatFromObjectInfo(ctx context.Context, bucket string, obj minio.ObjectInfo) (*GarageStat, error) { - gs := new(GarageStat) - gs.path = NewTrustedS3Path(bucket, obj) - gs.obj = obj - - cache := ctx.Value(garageEntry).(garageCtx).StatCache - cache[gs.path.path] = gs - return gs, nil -} - -/* - * Stat a path without additional information - */ -func NewGarageStat(ctx context.Context, path string) (*GarageStat, error) { - cache := ctx.Value(garageEntry).(garageCtx).StatCache - if entry, ok := cache[path]; ok { - return entry, nil - } - - gs := new(GarageStat) - gs.ctx = ctx - gs.path = NewS3Path(path) - if err := gs.Refresh(); err != nil { - return nil, err - } - - if gs.path.class&OPAQUE_KEY != 0 { - return nil, errors.New("Failed to precisely determine the key type, this a logic error.") - } - - cache[path] = gs - cache[gs.path.path] = gs - return gs, nil -} - -func (gs *GarageStat) Refresh() error { - if gs.path.class == ROOT || gs.path.class == BUCKET { - return nil - } - - mc := gs.ctx.Value(garageEntry).(garageCtx).MC - - // Compute the prefix to have the desired behaviour for our stat logic - prefix := gs.path.key - if prefix[len(prefix)-1:] == "/" { - prefix = prefix[:len(prefix)-1] - } - - // Get info and check if the key exists - objs_info := mc.ListObjects(gs.ctx, gs.path.bucket, minio.ListObjectsOptions{ - Prefix: prefix, - Recursive: false, - }) - - found := false - for object := range objs_info { - if object.Err != nil { - return object.Err - } - - if object.Key == prefix || object.Key == prefix+"/" { - gs.obj = object - gs.path = NewTrustedS3Path(gs.path.bucket, object) - found = true - break - } - } - - if !found { - return fs.ErrNotExist - } - - return nil -} - -func (gs *GarageStat) Name() string { - if gs.path.class == ROOT { - return "/" - } else if gs.path.class == BUCKET { - return gs.path.bucket - } else { - return path.Base(gs.path.key) - } -} - -func (gs *GarageStat) Size() int64 { - return gs.obj.Size -} - -func (gs *GarageStat) Mode() fs.FileMode { - if gs.path.class == OBJECT { - return fs.ModePerm - } else { - return fs.ModeDir | fs.ModePerm - } -} - -func (gs *GarageStat) ModTime() time.Time { - return gs.obj.LastModified -} - -func (gs *GarageStat) IsDir() bool { - return gs.path.class != OBJECT -} - -func (gs *GarageStat) Sys() interface{} { - return nil -} - -type S3Class int - -const ( - ROOT S3Class = 1 << iota - BUCKET - COMMON_PREFIX - OBJECT - OPAQUE_KEY - - KEY = COMMON_PREFIX | OBJECT | OPAQUE_KEY -) - -type S3Path struct { - path string - class S3Class - bucket string - key string -} - -func NewS3Path(path string) S3Path { - exploded_path := strings.SplitN(path, "/", 3) - - // If there is no bucket name (eg. "/") - if len(exploded_path) < 2 || exploded_path[1] == "" { - return S3Path{path, ROOT, "", ""} - } - - // If there is no key - if len(exploded_path) < 3 || exploded_path[2] == "" { - return S3Path{path, BUCKET, exploded_path[1], ""} - } - - return S3Path{path, OPAQUE_KEY, exploded_path[1], exploded_path[2]} -} - -func NewTrustedS3Path(bucket string, obj minio.ObjectInfo) S3Path { - cl := OBJECT - if obj.Key[len(obj.Key)-1:] == "/" { - cl = COMMON_PREFIX - } - - return S3Path{ - path: path.Join("/", bucket, obj.Key), - bucket: bucket, - key: obj.Key, - class: cl, - } -} diff --git a/middleware.go b/middleware.go new file mode 100644 index 0000000..8688347 --- /dev/null +++ b/middleware.go @@ -0,0 +1,17 @@ +package main + +import ( + "github.com/minio/minio-go/v7" + "net/http" +) + +/* We define some interface to enable our middleware to communicate */ +type ErrorHandler interface { + WithError(err error) http.Handler +} +type CredsHandler interface { + WithCreds(username, password string) http.Handler +} +type MinioClientHandler interface { + WithMC(mc *minio.Client) http.Handler +} diff --git a/s3_file.go b/s3_file.go new file mode 100644 index 0000000..a72397e --- /dev/null +++ b/s3_file.go @@ -0,0 +1,186 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "log" + "mime" + "path" + + "github.com/minio/minio-go/v7" + "golang.org/x/net/webdav" +) + +type S3File struct { + fs *S3FS + obj *minio.Object + objw *io.PipeWriter + donew chan error + pos int64 + path S3Path +} + +func NewS3File(s *S3FS, path string) (webdav.File, error) { + f := new(S3File) + f.fs = s + f.pos = 0 + f.path = NewS3Path(path) + return f, nil +} + +func (f *S3File) Close() error { + err := make([]error, 0) + + if f.obj != nil { + err = append(err, f.obj.Close()) + f.obj = nil + } + + if f.objw != nil { + // wait that minio completes its transfers in background + err = append(err, f.objw.Close()) + err = append(err, <-f.donew) + f.donew = nil + f.objw = nil + } + + count := 0 + for _, e := range err { + if e != nil { + count++ + log.Println(e) + } + } + if count > 0 { + return errors.New(fmt.Sprintf("%d errors when closing this WebDAV File. Read previous logs to know more.", count)) + } + return nil +} + +func (f *S3File) loadObject() error { + if f.obj == nil { + obj, err := f.fs.mc.GetObject(f.fs.ctx, f.path.bucket, f.path.key, minio.GetObjectOptions{}) + if err != nil { + return err + } + f.obj = obj + } + return nil +} + +func (f *S3File) Read(p []byte) (n int, err error) { + //if f.Stat() & OBJECT == 0 { /* @FIXME Ideally we would check against OBJECT but we need a non OPAQUE_KEY */ + // return 0, os.ErrInvalid + //} + if err := f.loadObject(); err != nil { + return 0, err + } + + return f.obj.Read(p) +} + +func (f *S3File) Write(p []byte) (n int, err error) { + /*if f.path.class != OBJECT { + return 0, os.ErrInvalid + }*/ + + if f.objw == nil { + if f.pos != 0 { + return 0, errors.New("writing with an offset is not implemented") + } + + r, w := io.Pipe() + f.donew = make(chan error, 1) + f.objw = w + + contentType := mime.TypeByExtension(path.Ext(f.path.key)) + go func() { + _, err := f.fs.mc.PutObject(context.Background(), f.path.bucket, f.path.key, r, -1, minio.PutObjectOptions{ContentType: contentType}) + f.donew <- err + }() + } + + return f.objw.Write(p) +} + +func (f *S3File) Seek(offset int64, whence int) (int64, error) { + if err := f.loadObject(); err != nil { + return 0, err + } + + pos, err := f.obj.Seek(offset, whence) + f.pos += pos + return pos, err +} + +/* +ReadDir reads the contents of the directory associated with the file f and returns a slice of DirEntry values in directory order. Subsequent calls on the same file will yield later DirEntry records in the directory. + +If n > 0, ReadDir returns at most n DirEntry records. In this case, if ReadDir returns an empty slice, it will return an error explaining why. At the end of a directory, the error is io.EOF. + +If n <= 0, ReadDir returns all the DirEntry records remaining in the directory. When it succeeds, it returns a nil error (not io.EOF). +*/ +func (f *S3File) Readdir(count int) ([]fs.FileInfo, error) { + if count > 0 { + return nil, errors.New("returning a limited number of directory entry is not supported in readdir") + } + + if f.path.class == ROOT { + return f.readDirRoot(count) + } else { + return f.readDirChild(count) + } +} + +func (f *S3File) readDirRoot(count int) ([]fs.FileInfo, error) { + buckets, err := f.fs.mc.ListBuckets(f.fs.ctx) + if err != nil { + return nil, err + } + + entries := make([]fs.FileInfo, 0, len(buckets)) + for _, bucket := range buckets { + //log.Println("Stat from GarageFile.readDirRoot()", "/"+bucket.Name) + nf, err := NewS3Stat(f.fs, "/"+bucket.Name) + if err != nil { + return nil, err + } + entries = append(entries, nf) + } + + return entries, nil +} + +func (f *S3File) readDirChild(count int) ([]fs.FileInfo, error) { + prefix := f.path.key + if len(prefix) > 0 && prefix[len(prefix)-1:] != "/" { + prefix = prefix + "/" + } + + objs_info := f.fs.mc.ListObjects(f.fs.ctx, f.path.bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: false, + }) + + entries := make([]fs.FileInfo, 0) + for object := range objs_info { + if object.Err != nil { + return nil, object.Err + } + //log.Println("Stat from GarageFile.readDirChild()", path.Join("/", f.path.bucket, object.Key)) + nf, err := NewS3StatFromObjectInfo(f.fs, f.path.bucket, object) + if err != nil { + return nil, err + } + entries = append(entries, nf) + } + + return entries, nil +} + +func (f *S3File) Stat() (fs.FileInfo, error) { + return NewS3Stat(f.fs, f.path.path) +} diff --git a/s3_fs.go b/s3_fs.go new file mode 100644 index 0000000..ecade9f --- /dev/null +++ b/s3_fs.go @@ -0,0 +1,66 @@ +package main + +import ( + "context" + "errors" + "os" + "time" + + "github.com/minio/minio-go/v7" + "golang.org/x/net/webdav" +) + +/* + * S3FS lifetime is limited to a single request + * Conversely, Golang's abstraction has been thought to be shared between users + * Sharing an instance between users would be very dangerous (as we would need many checks between shared values) + */ +type S3FS struct { + cache map[string]*S3Stat + mc *minio.Client + ctx context.Context +} + +func NewS3FS(mc *minio.Client) S3FS { + return S3FS{ + cache: make(map[string]*S3Stat), + mc: mc, + } +} + +func (s S3FS) Mkdir(ctx context.Context, name string, perm os.FileMode) error { + s.ctx = ctx + return errors.New("Not implemented Mkdir") +} + +func (s S3FS) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) { + s.ctx = ctx + + // If the file does not exist when opening it, we create a stub + if _, ok := s.cache[name]; !ok { + st := new(S3Stat) + st.fs = &s + st.path = NewS3Path(name) + st.path.class = OBJECT + st.obj.Key = st.path.key + st.obj.LastModified = time.Now() + s.cache[name] = st + } + + return NewS3File(&s, name) +} + +func (s S3FS) RemoveAll(ctx context.Context, name string) error { + s.ctx = ctx + return errors.New("Not implemented RemoveAll") +} + +func (s S3FS) Rename(ctx context.Context, oldName, newName string) error { + s.ctx = ctx + return errors.New("Not implemented Rename") +} + +func (s S3FS) Stat(ctx context.Context, name string) (os.FileInfo, error) { + s.ctx = ctx + return NewS3Stat(&s, name) +} diff --git a/s3_path.go b/s3_path.go new file mode 100644 index 0000000..e1933ef --- /dev/null +++ b/s3_path.go @@ -0,0 +1,57 @@ +package main + +import ( + "path" + "strings" + + "github.com/minio/minio-go/v7" +) + +type S3Class int + +const ( + ROOT S3Class = 1 << iota + BUCKET + COMMON_PREFIX + OBJECT + OPAQUE_KEY + + KEY = COMMON_PREFIX | OBJECT | OPAQUE_KEY +) + +type S3Path struct { + path string + class S3Class + bucket string + key string +} + +func NewS3Path(path string) S3Path { + exploded_path := strings.SplitN(path, "/", 3) + + // If there is no bucket name (eg. "/") + if len(exploded_path) < 2 || exploded_path[1] == "" { + return S3Path{path, ROOT, "", ""} + } + + // If there is no key + if len(exploded_path) < 3 || exploded_path[2] == "" { + return S3Path{path, BUCKET, exploded_path[1], ""} + } + + return S3Path{path, OPAQUE_KEY, exploded_path[1], exploded_path[2]} +} + +func NewTrustedS3Path(bucket string, obj minio.ObjectInfo) S3Path { + cl := OBJECT + if obj.Key[len(obj.Key)-1:] == "/" { + cl = COMMON_PREFIX + } + + return S3Path{ + path: path.Join("/", bucket, obj.Key), + bucket: bucket, + key: obj.Key, + class: cl, + } +} diff --git a/s3_stat.go b/s3_stat.go new file mode 100644 index 0000000..65b6faa --- /dev/null +++ b/s3_stat.go @@ -0,0 +1,128 @@ +package main + +import ( + "errors" + "io/fs" + "path" + "time" + + "github.com/minio/minio-go/v7" +) + +type S3Stat struct { + fs *S3FS + obj minio.ObjectInfo + path S3Path +} + +/* + * Stat a path knowing its ObjectInfo + */ +func NewS3StatFromObjectInfo(fs *S3FS, bucket string, obj minio.ObjectInfo) (*S3Stat, error) { + s := new(S3Stat) + s.path = NewTrustedS3Path(bucket, obj) + s.obj = obj + s.fs = fs + + fs.cache[s.path.path] = s + return s, nil +} + +/* + * Stat a path without additional information + */ +func NewS3Stat(fs *S3FS, path string) (*S3Stat, error) { + cache := fs.cache + if entry, ok := cache[path]; ok { + return entry, nil + } + + s := new(S3Stat) + s.fs = fs + s.path = NewS3Path(path) + if err := s.Refresh(); err != nil { + return nil, err + } + + if s.path.class&OPAQUE_KEY != 0 { + return nil, errors.New("Failed to precisely determine the key type, this a logic error.") + } + + cache[path] = s + cache[s.path.path] = s + return s, nil +} + +func (s *S3Stat) Refresh() error { + if s.path.class == ROOT || s.path.class == BUCKET { + return nil + } + + mc := s.fs.mc + + // Compute the prefix to have the desired behaviour for our stat logic + prefix := s.path.key + if prefix[len(prefix)-1:] == "/" { + prefix = prefix[:len(prefix)-1] + } + + // Get info and check if the key exists + objs_info := mc.ListObjects(s.fs.ctx, s.path.bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: false, + }) + + found := false + for object := range objs_info { + if object.Err != nil { + return object.Err + } + + if object.Key == prefix || object.Key == prefix+"/" { + s.obj = object + s.path = NewTrustedS3Path(s.path.bucket, object) + found = true + break + } + } + + if !found { + return fs.ErrNotExist + } + + return nil +} + +func (s *S3Stat) Name() string { + if s.path.class == ROOT { + return "/" + } else if s.path.class == BUCKET { + return s.path.bucket + } else { + return path.Base(s.path.key) + } +} + +func (s *S3Stat) Size() int64 { + return s.obj.Size +} + +func (s *S3Stat) Mode() fs.FileMode { + if s.path.class == OBJECT { + return fs.ModePerm + } else { + return fs.ModeDir | fs.ModePerm + } +} + +func (s *S3Stat) ModTime() time.Time { + return s.obj.LastModified +} + +func (s *S3Stat) IsDir() bool { + return s.path.class != OBJECT +} + +func (s *S3Stat) Sys() interface{} { + return nil +} diff --git a/webdav.go b/webdav.go new file mode 100644 index 0000000..3f3f7d5 --- /dev/null +++ b/webdav.go @@ -0,0 +1,28 @@ +package main + +import ( + "github.com/minio/minio-go/v7" + "golang.org/x/net/webdav" + "log" + "net/http" +) + +type WebDav struct { + WithConfig *Config +} + +func (wd WebDav) WithMC(mc *minio.Client) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + (&webdav.Handler{ + Prefix: wd.WithConfig.DavPath, + FileSystem: NewS3FS(mc), + LockSystem: webdav.NewMemLS(), + Logger: func(r *http.Request, err error) { + log.Printf("INFO: %s %s %s\n", r.RemoteAddr, r.Method, r.URL) + if err != nil { + log.Printf("ERR: %v", err) + } + }, + }).ServeHTTP(w, r) + }) +} |