Skip to content
This repository has been archived by the owner on Jan 27, 2021. It is now read-only.

Extract persistence layer in to own package [WIP] #124

Closed
wants to merge 2 commits into from
Closed

Extract persistence layer in to own package [WIP] #124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e3dab3f
Extract persistence layer in to own package
IljaN Aug 21, 2020
77276e5
CS3 Repo WIP
IljaN Sep 17, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 8 additions & 59 deletions pkg/service/v0/accounts.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,7 @@ package service

import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"regexp"
Expand Down Expand Up @@ -34,7 +32,6 @@ import (
_ "github.com/tredoe/osutil/user/crypt/sha512_crypt"
)

// accLock mutually exclude readers from writers on account files
var accLock sync.Mutex

func (s Service) indexAccounts(path string) (err error) {
Expand Down Expand Up @@ -63,7 +60,7 @@ func (s Service) indexAccount(id string) error {
a := &proto.BleveAccount{
BleveType: "account",
}
if err := s.loadAccount(id, &a.Account); err != nil {
if err := s.repo.LoadAccount(id, nil, &a.Account); err != nil {
s.log.Error().Err(err).Str("account", id).Msg("could not load account")
return err
}
Expand All @@ -79,37 +76,6 @@ func (s Service) indexAccount(id string) error {
// login eq \"teddy\" and password eq \"F&1!b90t111!\"
var authQuery = regexp.MustCompile(`^login eq '(.*)' and password eq '(.*)'$`) // TODO how is ' escaped in the password?

func (s Service) loadAccount(id string, a *proto.Account) (err error) {
path := filepath.Join(s.Config.Server.AccountsDataPath, "accounts", id)

var data []byte
if data, err = ioutil.ReadFile(path); err != nil {
return merrors.NotFound(s.id, "could not read account: %v", err.Error())
}

if err = json.Unmarshal(data, a); err != nil {
return merrors.InternalServerError(s.id, "could not unmarshal account: %v", err.Error())
}
return
}

func (s Service) writeAccount(a *proto.Account) (err error) {
// leave only the group id
s.deflateMemberOf(a)

var bytes []byte
if bytes, err = json.Marshal(a); err != nil {
return merrors.InternalServerError(s.id, "could not marshal account: %v", err.Error())
}

path := filepath.Join(s.Config.Server.AccountsDataPath, "accounts", a.Id)

if err = ioutil.WriteFile(path, bytes, 0600); err != nil {
return merrors.InternalServerError(s.id, "could not write account: %v", err.Error())
}
return
}

func (s Service) expandMemberOf(a *proto.Account) {
if a == nil {
return
Expand All @@ -118,7 +84,7 @@ func (s Service) expandMemberOf(a *proto.Account) {
for i := range a.MemberOf {
g := &proto.Group{}
// TODO resolve by name, when a create or update is issued they may not have an id? fall back to searching the group id in the index?
if err := s.loadGroup(a.MemberOf[i].Id, g); err == nil {
if err := s.repo.LoadGroup(a.MemberOf[i].Id, nil, g); err == nil {
g.Members = nil // always hide members when expanding
expanded = append(expanded, g)
} else {
Expand All @@ -129,23 +95,6 @@ func (s Service) expandMemberOf(a *proto.Account) {
a.MemberOf = expanded
}

// deflateMemberOf replaces the groups of a user with an instance that only contains the id
func (s Service) deflateMemberOf(a *proto.Account) {
if a == nil {
return
}
deflated := []*proto.Group{}
for i := range a.MemberOf {
if a.MemberOf[i].Id != "" {
deflated = append(deflated, &proto.Group{Id: a.MemberOf[i].Id})
} else {
// TODO fetch and use an id when group only has a name but no id
s.log.Error().Str("id", a.Id).Interface("group", a.MemberOf[i]).Msg("resolving groups by name is not implemented yet")
}
}
a.MemberOf = deflated
}

func (s Service) passwordIsValid(hash string, pwd string) (ok bool) {
defer func() {
if r := recover(); r != nil {
Expand Down Expand Up @@ -268,7 +217,7 @@ func (s Service) ListAccounts(ctx context.Context, in *proto.ListAccountsRequest

for _, hit := range searchResult.Hits {
a := &proto.Account{}
if err = s.loadAccount(hit.ID, a); err != nil {
if err = s.repo.LoadAccount(hit.ID, nil, a); err != nil {
s.log.Error().Err(err).Str("account", hit.ID).Msg("could not load account, skipping")
continue
}
Expand Down Expand Up @@ -316,7 +265,7 @@ func (s Service) GetAccount(ctx context.Context, in *proto.GetAccountRequest, ou
return merrors.InternalServerError(s.id, "could not clean up account id: %v", err.Error())
}

if err = s.loadAccount(id, out); err != nil {
if err = s.repo.LoadAccount(id, nil, out); err != nil {
s.log.Error().Err(err).Str("id", id).Msg("could not load account")
return
}
Expand Down Expand Up @@ -389,7 +338,7 @@ func (s Service) CreateAccount(ctx context.Context, in *proto.CreateAccountReque
// TODO groups should be ignored during create, use groups.AddMember? return error?

// write and index account - note: don't do anything else in between!
if err = s.writeAccount(acc); err != nil {
if err = s.repo.WriteAccount(nil, acc); err != nil {
s.log.Error().Err(err).Str("id", id).Msg("could not persist new account")
s.debugLogAccount(acc).Msg("could not persist new account")
return
Expand Down Expand Up @@ -450,7 +399,7 @@ func (s Service) UpdateAccount(ctx context.Context, in *proto.UpdateAccountReque

path := filepath.Join(s.Config.Server.AccountsDataPath, "accounts", id)

if err = s.loadAccount(id, out); err != nil {
if err = s.repo.LoadAccount(id, nil, out); err != nil {
s.log.Error().Err(err).Str("id", id).Msg("could not load account")
return
}
Expand Down Expand Up @@ -504,7 +453,7 @@ func (s Service) UpdateAccount(ctx context.Context, in *proto.UpdateAccountReque
out.ExternalUserStateChangeDateTime = tsnow
}

if err = s.writeAccount(out); err != nil {
if err = s.repo.WriteAccount(nil, out); err != nil {
s.log.Error().Err(err).Str("id", out.Id).Msg("could not persist updated account")
return
}
Expand Down Expand Up @@ -556,7 +505,7 @@ func (s Service) DeleteAccount(ctx context.Context, in *proto.DeleteAccountReque
path := filepath.Join(s.Config.Server.AccountsDataPath, "accounts", id)

a := &proto.Account{}
if err = s.loadAccount(id, a); err != nil {
if err = s.repo.LoadAccount(id, nil, a); err != nil {
s.log.Error().Err(err).Str("id", id).Msg("could not load account")
return
}
Expand Down
91 changes: 15 additions & 76 deletions pkg/service/v0/groups.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,8 @@ package service

import (
"context"
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"sync"

"github.com/CiscoM31/godata"
"github.com/blevesearch/bleve"
Expand Down Expand Up @@ -46,7 +43,7 @@ func (s Service) indexGroup(id string) error {
g := &proto.BleveGroup{
BleveType: "group",
}
if err := s.loadGroup(id, &g.Group); err != nil {
if err := s.repo.LoadGroup(id, nil, &g.Group); err != nil {
s.log.Error().Err(err).Str("group", id).Msg("could not load group")
return err
}
Expand All @@ -58,43 +55,6 @@ func (s Service) indexGroup(id string) error {
return nil
}

func (s Service) loadGroup(id string, g *proto.Group) (err error) {
path := filepath.Join(s.Config.Server.AccountsDataPath, "groups", id)

groupLock.Lock()
defer groupLock.Unlock()
var data []byte
if data, err = ioutil.ReadFile(path); err != nil {
return merrors.NotFound(s.id, "could not read group: %v", err.Error())
}

if err = json.Unmarshal(data, g); err != nil {
return merrors.InternalServerError(s.id, "could not unmarshal group: %v", err.Error())
}

return
}

func (s Service) writeGroup(g *proto.Group) (err error) {

// leave only the member id
s.deflateMembers(g)

var bytes []byte
if bytes, err = json.Marshal(g); err != nil {
return merrors.InternalServerError(s.id, "could not marshal group: %v", err.Error())
}

path := filepath.Join(s.Config.Server.AccountsDataPath, "groups", g.Id)

groupLock.Lock()
defer groupLock.Unlock()
if err = ioutil.WriteFile(path, bytes, 0600); err != nil {
return merrors.InternalServerError(s.id, "could not write group: %v", err.Error())
}
return
}

func (s Service) expandMembers(g *proto.Group) {
if g == nil {
return
Expand All @@ -103,7 +63,7 @@ func (s Service) expandMembers(g *proto.Group) {
for i := range g.Members {
// TODO resolve by name, when a create or update is issued they may not have an id? fall back to searching the group id in the index?
a := &proto.Account{}
if err := s.loadAccount(g.Members[i].Id, a); err == nil {
if err := s.repo.LoadAccount(g.Members[i].Id, nil, a); err == nil {
expanded = append(expanded, a)
} else {
// log errors but continue execution for now
Expand All @@ -113,23 +73,6 @@ func (s Service) expandMembers(g *proto.Group) {
g.Members = expanded
}

// deflateMembers replaces the users of a group with an instance that only contains the id
func (s Service) deflateMembers(g *proto.Group) {
if g == nil {
return
}
deflated := []*proto.Account{}
for i := range g.Members {
if g.Members[i].Id != "" {
deflated = append(deflated, &proto.Account{Id: g.Members[i].Id})
} else {
// TODO fetch and use an id when group only has a name but no id
s.log.Error().Str("id", g.Id).Interface("account", g.Members[i]).Msg("resolving members by name is not implemented yet")
}
}
g.Members = deflated
}

// ListGroups implements the GroupsServiceHandler interface
func (s Service) ListGroups(c context.Context, in *proto.ListGroupsRequest, out *proto.ListGroupsResponse) (err error) {

Expand Down Expand Up @@ -173,7 +116,7 @@ func (s Service) ListGroups(c context.Context, in *proto.ListGroupsRequest, out
for _, hit := range searchResult.Hits {

g := &proto.Group{}
if err = s.loadGroup(hit.ID, g); err != nil {
if err = s.repo.LoadGroup(hit.ID, nil, g); err != nil {
s.log.Error().Err(err).Str("group", hit.ID).Msg("could not load group, skipping")
continue
}
Expand All @@ -196,7 +139,7 @@ func (s Service) GetGroup(c context.Context, in *proto.GetGroupRequest, out *pro
return merrors.InternalServerError(s.id, "could not clean up group id: %v", err.Error())
}

if err = s.loadGroup(id, out); err != nil {
if err = s.repo.LoadGroup(id, nil, out); err != nil {
s.log.Error().Err(err).Str("id", id).Msg("could not load group")
return
}
Expand All @@ -223,10 +166,7 @@ func (s Service) CreateGroup(c context.Context, in *proto.CreateGroupRequest, ou
return merrors.InternalServerError(s.id, "could not clean up account id: %v", err.Error())
}

// extract member id
s.deflateMembers(in.Group)

if err = s.writeGroup(in.Group); err != nil {
if err = s.repo.WriteGroup(in.Group, nil); err != nil {
s.log.Error().Err(err).Interface("group", in.Group).Msg("could not persist new group")
return
}
Expand All @@ -252,7 +192,7 @@ func (s Service) DeleteGroup(c context.Context, in *proto.DeleteGroupRequest, ou
path := filepath.Join(s.Config.Server.AccountsDataPath, "groups", id)

g := &proto.Group{}
if err = s.loadGroup(id, g); err != nil {
if err = s.repo.LoadGroup(id, nil, g); err != nil {
s.log.Error().Err(err).Str("id", id).Msg("could not load account")
return
}
Expand Down Expand Up @@ -297,13 +237,13 @@ func (s Service) AddMember(c context.Context, in *proto.AddMemberRequest, out *p

// load structs
a := &proto.Account{}
if err = s.loadAccount(accountID, a); err != nil {
if err = s.repo.LoadAccount(accountID, nil, a); err != nil {
s.log.Error().Err(err).Str("id", accountID).Msg("could not load account")
return
}

g := &proto.Group{}
if err = s.loadGroup(groupID, g); err != nil {
if err = s.repo.LoadGroup(groupID, nil, g); err != nil {
s.log.Error().Err(err).Str("id", groupID).Msg("could not load group")
return
}
Expand Down Expand Up @@ -331,11 +271,11 @@ func (s Service) AddMember(c context.Context, in *proto.AddMemberRequest, out *p
a.MemberOf = append(a.MemberOf, g)
}

if err = s.writeAccount(a); err != nil {
if err = s.repo.WriteAccount(nil, a); err != nil {
s.log.Error().Err(err).Interface("account", a).Msg("could not persist account")
return
}
if err = s.writeGroup(g); err != nil {
if err = s.repo.WriteGroup(g, nil); err != nil {
s.log.Error().Err(err).Interface("group", g).Msg("could not persist group")
return
}
Expand All @@ -362,13 +302,13 @@ func (s Service) RemoveMember(c context.Context, in *proto.RemoveMemberRequest,

// load structs
a := &proto.Account{}
if err = s.loadAccount(accountID, a); err != nil {
if err = s.repo.LoadAccount(accountID, nil, a); err != nil {
s.log.Error().Err(err).Str("id", accountID).Msg("could not load account")
return
}

g := &proto.Group{}
if err = s.loadGroup(groupID, g); err != nil {
if err = s.repo.LoadGroup(groupID, nil, g); err != nil {
s.log.Error().Err(err).Str("id", groupID).Msg("could not load group")
return
}
Expand All @@ -391,11 +331,11 @@ func (s Service) RemoveMember(c context.Context, in *proto.RemoveMemberRequest,
}
a.MemberOf = newGroups

if err = s.writeAccount(a); err != nil {
if err = s.repo.WriteAccount(nil, a); err != nil {
s.log.Error().Err(err).Interface("account", a).Msg("could not persist account")
return
}
if err = s.writeGroup(g); err != nil {
if err = s.repo.WriteGroup(g, nil); err != nil {
s.log.Error().Err(err).Interface("group", g).Msg("could not persist group")
return
}
Expand All @@ -408,15 +348,14 @@ func (s Service) RemoveMember(c context.Context, in *proto.RemoveMemberRequest,

// ListMembers implements the GroupsServiceHandler interface
func (s Service) ListMembers(c context.Context, in *proto.ListMembersRequest, out *proto.ListMembersResponse) (err error) {

// cleanup ids
var groupID string
if groupID, err = cleanupID(in.Id); err != nil {
return merrors.InternalServerError(s.id, "could not clean up group id: %v", err.Error())
}

g := &proto.Group{}
if err = s.loadGroup(groupID, g); err != nil {
if err = s.repo.LoadGroup(groupID, nil, g); err != nil {
s.log.Error().Err(err).Str("id", groupID).Msg("could not load group")
return
}
Expand Down
Loading