mirror of
https://github.com/cwinfo/matterbridge.git
synced 2024-11-12 21:50:26 +00:00
Optimize handling of very large slack teams. Fixes #695
Stop getting users if we reach 2000 users. Slack will rate-limit us
even if we follow their limits.
This means that we now have to lookup every user that says a message
for the first time. This should be less intensive on the API.
This also disables partly fb713ed91b
for now.
ChannelMembers will not be filled.
This commit is contained in:
parent
707db950c8
commit
4ae028fe73
@ -75,8 +75,6 @@ func (b *Bslack) handleSlackClient(messages chan *config.Message) {
|
|||||||
// When we join a channel we update the full list of users as
|
// When we join a channel we update the full list of users as
|
||||||
// well as the information for the channel that we joined as this
|
// well as the information for the channel that we joined as this
|
||||||
// should now tell that we are a member of it.
|
// should now tell that we are a member of it.
|
||||||
b.populateUsers(false)
|
|
||||||
|
|
||||||
b.channelsMutex.Lock()
|
b.channelsMutex.Lock()
|
||||||
b.channelsByID[ev.Channel.ID] = &ev.Channel
|
b.channelsByID[ev.Channel.ID] = &ev.Channel
|
||||||
b.channelsByName[ev.Channel.Name] = &ev.Channel
|
b.channelsByName[ev.Channel.Name] = &ev.Channel
|
||||||
@ -202,7 +200,6 @@ func (b *Bslack) handleMessageEvent(ev *slack.MessageEvent) (*config.Message, er
|
|||||||
func (b *Bslack) handleStatusEvent(ev *slack.MessageEvent, rmsg *config.Message) bool {
|
func (b *Bslack) handleStatusEvent(ev *slack.MessageEvent, rmsg *config.Message) bool {
|
||||||
switch ev.SubType {
|
switch ev.SubType {
|
||||||
case sChannelJoined, sMemberJoined:
|
case sChannelJoined, sMemberJoined:
|
||||||
b.populateUsers(false)
|
|
||||||
// There's no further processing needed on channel events
|
// There's no further processing needed on channel events
|
||||||
// so we return 'true'.
|
// so we return 'true'.
|
||||||
return true
|
return true
|
||||||
|
@ -12,6 +12,13 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func (b *Bslack) getUser(id string) *slack.User {
|
func (b *Bslack) getUser(id string) *slack.User {
|
||||||
|
b.usersMutex.RLock()
|
||||||
|
user, ok := b.users[id]
|
||||||
|
b.usersMutex.RUnlock()
|
||||||
|
if ok {
|
||||||
|
return user
|
||||||
|
}
|
||||||
|
b.populateUser(id)
|
||||||
b.usersMutex.RLock()
|
b.usersMutex.RLock()
|
||||||
defer b.usersMutex.RUnlock()
|
defer b.usersMutex.RUnlock()
|
||||||
|
|
||||||
@ -102,9 +109,11 @@ func (b *Bslack) populateUsers(wait bool) {
|
|||||||
|
|
||||||
newUsers := map[string]*slack.User{}
|
newUsers := map[string]*slack.User{}
|
||||||
pagination := b.sc.GetUsersPaginated(slack.GetUsersOptionLimit(200))
|
pagination := b.sc.GetUsersPaginated(slack.GetUsersOptionLimit(200))
|
||||||
|
count := 0
|
||||||
for {
|
for {
|
||||||
var err error
|
var err error
|
||||||
pagination, err = pagination.Next(context.Background())
|
pagination, err = pagination.Next(context.Background())
|
||||||
|
time.Sleep(time.Second)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if pagination.Done(err) {
|
if pagination.Done(err) {
|
||||||
break
|
break
|
||||||
@ -120,6 +129,13 @@ func (b *Bslack) populateUsers(wait bool) {
|
|||||||
for i := range pagination.Users {
|
for i := range pagination.Users {
|
||||||
newUsers[pagination.Users[i].ID] = &pagination.Users[i]
|
newUsers[pagination.Users[i].ID] = &pagination.Users[i]
|
||||||
}
|
}
|
||||||
|
b.Log.Debugf("getting %d users", len(pagination.Users))
|
||||||
|
count++
|
||||||
|
// more > 2000 users, slack will complain and ratelimit. break
|
||||||
|
if count > 10 {
|
||||||
|
b.Log.Info("Large slack detected > 2000 users, skipping loading complete userlist.")
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
b.usersMutex.Lock()
|
b.usersMutex.Lock()
|
||||||
@ -172,15 +188,18 @@ func (b *Bslack) populateChannels(wait bool) {
|
|||||||
newChannelsByID[channels[i].ID] = &channels[i]
|
newChannelsByID[channels[i].ID] = &channels[i]
|
||||||
newChannelsByName[channels[i].Name] = &channels[i]
|
newChannelsByName[channels[i].Name] = &channels[i]
|
||||||
// also find all the members in every channel
|
// also find all the members in every channel
|
||||||
members, err := b.getUsersInConversation(channels[i].ID)
|
// comment for now, issues on big slacks
|
||||||
if err != nil {
|
/*
|
||||||
if err = b.handleRateLimit(err); err != nil {
|
members, err := b.getUsersInConversation(channels[i].ID)
|
||||||
b.Log.Errorf("Could not retrieve channel members: %#v", err)
|
if err != nil {
|
||||||
return
|
if err = b.handleRateLimit(err); err != nil {
|
||||||
|
b.Log.Errorf("Could not retrieve channel members: %#v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
continue
|
newChannelMembers[channels[i].ID] = members
|
||||||
}
|
*/
|
||||||
newChannelMembers[channels[i].ID] = members
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if nextCursor == "" {
|
if nextCursor == "" {
|
||||||
|
Loading…
Reference in New Issue
Block a user