Add LRU cache for some DB operations

This commit is contained in:
Alex 2020-02-28 11:06:43 +01:00
parent 21dc026a81
commit d03091dd01
2 changed files with 93 additions and 37 deletions

125
db.go
View file

@ -4,6 +4,7 @@ import (
"fmt" "fmt"
"sync" "sync"
"github.com/hashicorp/golang-lru"
"github.com/jinzhu/gorm" "github.com/jinzhu/gorm"
_ "github.com/jinzhu/gorm/dialects/mysql" _ "github.com/jinzhu/gorm/dialects/mysql"
_ "github.com/jinzhu/gorm/dialects/postgres" _ "github.com/jinzhu/gorm/dialects/postgres"
@ -16,6 +17,7 @@ import (
) )
var db *gorm.DB var db *gorm.DB
var dbCache *lru.TwoQueueCache
func InitDb() error { func InitDb() error {
var err error var err error
@ -41,6 +43,11 @@ func InitDb() error {
db.AutoMigrate(&DbJoinedRoom{}) db.AutoMigrate(&DbJoinedRoom{})
db.Model(&DbJoinedRoom{}).AddIndex("idx_user_protocol_account", "mx_user_id", "protocol", "account_name") db.Model(&DbJoinedRoom{}).AddIndex("idx_user_protocol_account", "mx_user_id", "protocol", "account_name")
dbCache, err = lru.New2Q(10000)
if err != nil {
return err
}
return nil return nil
} }
@ -54,12 +61,17 @@ type DbAccountConfig struct {
Config string Config string
} }
// Long-term cache entries // List of joined channels to be re-joined on reconnect
type DbKv struct { type DbJoinedRoom struct {
gorm.Model gorm.Model
Key string `gorm:"unique_index"` // User id and account name
Value string MxUserID string
Protocol string
AccountName string
// Room ID
RoomID connector.RoomID
} }
// User mapping between protocol user IDs and puppeted matrix ids // User mapping between protocol user IDs and puppeted matrix ids
@ -101,20 +113,18 @@ type DbPmRoomMap struct {
MxRoomID string `gorm:"index:mxroomoid"` MxRoomID string `gorm:"index:mxroomoid"`
} }
// List of joined channels to be re-joined on reconnect // Key-value store for various things
type DbJoinedRoom struct { type DbKv struct {
gorm.Model gorm.Model
// User id and account name Key string `gorm:"unique_index"`
MxUserID string Value string
Protocol string
AccountName string
// Room ID
RoomID connector.RoomID
} }
// ---- Simple locking mechanism // ---- Simple locking mechanism
// Slot keys are strings that identify the object we are acting upon
// They define which lock to lock for a certain operation
// They are also used as keys in the LRU cache
var dbLocks [256]sync.Mutex var dbLocks [256]sync.Mutex
@ -128,9 +138,19 @@ func dbUnlockSlot(key string) {
dbLocks[slot].Unlock() dbLocks[slot].Unlock()
} }
// ---- // ---- Key-value store supporting atomic test-and-set
func dbKvSlotKey(key string) string {
return "kv:" + key
}
func dbKvGet(key string) string { func dbKvGet(key string) string {
slot_key := dbKvSlotKey(key)
if ent, ok := dbCache.Get(slot_key); ok {
return ent.(string)
}
var entry DbKv var entry DbKv
if db.Where(&DbKv{Key: key}).First(&entry).RecordNotFound() { if db.Where(&DbKv{Key: key}).First(&entry).RecordNotFound() {
return "" return ""
@ -140,35 +160,53 @@ func dbKvGet(key string) string {
} }
func dbKvPut(key string, value string) { func dbKvPut(key string, value string) {
var entry DbKv slot_key := dbKvSlotKey(key)
db.Where(&DbKv{Key: key}).Assign(&DbKv{Value: value}).FirstOrCreate(&entry)
}
func dbKvTestAndSet(key string, value string) bool {
dbLockSlot(key)
defer dbUnlockSlot(key)
// True if value was changed, false if was already set
if dbKvGet(key) != value {
dbKvPut(key, value)
return true
}
return false
}
func dbGetMxRoom(protocol string, roomId connector.RoomID) (string, error) {
slot_key := fmt.Sprintf("room: %s / %s", protocol, roomId)
dbLockSlot(slot_key) dbLockSlot(slot_key)
defer dbUnlockSlot(slot_key) defer dbUnlockSlot(slot_key)
var room DbRoomMap var entry DbKv
db.Where(&DbKv{Key: key}).Assign(&DbKv{Value: value}).FirstOrCreate(&entry)
dbCache.Add(slot_key, value)
}
func dbKvTestAndSet(key string, value string) bool {
slot_key := dbKvSlotKey(key)
dbLockSlot(slot_key)
defer dbUnlockSlot(slot_key)
// True if value was changed, false if was already set
if dbKvGet(key) == value {
return false
}
var entry DbKv
db.Where(&DbKv{Key: key}).Assign(&DbKv{Value: value}).FirstOrCreate(&entry)
dbCache.Add(slot_key, value)
return true
}
// ----
func dbGetMxRoom(protocol string, roomId connector.RoomID) (string, error) {
slot_key := fmt.Sprintf("room:%s/%s", protocol, roomId)
dbLockSlot(slot_key)
defer dbUnlockSlot(slot_key)
if cached, ok := dbCache.Get(slot_key); ok {
return cached.(string), nil
}
// Check if room exists in our mapping, // Check if room exists in our mapping,
// If not create it // If not create it
var room DbRoomMap
must_create := db.First(&room, DbRoomMap{ must_create := db.First(&room, DbRoomMap{
Protocol: protocol, Protocol: protocol,
RoomID: roomId, RoomID: roomId,
}).RecordNotFound() }).RecordNotFound()
if must_create { if must_create {
alias := roomAlias(protocol, roomId) alias := roomAlias(protocol, roomId)
// Lookup alias // Lookup alias
@ -192,24 +230,31 @@ func dbGetMxRoom(protocol string, roomId connector.RoomID) (string, error) {
} }
db.Create(&room) db.Create(&room)
} }
log.Tracef("Got room id for %s %s: %s", protocol, roomId, room.MxRoomID)
log.Tracef("%s -> %s", slot_key, room.MxRoomID)
dbCache.Add(slot_key, room.MxRoomID)
return room.MxRoomID, nil return room.MxRoomID, nil
} }
func dbGetMxPmRoom(protocol string, them connector.UserID, themMxId string, usMxId string, usAccount string) (string, error) { func dbGetMxPmRoom(protocol string, them connector.UserID, themMxId string, usMxId string, usAccount string) (string, error) {
slot_key := fmt.Sprintf("pmroom:%s/%s/%s/%s", protocol, usMxId, usAccount, them) slot_key := fmt.Sprintf("pmroom:%s/%s/%s/%s", protocol, usMxId, usAccount, them)
dbLockSlot(slot_key) dbLockSlot(slot_key)
defer dbUnlockSlot(slot_key) defer dbUnlockSlot(slot_key)
var room DbPmRoomMap if cached, ok := dbCache.Get(slot_key); ok {
return cached.(string), nil
}
var room DbPmRoomMap
must_create := db.First(&room, DbPmRoomMap{ must_create := db.First(&room, DbPmRoomMap{
MxUserID: usMxId, MxUserID: usMxId,
Protocol: protocol, Protocol: protocol,
AccountName: usAccount, AccountName: usAccount,
UserID: them, UserID: them,
}).RecordNotFound() }).RecordNotFound()
if must_create { if must_create {
name := fmt.Sprintf("%s (%s)", them, protocol) name := fmt.Sprintf("%s (%s)", them, protocol)
@ -234,16 +279,23 @@ func dbGetMxPmRoom(protocol string, them connector.UserID, themMxId string, usMx
} }
db.Create(&room) db.Create(&room)
} }
log.Tracef("Got PM room id for %s %s %s %s: %s", usMxId, protocol, usAccount, them, room.MxRoomID)
log.Tracef("%s -> %s", slot_key, room.MxRoomID)
dbCache.Add(slot_key, room.MxRoomID)
return room.MxRoomID, nil return room.MxRoomID, nil
} }
func dbGetMxUser(protocol string, userId connector.UserID) (string, error) { func dbGetMxUser(protocol string, userId connector.UserID) (string, error) {
slot_key := fmt.Sprintf("user:%s/%s", protocol, userId) slot_key := fmt.Sprintf("user:%s/%s", protocol, userId)
dbLockSlot(slot_key) dbLockSlot(slot_key)
defer dbUnlockSlot(slot_key) defer dbUnlockSlot(slot_key)
if cached, ok := dbCache.Get(slot_key); ok {
return cached.(string), nil
}
var user DbUserMap var user DbUserMap
must_create := db.First(&user, DbUserMap{ must_create := db.First(&user, DbUserMap{
@ -272,6 +324,9 @@ func dbGetMxUser(protocol string, userId connector.UserID) (string, error) {
db.Create(&user) db.Create(&user)
} }
log.Tracef("%s -> %s", slot_key, user.MxUserID)
dbCache.Add(slot_key, user.MxUserID)
return user.MxUserID, nil return user.MxUserID, nil
} }

1
go.mod
View file

@ -10,6 +10,7 @@ require (
github.com/go-ldap/ldap/v3 v3.1.7 github.com/go-ldap/ldap/v3 v3.1.7
github.com/gorilla/mux v1.7.4 github.com/gorilla/mux v1.7.4
github.com/gorilla/sessions v1.2.0 github.com/gorilla/sessions v1.2.0
github.com/hashicorp/golang-lru v0.5.3
github.com/jinzhu/gorm v1.9.12 github.com/jinzhu/gorm v1.9.12
github.com/lrstanley/girc v0.0.0-20190801035559-4fc93959e1a7 github.com/lrstanley/girc v0.0.0-20190801035559-4fc93959e1a7
github.com/matterbridge/go-xmpp v0.0.0-20180529212104-cd19799fba91 github.com/matterbridge/go-xmpp v0.0.0-20180529212104-cd19799fba91