Update go packages; remove db cache
This commit is contained in:
parent
a688840fbf
commit
d3ab4fa8ca
7 changed files with 1162 additions and 84 deletions
6
Makefile
6
Makefile
|
|
@ -21,3 +21,9 @@ docker: $(BIN)
|
|||
docker push $(DOCKER):$(TAG)
|
||||
docker tag $(DOCKER):$(TAG) $(DOCKER):latest
|
||||
docker push $(DOCKER):latest
|
||||
|
||||
docker_clean: $(BIN)
|
||||
docker build --no-cache -t $(DOCKER):$(TAG) .
|
||||
docker push $(DOCKER):$(TAG)
|
||||
docker tag $(DOCKER):$(TAG) $(DOCKER):latest
|
||||
docker push $(DOCKER):latest
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/42wim/matterbridge/matterclient"
|
||||
"github.com/mattermost/mattermost-server/model"
|
||||
"github.com/mattermost/mattermost-server/v5/model"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
. "git.deuxfleurs.fr/Deuxfleurs/easybridge/connector"
|
||||
|
|
|
|||
84
db.go
84
db.go
|
|
@ -4,7 +4,6 @@ import (
|
|||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/hashicorp/golang-lru"
|
||||
"github.com/jinzhu/gorm"
|
||||
_ "github.com/jinzhu/gorm/dialects/mysql"
|
||||
_ "github.com/jinzhu/gorm/dialects/postgres"
|
||||
|
|
@ -17,7 +16,6 @@ import (
|
|||
)
|
||||
|
||||
var db *gorm.DB
|
||||
var dbCache *lru.TwoQueueCache
|
||||
|
||||
func InitDb() error {
|
||||
var err error
|
||||
|
|
@ -39,15 +37,10 @@ func InitDb() error {
|
|||
db.Model(&DbRoomMap{}).AddIndex("idx_room_map_protocol_room", "protocol", "room_id")
|
||||
|
||||
db.AutoMigrate(&DbPmRoomMap{})
|
||||
db.Model(&DbPmRoomMap{}).AddIndex("idx_pm_room_map_protocol_user_account_user", "protocol", "user_id", "mx_user_id", "account_name")
|
||||
db.Model(&DbPmRoomMap{}).AddIndex("idx_pm_room_map_protocol_user_account_user", "protocol", "mx_user_id", "account_name", "user_id")
|
||||
|
||||
db.AutoMigrate(&DbKv{})
|
||||
|
||||
dbCache, err = lru.New2Q(10000)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -63,7 +56,7 @@ type DbAccountConfig struct {
|
|||
|
||||
// List of joined channels to be re-joined on reconnect
|
||||
type DbJoinedRoom struct {
|
||||
ID uint `gorm:"primary_key"`
|
||||
ID uint `gorm:"primaryKey;autoIncrement:true"`
|
||||
|
||||
// User id and account name
|
||||
MxUserID string
|
||||
|
|
@ -76,22 +69,23 @@ type DbJoinedRoom struct {
|
|||
|
||||
// User mapping between protocol user IDs and puppeted matrix ids
|
||||
type DbUserMap struct {
|
||||
ID uint `gorm:"primary_key"`
|
||||
ID uint `gorm:"primaryKey;autoIncrement:true"`
|
||||
|
||||
// Protocol and user ID on the bridged network
|
||||
Protocol string
|
||||
UserID connector.UserID
|
||||
|
||||
// Puppetted Matrix ID
|
||||
MxUserID string `gorm:"index"`
|
||||
}
|
||||
|
||||
// Room mapping between Matrix rooms and outside rooms
|
||||
type DbRoomMap struct {
|
||||
ID uint `gorm:"primary_key"`
|
||||
ID uint `gorm:"primaryKey;autoIncrement:true"`
|
||||
|
||||
// Network protocol
|
||||
// Network protocol and room ID on the bridged network
|
||||
Protocol string
|
||||
|
||||
// Room id on the bridged network
|
||||
RoomID connector.RoomID
|
||||
RoomID connector.RoomID
|
||||
|
||||
// Bridged room matrix id
|
||||
MxRoomID string `gorm:"index"`
|
||||
|
|
@ -99,14 +93,14 @@ type DbRoomMap struct {
|
|||
|
||||
// Room mapping between Matrix rooms and private messages
|
||||
type DbPmRoomMap struct {
|
||||
ID uint `gorm:"primary_key"`
|
||||
ID uint `gorm:"primaryKey;autoIncrement:true"`
|
||||
|
||||
// User id and account name of the local end viewed on Matrix
|
||||
MxUserID string
|
||||
Protocol string
|
||||
AccountName string
|
||||
|
||||
// User id to reach them
|
||||
// User id to reach them on the bridged network
|
||||
UserID connector.UserID
|
||||
|
||||
// Bridged room for PMs
|
||||
|
|
@ -115,14 +109,13 @@ type DbPmRoomMap struct {
|
|||
|
||||
// Key-value store for various things
|
||||
type DbKv struct {
|
||||
Key string `gorm:"primary_key"`
|
||||
Key string `gorm:"primaryKey"`
|
||||
Value string
|
||||
}
|
||||
|
||||
// ---- Simple locking mechanism
|
||||
// Slot keys are strings that identify the object we are acting upon
|
||||
// They define which lock to lock for a certain operation
|
||||
// They are also used as keys in the LRU cache
|
||||
|
||||
var dbLocks [256]sync.Mutex
|
||||
|
||||
|
|
@ -138,17 +131,7 @@ func dbUnlockSlot(key string) {
|
|||
|
||||
// ---- Key-value store supporting atomic test-and-set
|
||||
|
||||
func dbKvSlotKey(key string) string {
|
||||
return "kv:" + key
|
||||
}
|
||||
|
||||
func dbKvGet(key string) string {
|
||||
slot_key := dbKvSlotKey(key)
|
||||
|
||||
if ent, ok := dbCache.Get(slot_key); ok {
|
||||
return ent.(string)
|
||||
}
|
||||
|
||||
var entry DbKv
|
||||
if db.Where(&DbKv{Key: key}).First(&entry).RecordNotFound() {
|
||||
return ""
|
||||
|
|
@ -158,10 +141,8 @@ func dbKvGet(key string) string {
|
|||
}
|
||||
|
||||
func dbKvPut(key string, value string) {
|
||||
slot_key := dbKvSlotKey(key)
|
||||
|
||||
dbLockSlot(slot_key)
|
||||
defer dbUnlockSlot(slot_key)
|
||||
dbLockSlot(key)
|
||||
defer dbUnlockSlot(key)
|
||||
|
||||
dbKvPutLocked(key, value)
|
||||
}
|
||||
|
|
@ -169,18 +150,13 @@ func dbKvPut(key string, value string) {
|
|||
// Variant of dbKvPut that does not take a lock,
|
||||
// use this if the slot is already locked
|
||||
func dbKvPutLocked(key string, value string) {
|
||||
slot_key := dbKvSlotKey(key)
|
||||
|
||||
var entry DbKv
|
||||
db.Where(&DbKv{Key: key}).Assign(&DbKv{Value: value}).FirstOrCreate(&entry)
|
||||
dbCache.Add(slot_key, value)
|
||||
}
|
||||
|
||||
func dbKvTestAndSet(key string, value string) bool {
|
||||
slot_key := dbKvSlotKey(key)
|
||||
|
||||
dbLockSlot(slot_key)
|
||||
defer dbUnlockSlot(slot_key)
|
||||
dbLockSlot(key)
|
||||
defer dbUnlockSlot(key)
|
||||
|
||||
// True if value was changed, false if was already set
|
||||
if dbKvGet(key) == value {
|
||||
|
|
@ -199,10 +175,6 @@ func dbGetMxRoom(protocol string, roomId connector.RoomID) (string, error) {
|
|||
dbLockSlot(slot_key)
|
||||
defer dbUnlockSlot(slot_key)
|
||||
|
||||
if cached, ok := dbCache.Get(slot_key); ok {
|
||||
return cached.(string), nil
|
||||
}
|
||||
|
||||
// Check if room exists in our mapping,
|
||||
// If not create it
|
||||
var room DbRoomMap
|
||||
|
|
@ -239,16 +211,10 @@ func dbGetMxRoom(protocol string, roomId connector.RoomID) (string, error) {
|
|||
}
|
||||
|
||||
log.Tracef("%s -> %s", slot_key, room.MxRoomID)
|
||||
dbCache.Add(slot_key, room.MxRoomID)
|
||||
|
||||
return room.MxRoomID, nil
|
||||
}
|
||||
|
||||
func dbPmRoomSlotKey(room *DbPmRoomMap) string {
|
||||
return fmt.Sprintf("pmroom:%s/%s/%s/%s",
|
||||
room.Protocol, room.MxUserID, room.AccountName, room.UserID)
|
||||
}
|
||||
|
||||
func dbGetMxPmRoom(protocol string, them connector.UserID, themMxId string, usMxId string, usAccount string) (string, error) {
|
||||
map_key := &DbPmRoomMap{
|
||||
MxUserID: usMxId,
|
||||
|
|
@ -256,15 +222,11 @@ func dbGetMxPmRoom(protocol string, them connector.UserID, themMxId string, usMx
|
|||
AccountName: usAccount,
|
||||
UserID: them,
|
||||
}
|
||||
slot_key := dbPmRoomSlotKey(map_key)
|
||||
slot_key := fmt.Sprintf("pmroom:%s/%s/%s/%s", protocol, usMxId, usAccount, them)
|
||||
|
||||
dbLockSlot(slot_key)
|
||||
defer dbUnlockSlot(slot_key)
|
||||
|
||||
if cached, ok := dbCache.Get(slot_key); ok {
|
||||
return cached.(string), nil
|
||||
}
|
||||
|
||||
var room DbPmRoomMap
|
||||
must_create := db.First(&room, map_key).RecordNotFound()
|
||||
|
||||
|
|
@ -288,19 +250,12 @@ func dbGetMxPmRoom(protocol string, them connector.UserID, themMxId string, usMx
|
|||
}
|
||||
|
||||
log.Tracef("%s -> %s", slot_key, room.MxRoomID)
|
||||
dbCache.Add(slot_key, room.MxRoomID)
|
||||
|
||||
return room.MxRoomID, nil
|
||||
}
|
||||
|
||||
func dbDeletePmRoom(room *DbPmRoomMap) {
|
||||
slot_key := dbPmRoomSlotKey(room)
|
||||
|
||||
dbLockSlot(slot_key)
|
||||
defer dbUnlockSlot(slot_key)
|
||||
|
||||
db.Delete(room)
|
||||
dbCache.Remove(slot_key)
|
||||
}
|
||||
|
||||
func dbGetMxUser(protocol string, userId connector.UserID) (string, error) {
|
||||
|
|
@ -309,10 +264,6 @@ func dbGetMxUser(protocol string, userId connector.UserID) (string, error) {
|
|||
dbLockSlot(slot_key)
|
||||
defer dbUnlockSlot(slot_key)
|
||||
|
||||
if cached, ok := dbCache.Get(slot_key); ok {
|
||||
return cached.(string), nil
|
||||
}
|
||||
|
||||
var user DbUserMap
|
||||
|
||||
must_create := db.First(&user, DbUserMap{
|
||||
|
|
@ -342,7 +293,6 @@ func dbGetMxUser(protocol string, userId connector.UserID) (string, error) {
|
|||
}
|
||||
|
||||
log.Tracef("%s -> %s", slot_key, user.MxUserID)
|
||||
dbCache.Add(slot_key, user.MxUserID)
|
||||
|
||||
return user.MxUserID, nil
|
||||
}
|
||||
|
|
|
|||
10
external/messenger.py
vendored
10
external/messenger.py
vendored
|
|
@ -251,7 +251,7 @@ class MessengerBridge:
|
|||
|
||||
self.my_user_id = self.getUserIdFromUid(self.client.uid)
|
||||
|
||||
threads = self.client.fetchThreadList()
|
||||
threads = self.client.fetchThreadList(limit=10)
|
||||
# ensure we have a correct mapping for bridged user IDs to fb uids
|
||||
# (this should be fast)
|
||||
for thread in threads:
|
||||
|
|
@ -478,7 +478,11 @@ class MessengerBridge:
|
|||
|
||||
for at in message_object.attachments:
|
||||
if isinstance(at, ImageAttachment):
|
||||
full_url = self.client.fetchImageUrl(at.uid)
|
||||
try:
|
||||
full_url = self.client.fetchImageUrl(at.uid)
|
||||
except:
|
||||
time.sleep(1)
|
||||
full_url = self.client.fetchImageUrl(at.uid)
|
||||
event["attachments"].append({
|
||||
"filename": full_url.split("?")[0].split("/")[-1],
|
||||
"url": full_url,
|
||||
|
|
@ -499,6 +503,8 @@ class MessengerBridge:
|
|||
"filename": at.filename,
|
||||
"url": url,
|
||||
})
|
||||
elif isinstance(at, ShareAttachment):
|
||||
event["text"] += "\n{}\n{}".format(at.description, at.url)
|
||||
else:
|
||||
event["text"] += "\nUnhandled attachment: {}".format(at)
|
||||
|
||||
|
|
|
|||
54
go.mod
54
go.mod
|
|
@ -3,22 +3,58 @@ module git.deuxfleurs.fr/Deuxfleurs/easybridge
|
|||
go 1.13
|
||||
|
||||
require (
|
||||
github.com/42wim/matterbridge v1.16.5
|
||||
github.com/42wim/matterbridge v1.18.3
|
||||
github.com/blang/semver v3.5.1+incompatible // indirect
|
||||
github.com/bwmarrin/discordgo v0.20.2 // indirect
|
||||
github.com/denisenkom/go-mssqldb v0.0.0-20200910202707-1e08a3fab204 // indirect
|
||||
github.com/dfordsoft/golib v0.0.0-20180902042739-76ee6ab99bec // indirect
|
||||
github.com/dyatlov/go-opengraph v0.0.0-20180429202543-816b6608b3c8 // indirect
|
||||
github.com/go-ldap/ldap v3.0.3+incompatible // indirect
|
||||
github.com/go-ldap/ldap/v3 v3.1.7
|
||||
github.com/gorilla/mux v1.7.4
|
||||
github.com/gorilla/sessions v1.2.0
|
||||
github.com/hashicorp/golang-lru v0.5.3
|
||||
github.com/jinzhu/gorm v1.9.12
|
||||
github.com/go-redis/redis v6.15.9+incompatible // indirect
|
||||
github.com/go-telegram-bot-api/telegram-bot-api v4.6.5-0.20181225215658-ec221ba9ea45+incompatible // indirect
|
||||
github.com/google/go-cmp v0.5.2 // indirect
|
||||
github.com/google/uuid v1.1.2 // indirect
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/gorilla/sessions v1.2.1
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.4
|
||||
github.com/jinzhu/gorm v1.9.16
|
||||
github.com/jinzhu/now v1.1.1 // indirect
|
||||
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 // indirect
|
||||
github.com/kr/pretty v0.2.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/labstack/echo v3.3.10+incompatible // indirect
|
||||
github.com/lib/pq v1.8.0 // indirect
|
||||
github.com/lrstanley/girc v0.0.0-20190801035559-4fc93959e1a7
|
||||
github.com/matterbridge/go-xmpp v0.0.0-20180529212104-cd19799fba91
|
||||
github.com/matterbridge/go-xmpp v0.0.0-20200418225040-c8a3a57b4050
|
||||
github.com/matterbridge/gomatrix v0.0.0-20191026211822-6fc7accd00ca // indirect
|
||||
github.com/mattermost/logr v1.0.13 // indirect
|
||||
github.com/mattermost/mattermost-server v5.11.1+incompatible
|
||||
github.com/mattermost/mattermost-server/v5 v5.27.0
|
||||
github.com/mattn/go-xmpp v0.0.0-20200128155807-a86b6abcb3ad
|
||||
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
|
||||
github.com/nelsam/hel/v2 v2.3.3 // indirect
|
||||
github.com/nicksnyder/go-i18n v1.10.1 // indirect
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
|
||||
github.com/nlopes/slack v0.6.0 // indirect
|
||||
github.com/nxadm/tail v1.4.5 // indirect
|
||||
github.com/onsi/ginkgo v1.14.1 // indirect
|
||||
github.com/onsi/gomega v1.10.2 // indirect
|
||||
github.com/pborman/uuid v1.2.1 // indirect
|
||||
github.com/pelletier/go-toml v1.8.1 // indirect
|
||||
github.com/poy/onpar v1.0.1 // indirect
|
||||
github.com/rs/xid v1.2.1
|
||||
github.com/sirupsen/logrus v1.4.2
|
||||
golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876
|
||||
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
|
||||
github.com/sirupsen/logrus v1.7.0
|
||||
github.com/stretchr/objx v0.3.0 // indirect
|
||||
go.uber.org/multierr v1.6.0 // indirect
|
||||
go.uber.org/zap v1.16.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0
|
||||
golang.org/x/net v0.0.0-20201002202402-0a1ea396d57c // indirect
|
||||
golang.org/x/tools v0.0.0-20201002184944-ecd9fd270d5d // indirect
|
||||
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
|
||||
gopkg.in/yaml.v2 v2.2.8
|
||||
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
|
||||
gopkg.in/yaml.v2 v2.3.0
|
||||
honnef.co/go/tools v0.0.1-2020.1.5 // indirect
|
||||
)
|
||||
|
|
|
|||
13
server.go
13
server.go
|
|
@ -171,7 +171,10 @@ func handleTxnEvent(e *mxlib.Event) error {
|
|||
} else if room := dbIsPublicRoom(e.RoomId); room != nil {
|
||||
// If this is a regular room
|
||||
acct := FindJoinedAccount(e.Sender, room.Protocol, room.RoomID)
|
||||
if acct != nil {
|
||||
if acct == nil {
|
||||
mx.RoomKick(e.RoomId, e.Sender, fmt.Sprintf("Not present in %s on %s, please talk with Easybridge to rejoin", room.RoomID, room.Protocol))
|
||||
return fmt.Errorf("not joined %s on %s", room.RoomID, room.Protocol)
|
||||
} else {
|
||||
ev.Author = acct.Conn.User()
|
||||
ev.Room = room.RoomID
|
||||
|
||||
|
|
@ -186,9 +189,6 @@ func handleTxnEvent(e *mxlib.Event) error {
|
|||
dbKvPutLocked(cache_key, "yes")
|
||||
}
|
||||
return err
|
||||
} else {
|
||||
mx.RoomKick(e.RoomId, e.Sender, fmt.Sprintf("Not present in %s on %s, please talk with Easybridge to rejoin", room.RoomID, room.Protocol))
|
||||
return fmt.Errorf("not joined %s on %s", room.RoomID, room.Protocol)
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("Room not bridged")
|
||||
|
|
@ -203,7 +203,10 @@ func handleTxnEvent(e *mxlib.Event) error {
|
|||
if pm_room := dbIsPmRoom(e.RoomId); pm_room != nil {
|
||||
// If user leaves a PM room, we must drop it
|
||||
dbDeletePmRoom(pm_room)
|
||||
them_mx := userMxId(pm_room.Protocol, pm_room.UserID)
|
||||
them_mx, err := dbGetMxUser(pm_room.Protocol, pm_room.UserID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mx.RoomLeaveAs(e.RoomId, them_mx)
|
||||
return nil
|
||||
} else if room := dbIsPublicRoom(e.RoomId); room != nil {
|
||||
|
|
|
|||
Loading…
Reference in a new issue