Update go packages; remove db cache

Makefile

@@ -21,3 +21,9 @@ docker: $(BIN)
 	docker push $(DOCKER):$(TAG)
 	docker tag $(DOCKER):$(TAG) $(DOCKER):latest
 	docker push $(DOCKER):latest
+
+docker_clean: $(BIN)
+	docker build --no-cache -t $(DOCKER):$(TAG) .
+	docker push $(DOCKER):$(TAG)
+	docker tag $(DOCKER):$(TAG) $(DOCKER):latest
+	docker push $(DOCKER):latest

connector/mattermost/mattermost.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	"github.com/42wim/matterbridge/matterclient"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 	log "github.com/sirupsen/logrus"
 
 	. "git.deuxfleurs.fr/Deuxfleurs/easybridge/connector"

db.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/hashicorp/golang-lru"
 	"github.com/jinzhu/gorm"
 	_ "github.com/jinzhu/gorm/dialects/mysql"
 	_ "github.com/jinzhu/gorm/dialects/postgres"
@@ -17,7 +16,6 @@ import (
 )
 
 var db *gorm.DB
-var dbCache *lru.TwoQueueCache
 
 func InitDb() error {
 	var err error
@@ -39,15 +37,10 @@ func InitDb() error {
 	db.Model(&DbRoomMap{}).AddIndex("idx_room_map_protocol_room", "protocol", "room_id")
 
 	db.AutoMigrate(&DbPmRoomMap{})
-	db.Model(&DbPmRoomMap{}).AddIndex("idx_pm_room_map_protocol_user_account_user", "protocol", "user_id", "mx_user_id", "account_name")
+	db.Model(&DbPmRoomMap{}).AddIndex("idx_pm_room_map_protocol_user_account_user", "protocol", "mx_user_id", "account_name", "user_id")
 
 	db.AutoMigrate(&DbKv{})
 
-	dbCache, err = lru.New2Q(10000)
-	if err != nil {
-		return err
-	}
-
 	return nil
 }
 
@@ -63,7 +56,7 @@ type DbAccountConfig struct {
 
 // List of joined channels to be re-joined on reconnect
 type DbJoinedRoom struct {
-	ID uint `gorm:"primary_key"`
+	ID uint `gorm:"primaryKey;autoIncrement:true"`
 
 	// User id and account name
 	MxUserID    string
@@ -76,21 +69,22 @@ type DbJoinedRoom struct {
 
 // User mapping between protocol user IDs and puppeted matrix ids
 type DbUserMap struct {
-	ID uint `gorm:"primary_key"`
+	ID uint `gorm:"primaryKey;autoIncrement:true"`
 
+	// Protocol and user ID on the bridged network
 	Protocol string
 	UserID   connector.UserID
+
+	// Puppetted Matrix ID
 	MxUserID string `gorm:"index"`
 }
 
 // Room mapping between Matrix rooms and outside rooms
 type DbRoomMap struct {
-	ID uint `gorm:"primary_key"`
+	ID uint `gorm:"primaryKey;autoIncrement:true"`
 
-	// Network protocol
+	// Network protocol and room ID on the bridged network
 	Protocol string
-
-	// Room id on the bridged network
 	RoomID   connector.RoomID
 
 	// Bridged room matrix id
@@ -99,14 +93,14 @@ type DbRoomMap struct {
 
 // Room mapping between Matrix rooms and private messages
 type DbPmRoomMap struct {
-	ID uint `gorm:"primary_key"`
+	ID uint `gorm:"primaryKey;autoIncrement:true"`
 
 	// User id and account name of the local end viewed on Matrix
 	MxUserID    string
 	Protocol    string
 	AccountName string
 
-	// User id to reach them
+	// User id to reach them on the bridged network
 	UserID connector.UserID
 
 	// Bridged room for PMs
@@ -115,14 +109,13 @@ type DbPmRoomMap struct {
 
 // Key-value store for various things
 type DbKv struct {
-	Key   string `gorm:"primary_key"`
+	Key   string `gorm:"primaryKey"`
 	Value string
 }
 
 // ---- Simple locking mechanism
 // Slot keys are strings that identify the object we are acting upon
 // They define which lock to lock for a certain operation
-// They are also used as keys in the LRU cache
 
 var dbLocks [256]sync.Mutex
 
@@ -138,17 +131,7 @@ func dbUnlockSlot(key string) {
 
 // ---- Key-value store supporting atomic test-and-set
 
-func dbKvSlotKey(key string) string {
-	return "kv:" + key
-}
-
 func dbKvGet(key string) string {
-	slot_key := dbKvSlotKey(key)
-
-	if ent, ok := dbCache.Get(slot_key); ok {
-		return ent.(string)
-	}
-
 	var entry DbKv
 	if db.Where(&DbKv{Key: key}).First(&entry).RecordNotFound() {
 		return ""
@@ -158,10 +141,8 @@ func dbKvGet(key string) string {
 }
 
 func dbKvPut(key string, value string) {
-	slot_key := dbKvSlotKey(key)
+	dbLockSlot(key)
-
+	defer dbUnlockSlot(key)
-	dbLockSlot(slot_key)
-	defer dbUnlockSlot(slot_key)
 
 	dbKvPutLocked(key, value)
 }
@@ -169,18 +150,13 @@ func dbKvPut(key string, value string) {
 // Variant of dbKvPut that does not take a lock,
 // use this if the slot is already locked
 func dbKvPutLocked(key string, value string) {
-	slot_key := dbKvSlotKey(key)
-
 	var entry DbKv
 	db.Where(&DbKv{Key: key}).Assign(&DbKv{Value: value}).FirstOrCreate(&entry)
-	dbCache.Add(slot_key, value)
 }
 
 func dbKvTestAndSet(key string, value string) bool {
-	slot_key := dbKvSlotKey(key)
+	dbLockSlot(key)
-
+	defer dbUnlockSlot(key)
-	dbLockSlot(slot_key)
-	defer dbUnlockSlot(slot_key)
 
 	// True if value was changed, false if was already set
 	if dbKvGet(key) == value {
@@ -199,10 +175,6 @@ func dbGetMxRoom(protocol string, roomId connector.RoomID) (string, error) {
 	dbLockSlot(slot_key)
 	defer dbUnlockSlot(slot_key)
 
-	if cached, ok := dbCache.Get(slot_key); ok {
-		return cached.(string), nil
-	}
-
 	// Check if room exists in our mapping,
 	// If not create it
 	var room DbRoomMap
@@ -239,16 +211,10 @@ func dbGetMxRoom(protocol string, roomId connector.RoomID) (string, error) {
 	}
 
 	log.Tracef("%s -> %s", slot_key, room.MxRoomID)
-	dbCache.Add(slot_key, room.MxRoomID)
 
 	return room.MxRoomID, nil
 }
 
-func dbPmRoomSlotKey(room *DbPmRoomMap) string {
-	return fmt.Sprintf("pmroom:%s/%s/%s/%s",
-		room.Protocol, room.MxUserID, room.AccountName, room.UserID)
-}
-
 func dbGetMxPmRoom(protocol string, them connector.UserID, themMxId string, usMxId string, usAccount string) (string, error) {
 	map_key := &DbPmRoomMap{
 		MxUserID:    usMxId,
@@ -256,15 +222,11 @@ func dbGetMxPmRoom(protocol string, them connector.UserID, themMxId string, usMx
 		AccountName: usAccount,
 		UserID:      them,
 	}
-	slot_key := dbPmRoomSlotKey(map_key)
+	slot_key := fmt.Sprintf("pmroom:%s/%s/%s/%s", protocol, usMxId, usAccount, them)
 
 	dbLockSlot(slot_key)
 	defer dbUnlockSlot(slot_key)
 
-	if cached, ok := dbCache.Get(slot_key); ok {
-		return cached.(string), nil
-	}
-
 	var room DbPmRoomMap
 	must_create := db.First(&room, map_key).RecordNotFound()
 
@@ -288,19 +250,12 @@ func dbGetMxPmRoom(protocol string, them connector.UserID, themMxId string, usMx
 	}
 
 	log.Tracef("%s -> %s", slot_key, room.MxRoomID)
-	dbCache.Add(slot_key, room.MxRoomID)
 
 	return room.MxRoomID, nil
 }
 
 func dbDeletePmRoom(room *DbPmRoomMap) {
-	slot_key := dbPmRoomSlotKey(room)
-
-	dbLockSlot(slot_key)
-	defer dbUnlockSlot(slot_key)
-
 	db.Delete(room)
-	dbCache.Remove(slot_key)
 }
 
 func dbGetMxUser(protocol string, userId connector.UserID) (string, error) {
@@ -309,10 +264,6 @@ func dbGetMxUser(protocol string, userId connector.UserID) (string, error) {
 	dbLockSlot(slot_key)
 	defer dbUnlockSlot(slot_key)
 
-	if cached, ok := dbCache.Get(slot_key); ok {
-		return cached.(string), nil
-	}
-
 	var user DbUserMap
 
 	must_create := db.First(&user, DbUserMap{
@@ -342,7 +293,6 @@ func dbGetMxUser(protocol string, userId connector.UserID) (string, error) {
 	}
 
 	log.Tracef("%s -> %s", slot_key, user.MxUserID)
-	dbCache.Add(slot_key, user.MxUserID)
 
 	return user.MxUserID, nil
 }

external/messenger.py

@@ -251,7 +251,7 @@ class MessengerBridge:
 
             self.my_user_id = self.getUserIdFromUid(self.client.uid)
 
-            threads = self.client.fetchThreadList()
+            threads = self.client.fetchThreadList(limit=10)
             # ensure we have a correct mapping for bridged user IDs to fb uids
             # (this should be fast)
             for thread in threads:
@@ -478,6 +478,10 @@ class MessengerBridge:
 
         for at in message_object.attachments:
             if isinstance(at, ImageAttachment):
+                try:
+                    full_url = self.client.fetchImageUrl(at.uid)
+                except:
+                    time.sleep(1)
                     full_url = self.client.fetchImageUrl(at.uid)
                 event["attachments"].append({
                     "filename": full_url.split("?")[0].split("/")[-1],
@@ -499,6 +503,8 @@ class MessengerBridge:
                     "filename": at.filename,
                     "url": url,
                 })
+            elif isinstance(at, ShareAttachment):
+                event["text"] += "\n{}\n{}".format(at.description, at.url)
             else:
                 event["text"] += "\nUnhandled attachment: {}".format(at)
 

go.mod

@@ -3,22 +3,58 @@ module git.deuxfleurs.fr/Deuxfleurs/easybridge
 go 1.13
 
 require (
-	github.com/42wim/matterbridge v1.16.5
+	github.com/42wim/matterbridge v1.18.3
 	github.com/blang/semver v3.5.1+incompatible // indirect
+	github.com/bwmarrin/discordgo v0.20.2 // indirect
+	github.com/denisenkom/go-mssqldb v0.0.0-20200910202707-1e08a3fab204 // indirect
+	github.com/dfordsoft/golib v0.0.0-20180902042739-76ee6ab99bec // indirect
 	github.com/dyatlov/go-opengraph v0.0.0-20180429202543-816b6608b3c8 // indirect
 	github.com/go-ldap/ldap v3.0.3+incompatible // indirect
 	github.com/go-ldap/ldap/v3 v3.1.7
-	github.com/gorilla/mux v1.7.4
+	github.com/go-redis/redis v6.15.9+incompatible // indirect
-	github.com/gorilla/sessions v1.2.0
+	github.com/go-telegram-bot-api/telegram-bot-api v4.6.5-0.20181225215658-ec221ba9ea45+incompatible // indirect
-	github.com/hashicorp/golang-lru v0.5.3
+	github.com/google/go-cmp v0.5.2 // indirect
-	github.com/jinzhu/gorm v1.9.12
+	github.com/google/uuid v1.1.2 // indirect
+	github.com/gorilla/mux v1.8.0
+	github.com/gorilla/sessions v1.2.1
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/golang-lru v0.5.4
+	github.com/jinzhu/gorm v1.9.16
+	github.com/jinzhu/now v1.1.1 // indirect
+	github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 // indirect
+	github.com/kr/pretty v0.2.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/labstack/echo v3.3.10+incompatible // indirect
+	github.com/lib/pq v1.8.0 // indirect
 	github.com/lrstanley/girc v0.0.0-20190801035559-4fc93959e1a7
-	github.com/matterbridge/go-xmpp v0.0.0-20180529212104-cd19799fba91
+	github.com/matterbridge/go-xmpp v0.0.0-20200418225040-c8a3a57b4050
+	github.com/matterbridge/gomatrix v0.0.0-20191026211822-6fc7accd00ca // indirect
+	github.com/mattermost/logr v1.0.13 // indirect
 	github.com/mattermost/mattermost-server v5.11.1+incompatible
+	github.com/mattermost/mattermost-server/v5 v5.27.0
 	github.com/mattn/go-xmpp v0.0.0-20200128155807-a86b6abcb3ad
+	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
+	github.com/nelsam/hel/v2 v2.3.3 // indirect
+	github.com/nicksnyder/go-i18n v1.10.1 // indirect
+	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
+	github.com/nlopes/slack v0.6.0 // indirect
+	github.com/nxadm/tail v1.4.5 // indirect
+	github.com/onsi/ginkgo v1.14.1 // indirect
+	github.com/onsi/gomega v1.10.2 // indirect
+	github.com/pborman/uuid v1.2.1 // indirect
+	github.com/pelletier/go-toml v1.8.1 // indirect
+	github.com/poy/onpar v1.0.1 // indirect
 	github.com/rs/xid v1.2.1
-	github.com/sirupsen/logrus v1.4.2
+	github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
-	golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876
+	github.com/sirupsen/logrus v1.7.0
+	github.com/stretchr/objx v0.3.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	go.uber.org/zap v1.16.0 // indirect
+	golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0
+	golang.org/x/net v0.0.0-20201002202402-0a1ea396d57c // indirect
+	golang.org/x/tools v0.0.0-20201002184944-ecd9fd270d5d // indirect
 	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
-	gopkg.in/yaml.v2 v2.2.8
+	gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
+	gopkg.in/yaml.v2 v2.3.0
+	honnef.co/go/tools v0.0.1-2020.1.5 // indirect
 )

server.go

@@ -171,7 +171,10 @@ func handleTxnEvent(e *mxlib.Event) error {
 		} else if room := dbIsPublicRoom(e.RoomId); room != nil {
 			// If this is a regular room
 			acct := FindJoinedAccount(e.Sender, room.Protocol, room.RoomID)
-			if acct != nil {
+			if acct == nil {
+				mx.RoomKick(e.RoomId, e.Sender, fmt.Sprintf("Not present in %s on %s, please talk with Easybridge to rejoin", room.RoomID, room.Protocol))
+				return fmt.Errorf("not joined %s on %s", room.RoomID, room.Protocol)
+			} else {
 				ev.Author = acct.Conn.User()
 				ev.Room = room.RoomID
 
@@ -186,9 +189,6 @@ func handleTxnEvent(e *mxlib.Event) error {
 					dbKvPutLocked(cache_key, "yes")
 				}
 				return err
-			} else {
-				mx.RoomKick(e.RoomId, e.Sender, fmt.Sprintf("Not present in %s on %s, please talk with Easybridge to rejoin", room.RoomID, room.Protocol))
-				return fmt.Errorf("not joined %s on %s", room.RoomID, room.Protocol)
 			}
 		} else {
 			return fmt.Errorf("Room not bridged")
@@ -203,7 +203,10 @@ func handleTxnEvent(e *mxlib.Event) error {
 			if pm_room := dbIsPmRoom(e.RoomId); pm_room != nil {
 				// If user leaves a PM room, we must drop it
 				dbDeletePmRoom(pm_room)
-				them_mx := userMxId(pm_room.Protocol, pm_room.UserID)
+				them_mx, err := dbGetMxUser(pm_room.Protocol, pm_room.UserID)
+				if err != nil {
+					return err
+				}
 				mx.RoomLeaveAs(e.RoomId, them_mx)
 				return nil
 			} else if room := dbIsPublicRoom(e.RoomId); room != nil {