Crash hotfix for the ANNOYING PALOCHKA fix

.gitignore

@@ -4,3 +4,4 @@ sessions/
 session.dat
 session.dat.new
 release/
+tdlib/

Dockerfile

@@ -29,7 +29,7 @@ WORKDIR /src
 RUN make ${MAKEOPTS}
 
 FROM scratch AS telegabber
-COPY --from=build /src/telegabber /usr/local/bin/
+COPY --from=build /src/release/telegabber /usr/local/bin/
 ENTRYPOINT ["/usr/local/bin/telegabber"]
 
 FROM scratch AS binaries

Makefile

@@ -2,11 +2,12 @@
 
 COMMIT := $(shell git rev-parse --short HEAD)
 TD_COMMIT := "5bbfc1cf5dab94f82e02f3430ded7241d4653551"
-VERSION := "v1.9.1"
+VERSION := "v1.12.8"
 MAKEOPTS := "-j4"
 
 all:
-	go build -ldflags "-X main.commit=${COMMIT}" -o telegabber
+	mkdir -p release
+	go build -ldflags "-X main.commit=${COMMIT}" -o release/telegabber
 
 test:
 	go test -v ./config ./ ./telegram ./xmpp ./xmpp/gateway ./persistence ./telegram/formatter ./badger
@@ -16,3 +17,9 @@ lint:
 
 build_indocker:
 	docker build --build-arg "TD_COMMIT=${TD_COMMIT}" --build-arg "VERSION=${VERSION}" --build-arg "MAKEOPTS=${MAKEOPTS}" --output=release --target binaries .
+
+build_indocker_staging:
+	DOCKER_BUILDKIT=1 docker build --build-arg "TD_COMMIT=${TD_COMMIT}" --build-arg "MAKEOPTS=${MAKEOPTS}" --network host --output=release --target binaries -f staging.Dockerfile .
+
+build_tdlib:
+	DOCKER_BUILDKIT=1 docker build --build-arg "TD_COMMIT=${TD_COMMIT}" --build-arg "MAKEOPTS=${MAKEOPTS}" --output=tdlib --target binaries -f tdlib.Dockerfile .

go.mod

@@ -4,6 +4,7 @@ go 1.19
 
 require (
 	github.com/dgraph-io/badger/v4 v4.1.0
+	github.com/google/uuid v1.1.1
 	github.com/pkg/errors v0.9.1
 	github.com/santhosh-tekuri/jsonschema v1.2.4
 	github.com/sirupsen/logrus v1.4.2
@@ -23,7 +24,6 @@ require (
 	github.com/golang/protobuf v1.3.2 // indirect
 	github.com/golang/snappy v0.0.3 // indirect
 	github.com/google/flatbuffers v1.12.1 // indirect
-	github.com/google/uuid v1.1.1 // indirect
 	github.com/klauspost/compress v1.12.3 // indirect
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	go.opencensus.io v0.22.5 // indirect
@@ -33,5 +33,6 @@ require (
 	nhooyr.io/websocket v1.6.5 // indirect
 )
 
-replace gosrc.io/xmpp => dev.narayana.im/narayana/go-xmpp v0.0.0-20220524203317-306b4ff58e8f
+replace gosrc.io/xmpp => dev.narayana.im/narayana/go-xmpp v0.0.0-20240512132113-6725c3862314
+
 replace github.com/zelenin/go-tdlib => dev.narayana.im/narayana/go-tdlib v0.0.0-20240124222245-b4c12addb061

go.sum

@@ -7,6 +7,10 @@ dev.narayana.im/narayana/go-tdlib v0.0.0-20240124222245-b4c12addb061 h1:CWAQT74L
 dev.narayana.im/narayana/go-tdlib v0.0.0-20240124222245-b4c12addb061/go.mod h1:Xs8fXbk5n7VaPyrSs9DP7QYoBScWYsjX+lUcWmx1DIU=
 dev.narayana.im/narayana/go-xmpp v0.0.0-20220524203317-306b4ff58e8f h1:6249ajbMjgYz53Oq0IjTvjHXbxTfu29Mj1J/6swRHs4=
 dev.narayana.im/narayana/go-xmpp v0.0.0-20220524203317-306b4ff58e8f/go.mod h1:L3NFMqYOxyLz3JGmgFyWf7r9htE91zVGiK40oW4RwdY=
+dev.narayana.im/narayana/go-xmpp v0.0.0-20240131013505-18c46e6c59fd h1:+UW+E7JjI88aH4beDn1cw6D8rs1I061hN91HU4Y4pT8=
+dev.narayana.im/narayana/go-xmpp v0.0.0-20240131013505-18c46e6c59fd/go.mod h1:L3NFMqYOxyLz3JGmgFyWf7r9htE91zVGiK40oW4RwdY=
+dev.narayana.im/narayana/go-xmpp v0.0.0-20240512132113-6725c3862314 h1:29/NjOGOUDceO73Hk4Nj4uVa1je8MULJlsDSvKxSN/k=
+dev.narayana.im/narayana/go-xmpp v0.0.0-20240512132113-6725c3862314/go.mod h1:L3NFMqYOxyLz3JGmgFyWf7r9htE91zVGiK40oW4RwdY=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/agnivade/wasmbrowsertest v0.3.1/go.mod h1:zQt6ZTdl338xxRaMW395qccVE2eQm0SjC/SDz0mPWQI=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=

persistence/sessions.go

@@ -3,6 +3,7 @@ package persistence
 import (
 	"github.com/pkg/errors"
 	"io/ioutil"
+	"sync"
 	"time"
 
 	"dev.narayana.im/narayana/telegabber/yamldb"
@@ -44,9 +45,19 @@ type Session struct {
 	HideIds         bool           `yaml:":hideids"`
 	Receipts        bool           `yaml:":receipts"`
 	NativeEdits     bool           `yaml:":nativeedits"`
+	IgnoredChats    []int64        `yaml:":ignoredchats"`
+	ignoredChatsMap map[int64]bool `yaml:"-"`
+
+	IgnoreGroupDeletions bool `yaml:":ignoregroupdeletions"`
 }
 
-var configKeys = []string{
+const (
+	PropertyTypeUnknown byte = iota
+	PropertyTypeString
+	PropertyTypeBool
+)
+
+var ConfigKeys = []string{
 	"timezone",
 	"keeponline",
 	"rawmessages",
@@ -56,17 +67,25 @@ var configKeys = []string{
 	"hideids",
 	"receipts",
 	"nativeedits",
+	"ignoregroupdeletions",
 }
 
 var sessionDB *SessionsYamlDB
+var sessionsLock sync.Mutex
 
 // SessionMarshaller implementation for YamlDB
 func SessionMarshaller() ([]byte, error) {
 	cleanedMap := SessionsMap{}
 	emptySessionsMap(&cleanedMap)
 
+	sessionsLock.Lock()
+	defer sessionsLock.Unlock()
 	for jid, session := range sessionDB.Data.Sessions {
 		if session.Login != "" {
+			session.IgnoredChats = make([]int64, 0, len(session.ignoredChatsMap))
+			for chatID := range session.ignoredChatsMap {
+				session.IgnoredChats = append(session.IgnoredChats, chatID)
+			}
 			cleanedMap.Sessions[jid] = session
 		}
 	}
@@ -108,6 +127,16 @@ func initYamlDB(path string, dataPtr *SessionsMap) (*SessionsYamlDB, error) {
 		emptySessionsMap(dataPtr)
 	}
 
+	// convert ignored users slice to map
+	for jid, session := range dataPtr.Sessions {
+		session.ignoredChatsMap = make(map[int64]bool)
+		for _, chatID := range session.IgnoredChats {
+			session.ignoredChatsMap[chatID] = true
+		}
+		session.IgnoredChats = nil
+		dataPtr.Sessions[jid] = session
+	}
+
 	return &SessionsYamlDB{
 		YamlDB: yamldb.YamlDB{
 			Path:    path,
@@ -119,6 +148,13 @@ func initYamlDB(path string, dataPtr *SessionsMap) (*SessionsYamlDB, error) {
 
 // Get retrieves a session value
 func (s *Session) Get(key string) (string, error) {
+	sessionsLock.Lock()
+	defer sessionsLock.Unlock()
+
+	return s.get(key)
+}
+
+func (s *Session) get(key string) (string, error) {
 	switch key {
 	case "timezone":
 		return s.Timezone, nil
@@ -138,6 +174,8 @@ func (s *Session) Get(key string) (string, error) {
 		return fromBool(s.Receipts), nil
 	case "nativeedits":
 		return fromBool(s.NativeEdits), nil
+	case "ignoregroupdeletions":
+		return fromBool(s.IgnoreGroupDeletions), nil
 	}
 
 	return "", errors.New("Unknown session property")
@@ -145,9 +183,12 @@ func (s *Session) Get(key string) (string, error) {
 
 // ToMap converts the session to a map
 func (s *Session) ToMap() map[string]string {
+	sessionsLock.Lock()
+	defer sessionsLock.Unlock()
+
 	m := make(map[string]string)
-	for _, configKey := range configKeys {
-		value, _ := s.Get(configKey)
+	for _, configKey := range ConfigKeys {
+		value, _ := s.get(configKey)
 		m[configKey] = value
 	}
 
@@ -156,6 +197,9 @@ func (s *Session) ToMap() map[string]string {
 
 // Set sets a session value
 func (s *Session) Set(key string, value string) (string, error) {
+	sessionsLock.Lock()
+	defer sessionsLock.Unlock()
+
 	switch key {
 	case "timezone":
 		s.Timezone = value
@@ -216,11 +260,30 @@ func (s *Session) Set(key string, value string) (string, error) {
 		}
 		s.NativeEdits = b
 		return value, nil
+	case "ignoregroupdeletions":
+		b, err := toBool(value)
+		if err != nil {
+			return "", err
+		}
+		s.IgnoreGroupDeletions = b
+		return value, nil
 	}
 
 	return "", errors.New("Unknown session property")
 }
 
+// PropertyType determines the property type
+func PropertyType(key string) byte {
+	switch key {
+	case "timezone":
+		return PropertyTypeString
+	case "keeponline", "rawmessages", "asciiarrows", "oobmode", "carbons", "hideids",
+		"receipts", "nativeedits", "ignoregroupdeletions":
+		return PropertyTypeBool
+	}
+	return PropertyTypeUnknown
+}
+
 // TimezoneToLocation tries to convert config timezone to location
 func (s *Session) TimezoneToLocation() *time.Location {
 	time, err := time.Parse("-07:00", s.Timezone)
@@ -232,6 +295,51 @@ func (s *Session) TimezoneToLocation() *time.Location {
 	return zeroLocation
 }
 
+// IgnoreChat adds a chat id to ignore list, returns false if already ignored
+func (s *Session) IgnoreChat(chatID int64) bool {
+	sessionsLock.Lock()
+	defer sessionsLock.Unlock()
+
+	if s.ignoredChatsMap == nil {
+		s.ignoredChatsMap = make(map[int64]bool)
+	} else if _, ok := s.ignoredChatsMap[chatID]; ok {
+		return false
+	}
+
+	s.ignoredChatsMap[chatID] = true
+	return true
+}
+
+// UnignoreChat removes a chat id from ignore list, returns false if not already ignored
+func (s *Session) UnignoreChat(chatID int64) bool {
+	sessionsLock.Lock()
+	defer sessionsLock.Unlock()
+
+	if s.ignoredChatsMap == nil {
+		return false
+	}
+
+	if _, ok := s.ignoredChatsMap[chatID]; !ok {
+		return false
+	}
+
+	delete(s.ignoredChatsMap, chatID)
+	return true
+}
+
+// IsChatIgnored checks the chat id against the ignore list
+func (s *Session) IsChatIgnored(chatID int64) bool {
+	sessionsLock.Lock()
+	defer sessionsLock.Unlock()
+
+	if s.ignoredChatsMap == nil {
+		return false
+	}
+
+	_, ok := s.ignoredChatsMap[chatID]
+	return ok
+}
+
 func fromBool(b bool) string {
 	if b {
 		return "true"

persistence/sessions_test.go

@@ -88,3 +88,31 @@ func TestSessionSetAbsent(t *testing.T) {
 		t.Error("There shouldn't come a donkey!")
 	}
 }
+
+func TestSessionIgnore(t *testing.T) {
+	session := Session{}
+	if session.IsChatIgnored(3) {
+		t.Error("Shouldn't be ignored yet")
+	}
+	if !session.IgnoreChat(3) {
+		t.Error("Shouldn't have been ignored")
+	}
+	if session.IgnoreChat(3) {
+		t.Error("Shouldn't ignore second time")
+	}
+	if !session.IsChatIgnored(3) {
+		t.Error("Should be ignored already")
+	}
+	if session.IsChatIgnored(-145) {
+		t.Error("Wrong chat is ignored")
+	}
+	if !session.UnignoreChat(3) {
+		t.Error("Should successfully unignore")
+	}
+	if session.UnignoreChat(3) {
+		t.Error("Should unignore second time")
+	}
+	if session.IsChatIgnored(3) {
+		t.Error("Shouldn't be ignored already")
+	}
+}

staging.Dockerfile

@@ -0,0 +1,46 @@
+FROM golang:1.19-bullseye AS base
+
+RUN apt-get update
+RUN apt-get install -y libssl-dev cmake build-essential gperf libz-dev make git php
+
+FROM base AS tdlib
+
+ARG TD_COMMIT
+ARG MAKEOPTS
+RUN git clone https://github.com/tdlib/td /src/
+RUN git -C /src/ checkout "${TD_COMMIT}"
+RUN mkdir build
+WORKDIR /build/
+RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/compiled/ /src/
+RUN cmake --build . --target prepare_cross_compiling ${MAKEOPTS}
+WORKDIR /src/
+RUN php SplitSource.php
+WORKDIR /build/
+RUN cmake --build . ${MAKEOPTS}
+RUN make install
+
+FROM base AS cache
+ARG VERSION
+COPY --from=tdlib /compiled/ /usr/local/
+WORKDIR /src
+RUN go env -w GOCACHE=/go-cache
+RUN go env -w GOMODCACHE=/gomod-cache
+RUN --mount=type=cache,target=/gomod-cache \
+    --mount=type=bind,source=./,target=/src,rw \
+    /bin/bash -c 'go mod tidy; go get -t'
+
+FROM cache AS build
+ARG MAKEOPTS
+WORKDIR /src
+RUN --mount=type=bind,source=./,target=/src,rw \
+    --mount=type=cache,target=/go-cache \
+    --mount=type=cache,target=/gomod-cache \
+    --mount=type=cache,destination=/src/release \
+    make ${MAKEOPTS}
+
+FROM build AS release
+RUN --mount=type=cache,destination=/src/release \
+    cp /src/release/telegabber /
+
+FROM scratch AS binaries
+COPY --from=release /telegabber /

tdlib.Dockerfile

@@ -0,0 +1,23 @@
+FROM golang:1.19-bullseye AS base
+
+RUN apt-get update
+RUN apt-get install -y libssl-dev cmake build-essential gperf libz-dev make git php
+
+FROM base AS tdlib
+
+ARG TD_COMMIT
+ARG MAKEOPTS
+RUN git clone https://github.com/tdlib/td /src/
+RUN git -C /src/ checkout "${TD_COMMIT}"
+RUN mkdir build
+WORKDIR /build/
+RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/compiled/ /src/
+RUN cmake --build . --target prepare_cross_compiling ${MAKEOPTS}
+WORKDIR /src/
+RUN php SplitSource.php
+WORKDIR /build/
+RUN cmake --build . ${MAKEOPTS}
+RUN make install
+
+FROM scratch AS binaries
+COPY --from=tdlib /compiled/ /

telegabber.go

@@ -16,7 +16,7 @@ import (
 	goxmpp "gosrc.io/xmpp"
 )
 
-var version string = "1.9.1"
+var version string = "1.12.8"
 var commit string
 
 var sm *goxmpp.StreamManager
@@ -68,7 +68,7 @@ func main() {
 
 	log.Infof("Starting telegabber version %v", version)
 
-	sm, component, err = xmpp.NewComponent(config.XMPP, config.Telegram, *idsPath)
+	sm, component, err = xmpp.NewComponent(config.XMPP, config.Telegram, *idsPath, version)
 	if err != nil {
 		log.Fatal(err)
 	}

telegram/cache/cache.go

@@ -16,7 +16,8 @@ type Status struct {
 // Cache allows operating the chats and users cache in
 // a thread-safe manner
 type Cache struct {
-	chats        map[int64]*client.Chat
+	ownChats     map[int64]*client.Chat
+	auxChats     map[int64]*client.Chat
 	users        map[int64]*client.User
 	statuses     map[int64]*Status
 	chatsLock    sync.Mutex
@@ -27,7 +28,8 @@ type Cache struct {
 // NewCache initializes a cache
 func NewCache() *Cache {
 	return &Cache{
-		chats:    map[int64]*client.Chat{},
+		ownChats: map[int64]*client.Chat{},
+		auxChats: map[int64]*client.Chat{},
 		users:    map[int64]*client.User{},
 		statuses: map[int64]*Status{},
 	}
@@ -40,7 +42,23 @@ func (cache *Cache) ChatsKeys() []int64 {
 	defer cache.chatsLock.Unlock()
 
 	var keys []int64
-	for id := range cache.chats {
+	for id := range cache.ownChats {
+		keys = append(keys, id)
+	}
+	for id := range cache.auxChats {
+		keys = append(keys, id)
+	}
+	return keys
+}
+
+// OwnChatsKeys grabs only own chat ids synchronously to avoid lockups
+// while they are used
+func (cache *Cache) OwnChatsKeys() []int64 {
+	cache.chatsLock.Lock()
+	defer cache.chatsLock.Unlock()
+
+	var keys []int64
+	for id := range cache.ownChats {
 		keys = append(keys, id)
 	}
 	return keys
@@ -84,7 +102,10 @@ func (cache *Cache) GetChat(id int64) (*client.Chat, bool) {
 	cache.chatsLock.Lock()
 	defer cache.chatsLock.Unlock()
 
-	chat, ok := cache.chats[id]
+	chat, ok := cache.ownChats[id]
+	if !ok {
+		chat, ok = cache.auxChats[id]
+	}
 	return chat, ok
 }
 
@@ -107,11 +128,21 @@ func (cache *Cache) GetStatus(id int64) (*Status, bool) {
 }
 
 // SetChat stores a chat in the cache
-func (cache *Cache) SetChat(id int64, chat *client.Chat) {
+func (cache *Cache) SetChat(id int64, chat *client.Chat, own bool) {
 	cache.chatsLock.Lock()
 	defer cache.chatsLock.Unlock()
 
-	cache.chats[id] = chat
+	if own {
+		cache.ownChats[id] = chat
+		// move from aux to own, but not vice versa
+		// (own: true means that presences for the chat are needed
+		// for sure, false means just "not necessarily")
+		if _, ok := cache.auxChats[id]; ok {
+			delete(cache.auxChats, id)
+		}
+	} else {
+		cache.auxChats[id] = chat
+	}
 }
 
 // SetUser stores a user in the cache

telegram/client.go

@@ -2,7 +2,6 @@ package telegram
 
 import (
 	"github.com/pkg/errors"
-	"hash/maphash"
 	"path/filepath"
 	"strconv"
 	"sync"
@@ -22,6 +21,47 @@ type DelayedStatus struct {
 	TimestampExpired int64
 }
 
+// HashedAvatar stores a SHA-1 hash and a Telegram file ID
+type HashedAvatar struct {
+	Hash string
+	File int32
+}
+
+// NewId stores message ids and timestamps of their additions so old ones can be truncated to save memory
+type newId struct {
+	Id      int64
+	Ts      int64
+	lock    sync.Mutex
+	ownLock sync.Mutex
+	locked  bool
+	fired   bool
+}
+
+func newNewId() *newId {
+	return &newId{Ts: time.Now().Unix()}
+}
+
+func (i *newId) Lock() {
+	i.ownLock.Lock()
+	if i.fired {
+		i.ownLock.Unlock()
+		return
+	}
+	i.locked = true
+	i.ownLock.Unlock()
+	i.lock.Lock()
+}
+
+func (i *newId) Unlock() {
+	i.ownLock.Lock()
+	if i.locked {
+		i.lock.Unlock()
+		i.locked = false
+		i.fired = true
+	}
+	i.ownLock.Unlock()
+}
+
 // Client stores the metadata for lazily invoked TDlib instance
 type Client struct {
 	client     *client.Client
@@ -38,6 +78,10 @@ type Client struct {
 	cache     *cache.Cache
 	online    bool
 
+	loginWizard *loginWizardMetadata
+
+	lastAuthorizationStateType string
+
 	outbox     map[string]string
 	editOutbox map[string]string
 
@@ -46,7 +90,17 @@ type Client struct {
 
 	lastMsgHashes map[int64]uint64
 	lastMsgIds    map[int64]string
-	msgHashSeed   maphash.Seed
+
+	LastBotCmdString string
+
+	XmppClientFeatures     map[string]*[]string
+	XmppClientFeaturesLock sync.Mutex
+
+	AvatarHashes     map[int64]*HashedAvatar
+	AvatarHashesLock sync.Mutex
+
+	MessageIdChanges     map[int64]map[int64]*newId
+	MessageIdChangesLock sync.Mutex
 
 	locks           clientLocks
 	SendMessageLock sync.Mutex
@@ -63,6 +117,15 @@ type clientLocks struct {
 
 	authorizerReadLock  sync.Mutex
 	authorizerWriteLock sync.Mutex
+
+	loginWizardReadLock  sync.Mutex
+	loginWizardWriteLock sync.Mutex
+}
+
+type loginWizardMetadata struct {
+	nextStage   chan string
+	chanBusy    bool
+	commandSent bool
 }
 
 // NewClient instantiates a Telegram App
@@ -122,7 +185,9 @@ func NewClient(conf config.TelegramConfig, jid string, component *xmpp.Component
 		DelayedStatuses:    make(map[int64]*DelayedStatus),
 		lastMsgHashes:      make(map[int64]uint64),
 		lastMsgIds:         make(map[int64]string),
-		msgHashSeed:     maphash.MakeSeed(),
+		XmppClientFeatures: make(map[string]*[]string),
+		AvatarHashes:       make(map[int64]*HashedAvatar),
+		MessageIdChanges:   make(map[int64]map[int64]*newId),
 		locks: clientLocks{
 			chatMessageLocks: make(map[int64]*sync.Mutex),
 		},

telegram/connect.go

@@ -129,6 +129,7 @@ func (c *Client) Connect(resource string) error {
 	tdlibClient, err := client.NewClient(c.authorizer, c.options...)
 	if err != nil {
 		c.locks.authorizationReady.Unlock()
+		c.wizardStageOrPrompt("cancel", "")
 		return errors.Wrap(err, "Couldn't initialize a Telegram client instance")
 	}
 
@@ -137,6 +138,8 @@ func (c *Client) Connect(resource string) error {
 	// stage 3: if a client is succesfully created, AuthorizationStateReady is already reached
 	log.Warn("Authorization successful!")
 
+	c.wizardStageOrPrompt("success", "")
+
 	c.me, err = c.client.GetMe()
 	if err != nil {
 		log.Error("Could not retrieve me info")
@@ -150,11 +153,13 @@ func (c *Client) Connect(resource string) error {
 	c.addResource(resource)
 
 	go func() {
-		_, err = c.client.GetChats(&client.GetChatsRequest{
+		chats, err := c.client.GetChats(&client.GetChatsRequest{
 			Limit: chatsLimit,
 		})
 		if err != nil {
 			log.Errorf("Could not retrieve chats: %v", err)
+		} else {
+			log.Infof("Obtained ≈%v chats for initialization", chats.TotalCount)
 		}
 
 		gateway.SubscribeToTransport(c.xmpp, c.jid)
@@ -226,7 +231,7 @@ func (c *Client) Disconnect(resource string, quit bool) bool {
 	log.Warn("Disconnecting from Telegram network...")
 
 	// we're offline (unsubscribe if logout)
-	for _, id := range c.cache.ChatsKeys() {
+	for _, id := range c.cache.OwnChatsKeys() {
 		args := gateway.SimplePresence(id, "unavailable")
 		c.sendPresence(args...)
 	}
@@ -255,6 +260,8 @@ func (c *Client) interactor() {
 		log.Infof("Telegram authorization state: %#v", stateType)
 		log.Debugf("%#v", state)
 
+		c.lastAuthorizationStateType = stateType
+
 		switch stateType {
 		// stage 0: set login
 		case client.TypeAuthorizationStateWaitPhoneNumber:
@@ -262,12 +269,12 @@ func (c *Client) interactor() {
 			if c.Session.Login != "" {
 				c.authorizer.PhoneNumber <- c.Session.Login
 			} else {
-				gateway.SendServiceMessage(c.jid, "Please, enter your Telegram login via /login 12345", c.xmpp)
+				c.wizardStageOrPrompt("login", "Please, enter your Telegram login via /login 12345, or use the Login Wizard via Ad-Hoc commands")
 			}
 		// stage 1: wait for auth code
 		case client.TypeAuthorizationStateWaitCode:
 			log.Warn("Waiting for authorization code...")
-			gateway.SendServiceMessage(c.jid, "Please, enter authorization code via /code 12345", c.xmpp)
+			c.wizardStageOrPrompt("code", "Please, enter authorization code via /code 12345")
 		// stage 1b: wait for registration
 		case client.TypeAuthorizationStateWaitRegistration:
 			log.Warn("Waiting for full name...")
@@ -275,7 +282,7 @@ func (c *Client) interactor() {
 		// stage 2: wait for 2fa
 		case client.TypeAuthorizationStateWaitPassword:
 			log.Warn("Waiting for 2FA password...")
-			gateway.SendServiceMessage(c.jid, "Please, enter 2FA passphrase via /password 12345", c.xmpp)
+			c.wizardStageOrPrompt("password", "Please, enter 2FA passphrase via /password 12345")
 		}
 		c.locks.authorizerReadLock.Unlock()
 	}
@@ -294,6 +301,7 @@ func (c *Client) forceClose() {
 func (c *Client) close() {
 	c.locks.authorizerWriteLock.Lock()
 	if c.authorizer != nil && !c.authorizer.isClosed {
+		log.Debug("Closing authorizer")
 		c.authorizer.Close()
 	}
 	c.locks.authorizerWriteLock.Unlock()
@@ -308,6 +316,7 @@ func (c *Client) close() {
 }
 
 func (c *Client) cancelAuth() {
+	c.StopLoginWizard()
 	c.close()
 	c.Session.Login = ""
 }

telegram/handlers.go

@@ -56,8 +56,10 @@ func (c *Client) cleanTempFile(path string) {
 }
 
 func (c *Client) sendMarker(chatId, messageId int64, typ gateway.MarkerType) {
-	if xmppId, err := gateway.IdsDB.GetByTgIds(c.Session.Login, c.jid, chatId, messageId); err == nil {
-		resource := c.getFromOutbox(xmppId)
+	xmppId, err := gateway.IdsDB.GetByTgIds(c.Session.Login, c.jid, chatId, messageId)
+	if err != nil {
+		xmppId = strconv.FormatInt(messageId, 10)
+	}
 
 	var stringType string
 	if typ == gateway.MarkerTypeReceived {
@@ -67,20 +69,16 @@ func (c *Client) sendMarker(chatId, messageId int64, typ gateway.MarkerType) {
 	}
 	log.WithFields(log.Fields{
 		"xmppId": xmppId,
-			"resource": resource,
 	}).Debugf("marker: %s", stringType)
 
-		if resource != "" {
 	gateway.SendMessageMarker(
-				c.jid+"/"+resource,
+		c.jid,
 		strconv.FormatInt(chatId, 10),
 		c.xmpp,
 		typ,
 		xmppId,
 	)
 }
-	}
-}
 
 func (c *Client) updateHandler() {
 	listener := c.client.GetListener()
@@ -188,13 +186,13 @@ func (c *Client) updateHandler() {
 func (c *Client) updateUser(update *client.UpdateUser) {
 	c.cache.SetUser(update.User.Id, update.User)
 	show, status, presenceType := c.userStatusToText(update.User.Status, update.User.Id)
-	go c.ProcessStatusUpdate(update.User.Id, status, show, gateway.SPType(presenceType))
+	go c.ProcessStatusUpdate(update.User.Id, status, show, false, gateway.SPType(presenceType))
 }
 
 // user status changed
 func (c *Client) updateUserStatus(update *client.UpdateUserStatus) {
 	show, status, presenceType := c.userStatusToText(update.Status, update.UserId)
-	go c.ProcessStatusUpdate(update.UserId, status, show, gateway.SPImmed(false), gateway.SPType(presenceType))
+	go c.ProcessStatusUpdate(update.UserId, status, show, false, gateway.SPImmed(false), gateway.SPType(presenceType))
 }
 
 // new chat discovered
@@ -208,14 +206,14 @@ func (c *Client) updateNewChat(update *client.UpdateNewChat) {
 			}
 		}
 
-		c.cache.SetChat(update.Chat.Id, update.Chat)
+		c.cache.SetChat(update.Chat.Id, update.Chat, true)
 
 		if update.Chat.Positions != nil && len(update.Chat.Positions) > 0 {
 			c.subscribeToID(update.Chat.Id, update.Chat)
 		}
 
 		if update.Chat.Id < 0 {
-			c.ProcessStatusUpdate(update.Chat.Id, update.Chat.Title, "chat")
+			c.ProcessStatusUpdate(update.Chat.Id, update.Chat.Title, "chat", true)
 		}
 	}()
 }
@@ -237,6 +235,9 @@ func (c *Client) updateChatLastMessage(update *client.UpdateChatLastMessage) {
 // message received
 func (c *Client) updateNewMessage(update *client.UpdateNewMessage) {
 	chatId := update.Message.ChatId
+	if c.Session.IsChatIgnored(chatId) {
+		return
+	}
 
 	// guarantee sequential message delivering per chat
 	lock := c.getChatMessageLock(chatId)
@@ -244,12 +245,23 @@ func (c *Client) updateNewMessage(update *client.UpdateNewMessage) {
 		lock.Lock()
 		defer lock.Unlock()
 
-		c.updateLastMessageHash(update.Message.ChatId, update.Message.Id, update.Message.Content)
+		var forceCmd bool
+		if c.LastBotCmdString != "" && update.Message.IsOutgoing {
+			if update.Message.Content.MessageContentType() == client.TypeMessageText {
+				textMessage, _ := update.Message.Content.(*client.MessageText)
+
+				if textMessage.Text != nil && textMessage.Text.Text == c.LastBotCmdString {
+					forceCmd = true
+					c.LastBotCmdString = ""
+				}
+			}
+		}
 
 		// ignore self outgoing messages
 		if update.Message.IsOutgoing &&
 			update.Message.SendingState != nil &&
-			update.Message.SendingState.MessageSendingStateType() == client.TypeMessageSendingStatePending {
+			update.Message.SendingState.MessageSendingStateType() == client.TypeMessageSendingStatePending &&
+			!forceCmd {
 			return
 		}
 
@@ -263,9 +275,11 @@ func (c *Client) updateNewMessage(update *client.UpdateNewMessage) {
 
 // message content updated
 func (c *Client) updateMessageContent(update *client.UpdateMessageContent) {
-	markupFunction := c.getFormatter()
+	if c.Session.IsChatIgnored(update.ChatId) {
+		return
+	}
 
-	defer c.updateLastMessageHash(update.ChatId, update.MessageId, update.NewContent)
+	markupFunction := c.getFormatter()
 
 	log.Debugf("newContent: %#v", update.NewContent)
 
@@ -290,7 +304,7 @@ func (c *Client) updateMessageContent(update *client.UpdateMessageContent) {
 		return
 	}
 
-	if update.NewContent.MessageContentType() == client.TypeMessageText && c.hasLastMessageHashChanged(update.ChatId, update.MessageId, update.NewContent) {
+	if update.NewContent.MessageContentType() == client.TypeMessageText {
 		textContent := update.NewContent.(*client.MessageText)
 		log.Debugf("textContent: %#v", textContent.Text)
 
@@ -298,6 +312,50 @@ func (c *Client) updateMessageContent(update *client.UpdateMessageContent) {
 		sId := strconv.FormatInt(update.MessageId, 10)
 		var isCarbon bool
 
+		go func() {
+			message, messageErr := c.client.GetMessage(&client.GetMessageRequest{
+				ChatId:    update.ChatId,
+				MessageId: update.MessageId,
+			})
+			if messageErr != nil {
+				// odnako za vremya puti
+				// sobaka mogla podrasti
+				c.MessageIdChangesLock.Lock()
+				idsMap, idsMapOk := c.MessageIdChanges[update.ChatId]
+				hadNoId := false
+				if idsMapOk {
+					newId, newIdOk := idsMap[update.MessageId]
+					if newIdOk {
+						if newId.Id == 0 {
+							hadNoId = true
+							c.MessageIdChangesLock.Unlock()
+							newId.Lock()
+						}
+						log.Infof("falling back to updated message id: %v/%v->%v", update.ChatId, update.MessageId, newId.Id)
+						message, messageErr = c.client.GetMessage(&client.GetMessageRequest{
+							ChatId:    update.ChatId,
+							MessageId: newId.Id,
+						})
+					}
+				}
+				if !hadNoId {
+					c.MessageIdChangesLock.Unlock()
+				}
+			}
+			var prefix string
+			if messageErr == nil {
+				if message.EditDate == 0 {
+					return
+				}
+				log.Debugf("editDate: %v", message.EditDate)
+
+				isCarbon = c.isCarbonsEnabled() && message.IsOutgoing
+				// reply correction support in clients is suboptimal yet, so cut them out for now
+				prefix, _ = c.messageToPrefix(message, "", "", true)
+			} else {
+				log.Errorf("No message %v/%v found, cannot reliably determine if it is a carbon and if it is edited: %v", update.ChatId, update.MessageId, messageErr.Error())
+			}
+
 			// use XEP-0308 edits only if the last message is edited for sure, fallback otherwise
 			if c.Session.NativeEdits {
 				lastXmppId, ok := c.getLastChatMessageId(update.ChatId)
@@ -306,25 +364,12 @@ func (c *Client) updateMessageContent(update *client.UpdateMessageContent) {
 				}
 				if ok && lastXmppId == xmppId {
 					replaceId = xmppId
-				message, err := c.client.GetMessage(&client.GetMessageRequest{
-					ChatId:    update.ChatId,
-					MessageId: update.MessageId,
-				})
-				if err == nil {
-					isCarbon = c.isCarbonsEnabled() && message.IsOutgoing
-				} else {
-					log.Errorf("No message %v/%v found, cannot reliably determine if it's a carbon", update.ChatId, update.MessageId)
-				}
 				} else {
 					log.Infof("Mismatching message ids: %v %v, falling back to separate edit message", lastXmppId, xmppId)
 				}
 			}
 
-		text := formatter.Format(
-			textContent.Text.Text,
-			textContent.Text.Entities,
-			markupFunction,
-		)
+			var text strings.Builder
 
 			if replaceId == "" {
 				var editChar string
@@ -333,18 +378,43 @@ func (c *Client) updateMessageContent(update *client.UpdateMessageContent) {
 				} else {
 					editChar = "✎"
 				}
-			text = editChar + fmt.Sprintf("%v | %s", update.MessageId, text)
+				text.WriteString(fmt.Sprintf("%s %v | ", editChar, update.MessageId))
+			} else if prefix != "" {
+				text.WriteString(prefix)
+				text.WriteString(c.getPrefixSeparator(update.ChatId))
 			}
 
+			text.WriteString(formatter.Format(
+				textContent.Text.Text,
+				textContent.Text.Entities,
+				markupFunction,
+			))
+
+			sChatId := strconv.FormatInt(update.ChatId, 10)
 			for _, jid := range jids {
-			gateway.SendMessage(jid, strconv.FormatInt(update.ChatId, 10), text, "e"+sId, c.xmpp, nil, replaceId, isCarbon, false)
+				gateway.SendMessage(jid, sChatId, text.String(), "e"+sId, c.xmpp, nil, replaceId, isCarbon, false)
 			}
+		}()
 	}
 }
 
 // message(s) deleted
 func (c *Client) updateDeleteMessages(update *client.UpdateDeleteMessages) {
 	if update.IsPermanent {
+		for _, deleteId := range update.MessageIds {
+			c.tryUnlockMessageId(update.ChatId, deleteId)
+		}
+
+		if c.Session.IsChatIgnored(update.ChatId) {
+			return
+		}
+		if c.Session.IgnoreGroupDeletions {
+			chatType, chatTypeErr := c.GetChatType(update.ChatId, false)
+			if chatTypeErr == nil && (chatType == ChatTypeBasicGroup || chatType == ChatTypeSupergroup) {
+				return
+			}
+		}
+
 		var deleteChar string
 		if c.Session.AsciiArrows {
 			deleteChar = "X "
@@ -373,6 +443,20 @@ func (c *Client) updateMessageSendSucceeded(update *client.UpdateMessageSendSucc
 		log.Errorf("failed to replace %v with %v: %v", update.OldMessageId, update.Message.Id, err.Error())
 	}
 
+	c.MessageIdChangesLock.Lock()
+	idsMap, ok := c.MessageIdChanges[update.Message.ChatId]
+	if !ok {
+		idsMap = make(map[int64]*newId)
+		c.MessageIdChanges[update.Message.ChatId] = idsMap
+	}
+	id, ok := idsMap[update.OldMessageId]
+	if !ok {
+		id = newNewId()
+		idsMap[update.OldMessageId] = id
+	}
+	id.Id = update.Message.Id
+	c.MessageIdChangesLock.Unlock()
+
 	c.sendMarker(update.Message.ChatId, update.Message.Id, gateway.MarkerTypeReceived)
 
 	// clean uploaded files
@@ -382,6 +466,8 @@ func (c *Client) updateMessageSendSucceeded(update *client.UpdateMessageSendSucc
 	}
 }
 func (c *Client) updateMessageSendFailed(update *client.UpdateMessageSendFailed) {
+	c.tryUnlockMessageId(update.Message.ChatId, update.OldMessageId)
+
 	// clean uploaded files
 	file, _ := c.contentToFile(update.Message.Content)
 	if file != nil && file.Local != nil {
@@ -394,9 +480,9 @@ func (c *Client) updateChatTitle(update *client.UpdateChatTitle) {
 	gateway.SetNickname(c.jid, strconv.FormatInt(update.ChatId, 10), update.Title, c.xmpp)
 
 	// set also the status (for group chats only)
-	chat, user, _ := c.GetContactByID(update.ChatId, nil)
+	chat, user, _ := c.GetContactByID(update.ChatId, nil, false)
 	if user == nil {
-		c.ProcessStatusUpdate(update.ChatId, update.Title, "chat", gateway.SPImmed(true))
+		c.ProcessStatusUpdate(update.ChatId, update.Title, "chat", false, gateway.SPImmed(true))
 	}
 
 	// update chat title in the cache
@@ -408,3 +494,15 @@ func (c *Client) updateChatTitle(update *client.UpdateChatTitle) {
 func (c *Client) updateChatReadOutbox(update *client.UpdateChatReadOutbox) {
 	c.sendMarker(update.ChatId, update.LastReadOutboxMessageId, gateway.MarkerTypeDisplayed)
 }
+
+func (c *Client) tryUnlockMessageId(chatId, messageId int64) {
+	c.MessageIdChangesLock.Lock()
+	idsMap, ok := c.MessageIdChanges[chatId]
+	if ok {
+		id, ok := idsMap[messageId]
+		if ok {
+			id.Unlock()
+		}
+	}
+	c.MessageIdChangesLock.Unlock()
+}

telegram/loginwizard.go

@@ -0,0 +1,82 @@
+package telegram
+
+import (
+	"dev.narayana.im/narayana/telegabber/xmpp/gateway"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/zelenin/go-tdlib/client"
+)
+
+// StartLoginWizard initiates a loginWizard object
+func (c *Client) StartLoginWizard(inCommand bool) {
+	if c.loginWizard == nil {
+		c.loginWizard = &loginWizardMetadata{
+			nextStage:   make(chan string, 1),
+			commandSent: inCommand,
+		}
+	} else {
+		c.loginWizard.commandSent = inCommand
+	}
+}
+
+// StopLoginWizard safely destroys the loginWizard object
+func (c *Client) StopLoginWizard() {
+	c.locks.loginWizardReadLock.Lock()
+	c.locks.loginWizardWriteLock.Lock()
+	if c.loginWizard != nil {
+		close(c.loginWizard.nextStage)
+		c.loginWizard = nil
+	}
+	c.locks.loginWizardReadLock.Unlock()
+	c.locks.loginWizardWriteLock.Unlock()
+}
+
+// GetLoginWizardNextStage waits for the next stage from the channel
+func (c *Client) GetLoginWizardNextStage() string {
+	c.locks.loginWizardReadLock.Lock()
+	defer c.locks.loginWizardReadLock.Unlock()
+
+	if c.loginWizard != nil {
+		if c.loginWizard.commandSent {
+			log.Debugf("waiting for nextStage...")
+			nextStage := <-c.loginWizard.nextStage
+			c.loginWizard.commandSent = false
+			c.loginWizard.chanBusy = false
+			log.Debugf("yielded stage %v", nextStage)
+			return nextStage
+		} else {
+			if c.lastAuthorizationStateType == client.TypeAuthorizationStateWaitPhoneNumber ||
+				c.lastAuthorizationStateType == client.TypeAuthorizationStateClosing ||
+				c.Session.Login == "" {
+				return "login"
+			}
+			switch c.lastAuthorizationStateType {
+			case client.TypeAuthorizationStateWaitCode:
+				return "code"
+			case client.TypeAuthorizationStateWaitPassword:
+				return "password"
+			}
+		}
+	}
+
+	return ""
+}
+
+func (c *Client) wizardStageOrPrompt(stage, message string) {
+	c.locks.loginWizardWriteLock.Lock()
+	if c.loginWizard == nil {
+		c.locks.loginWizardWriteLock.Unlock()
+		if message != "" {
+			gateway.SendServiceMessage(c.jid, message, c.xmpp)
+		}
+	} else {
+		if !c.loginWizard.chanBusy {
+			log.Debugf("writing wizard stage %v", stage)
+			c.loginWizard.nextStage <- stage
+		} else {
+			log.Warn("Skipping stage %v, wizard cannot keep up", stage)
+		}
+		c.loginWizard.chanBusy = true
+		c.locks.loginWizardWriteLock.Unlock()
+	}
+}

telegram/utils.go

@@ -1,11 +1,11 @@
 package telegram
 
 import (
+	"bytes"
 	"crypto/sha1"
-	"encoding/binary"
+	"encoding/base64"
 	"fmt"
 	"github.com/pkg/errors"
-	"hash/maphash"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -45,7 +45,23 @@ type messageStub struct {
 	Text      string
 }
 
+type BotCommand struct {
+	Command     string
+	Description string
+}
+
+type BotLink struct {
+	Description string
+	Link        string
+}
+
+const (
+	typeFileDataSha1 byte = iota
+	typeFileDataBase64
+)
+
 var errOffline = errors.New("TDlib instance is offline")
+var errOverLimit = errors.New("Over limit")
 
 var spaceRegex = regexp.MustCompile(`\s+`)
 var replyRegex = regexp.MustCompile("\\A>>? ?([0-9]+)\\n")
@@ -53,8 +69,32 @@ var replyRegex = regexp.MustCompile("\\A>>? ?([0-9]+)\\n")
 const newlineChar string = "\n"
 const messageHeaderSeparator string = " | " // no hrunicode allowed here yet
 
+// ChatType is an enum of chat types, roughly corresponding to TDLib's one but better
+type ChatType int
+
+const (
+	ChatTypeUnknown ChatType = iota
+	ChatTypePrivate
+	ChatTypeBasicGroup
+	ChatTypeSupergroup
+	ChatTypeSecret
+	ChatTypeChannel
+)
+
+// MembersList is an enum of member list filters
+type MembersList int
+
+const (
+	MembersListMembers MembersList = iota
+	MembersListRestricted
+	MembersListBanned
+	MembersListBannedAndAdministrators
+)
+
+const AVATAR_SIZE_LIMIT int64 = 128 * 1024
+
 // GetContactByUsername resolves username to user id retrieves user and chat information
-func (c *Client) GetContactByUsername(username string) (*client.Chat, *client.User, error) {
+func (c *Client) GetContactByUsername(username string, own bool) (*client.Chat, *client.User, error) {
 	if !c.Online() {
 		return nil, nil, errOffline
 	}
@@ -79,11 +119,11 @@ func (c *Client) GetContactByUsername(username string) (*client.Chat, *client.Us
 		}
 	}
 
-	return c.GetContactByID(userID, chat)
+	return c.GetContactByID(userID, chat, own)
 }
 
 // GetContactByID gets user and chat information from cache (or tries to retrieve it, if missing)
-func (c *Client) GetContactByID(id int64, chat *client.Chat) (*client.Chat, *client.User, error) {
+func (c *Client) GetContactByID(id int64, chat *client.Chat, own bool) (*client.Chat, *client.User, error) {
 	if !c.Online() || id == 0 {
 		return nil, nil, errOffline
 	}
@@ -118,9 +158,9 @@ func (c *Client) GetContactByID(id int64, chat *client.Chat) (*client.Chat, *cli
 				return nil, nil, err
 			}
 
-			c.cache.SetChat(id, cacheChat)
+			c.cache.SetChat(id, cacheChat, own)
 		} else {
-			c.cache.SetChat(id, chat)
+			c.cache.SetChat(id, chat, own)
 		}
 	}
 	if chat == nil {
@@ -130,10 +170,10 @@ func (c *Client) GetContactByID(id int64, chat *client.Chat) (*client.Chat, *cli
 	return chat, user, nil
 }
 
-// IsPM checks if a chat is PM
-func (c *Client) IsPM(id int64) (bool, error) {
+// GetChatType obtains chat type from its information
+func (c *Client) GetChatType(id int64, own bool) (ChatType, error) {
 	if !c.Online() || id == 0 {
-		return false, errOffline
+		return ChatTypeUnknown, errOffline
 	}
 
 	var err error
@@ -144,19 +184,77 @@ func (c *Client) IsPM(id int64) (bool, error) {
 			ChatId: id,
 		})
 		if err != nil {
-			return false, err
+			return ChatTypeUnknown, err
 		}
 
-		c.cache.SetChat(id, chat)
+		c.cache.SetChat(id, chat, own)
 	}
 
 	chatType := chat.Type.ChatTypeType()
-	if chatType == client.TypeChatTypePrivate || chatType == client.TypeChatTypeSecret {
+	if chatType == client.TypeChatTypePrivate {
+		return ChatTypePrivate, nil
+	} else if chatType == client.TypeChatTypeBasicGroup {
+		return ChatTypeBasicGroup, nil
+	} else if chatType == client.TypeChatTypeSupergroup {
+		supergroup, _ := chat.Type.(*client.ChatTypeSupergroup)
+		if supergroup.IsChannel {
+			return ChatTypeChannel, nil
+		}
+		return ChatTypeSupergroup, nil
+	} else if chatType == client.TypeChatTypeSecret {
+		return ChatTypeSecret, nil
+	}
+
+	return ChatTypeUnknown, errors.New("Unknown chat type")
+}
+
+// IsPM checks if a chat is PM
+func (c *Client) IsPM(id int64, own bool) (bool, error) {
+	typ, err := c.GetChatType(id, own)
+	if err != nil {
+		return false, err
+	}
+
+	if typ == ChatTypePrivate || typ == ChatTypeSecret {
 		return true, nil
 	}
 	return false, nil
 }
 
+// IsBot checks if a chat is a bot
+func (c *Client) IsBot(id int64, own bool) (bool, error) {
+	_, user, err := c.GetContactByID(id, nil, own)
+	if err != nil {
+		return false, err
+	}
+	if user == nil || user.Type == nil {
+		return false, nil
+	}
+
+	return user.Type.UserTypeType() == client.TypeUserTypeBot, nil
+}
+
+// GetBotMenu retrieves the bot's attachment menu
+func (c *Client) GetBotMenu(id int64) (*BotLink, []*BotCommand, error) {
+	fullInfo, err := c.client.GetUserFullInfo(&client.GetUserFullInfoRequest{
+		UserId: id,
+	})
+	if err == nil && fullInfo.BotInfo != nil {
+		if fullInfo.BotInfo.MenuButton != nil {
+			menuButton := fullInfo.BotInfo.MenuButton
+			return &BotLink{menuButton.Text, menuButton.Url}, nil, nil
+		} else {
+			var commands []*BotCommand
+			for _, command := range fullInfo.BotInfo.Commands {
+				commands = append(commands, &BotCommand{command.Command, command.Description})
+			}
+			return nil, commands, nil
+		}
+	}
+
+	return nil, nil, err
+}
+
 func (c *Client) userStatusToText(status client.UserStatus, chatID int64) (string, string, string) {
 	var show, textStatus, presenceType string
 
@@ -211,8 +309,93 @@ func (c *Client) LastSeenStatus(timestamp int64) string {
 		Format("Last seen at 15:04 02/01/2006")
 }
 
+func (c *Client) getFileData(tgFile *client.File, typ byte) string {
+	var priority int32
+	if typ == typeFileDataSha1 {
+		priority = 1
+	} else if typ == typeFileDataBase64 {
+		priority = 32
+	}
+
+	// avoid not-well-formed stanza errors
+	if typ == typeFileDataBase64 && c.GetPhotoSize(tgFile) > AVATAR_SIZE_LIMIT {
+		log.Warnf("Photo %v skipped as it's too huge", tgFile.Id)
+		return ""
+	}
+
+	file, path, err := c.ForceOpenFile(tgFile, priority)
+	if err == nil {
+		defer file.Close()
+
+		if typ == typeFileDataSha1 {
+			hash := sha1.New()
+			_, err = io.Copy(hash, file)
+			if err == nil {
+				return fmt.Sprintf("%x", hash.Sum(nil))
+			} else {
+				log.Errorf("Error calculating hash: %v", path)
+			}
+		} else if typ == typeFileDataBase64 {
+			buf := new(bytes.Buffer)
+			binval := base64.NewEncoder(base64.StdEncoding, buf)
+			_, err = io.Copy(binval, file)
+			binval.Close()
+			if err == nil {
+				return buf.String()
+			} else {
+				log.Errorf("Error calculating base64: %v", path)
+			}
+		}
+	} else if path != "" {
+		log.Errorf("Photo does not exist: %v", path)
+	} else {
+		log.Errorf("PHOTO: %#v", err.Error())
+	}
+
+	return ""
+}
+
+// SetEmptyAvatarHash puts a dummy value into the cache to avoid attempting to fetch surely missing avatars
+func (c *Client) SetEmptyAvatarHash(chatId int64) {
+	c.AvatarHashesLock.Lock()
+	c.AvatarHashes[chatId] = &HashedAvatar{
+		Hash: "",
+		File: 0,
+	}
+	c.AvatarHashesLock.Unlock()
+}
+
+// GetPhotoSize return at least a rough size
+func (c *Client) GetPhotoSize(photo *client.File) int64 {
+	if photo == nil {
+		return 0
+	}
+	size := photo.Size
+	if size == 0 {
+		size = photo.ExpectedSize
+	}
+	return size
+}
+
+// GetPhotoSha1 computes the photo hash
+func (c *Client) GetPhotoSha1(photo *client.File, chatId int64) string {
+	sha1 := c.getFileData(photo, typeFileDataSha1)
+	c.AvatarHashesLock.Lock()
+	c.AvatarHashes[chatId] = &HashedAvatar{
+		Hash: sha1,
+		File: photo.Id,
+	}
+	c.AvatarHashesLock.Unlock()
+	return sha1
+}
+
+// GetPhotoBase64 reads file data as Base64
+func (c *Client) GetPhotoBase64(photo *client.File) string {
+	return c.getFileData(photo, typeFileDataBase64)
+}
+
 // ProcessStatusUpdate sets contact status
-func (c *Client) ProcessStatusUpdate(chatID int64, status string, show string, oldArgs ...args.V) error {
+func (c *Client) ProcessStatusUpdate(chatID int64, status string, show string, own bool, oldArgs ...args.V) error {
 	if !c.Online() {
 		return nil
 	}
@@ -221,27 +404,14 @@ func (c *Client) ProcessStatusUpdate(chatID int64, status string, show string, o
 		"chat_id": chatID,
 	}).Info("Status update for")
 
-	chat, user, err := c.GetContactByID(chatID, nil)
+	chat, user, err := c.GetContactByID(chatID, nil, own)
 	if err != nil {
 		return err
 	}
 
 	var photo string
 	if chat != nil && chat.Photo != nil {
-		file, path, err := c.ForceOpenFile(chat.Photo.Small, 1)
-		if err == nil {
-			defer file.Close()
-
-			hash := sha1.New()
-			_, err = io.Copy(hash, file)
-			if err == nil {
-				photo = fmt.Sprintf("%x", hash.Sum(nil))
-			} else {
-				log.Errorf("Error calculating hash: %v", path)
-			}
-		} else if path != "" {
-			log.Errorf("Photo does not exist: %v", path)
-		}
+		photo = c.GetPhotoSha1(chat.Photo.Small, chatID)
 	}
 
 	var presenceType string
@@ -294,12 +464,13 @@ func (c *Client) ProcessStatusUpdate(chatID int64, status string, show string, o
 	return c.sendPresence(newArgs...)
 }
 
-func (c *Client) formatContact(chatID int64) string {
+// FormatContact retrieves a complete "full name (@usernames)" string for display
+func (c *Client) FormatContact(chatID int64) string {
 	if chatID == 0 {
 		return ""
 	}
 
-	chat, user, err := c.GetContactByID(chatID, nil)
+	chat, user, err := c.GetContactByID(chatID, nil, false)
 	if err != nil {
 		return "unknown contact: " + err.Error()
 	}
@@ -326,23 +497,27 @@ func (c *Client) formatContact(chatID int64) string {
 	return str
 }
 
-func (c *Client) getSenderId(message *client.Message) (senderId int64) {
-	if message.SenderId != nil {
-		switch message.SenderId.MessageSenderType() {
+func (c *Client) GetSenderId(sender client.MessageSender) (senderId int64) {
+	switch sender.MessageSenderType() {
 	case client.TypeMessageSenderUser:
-			senderUser, _ := message.SenderId.(*client.MessageSenderUser)
+		senderUser, _ := sender.(*client.MessageSenderUser)
 		senderId = senderUser.UserId
 	case client.TypeMessageSenderChat:
-			senderChat, _ := message.SenderId.(*client.MessageSenderChat)
+		senderChat, _ := sender.(*client.MessageSenderChat)
 		senderId = senderChat.ChatId
 	}
+	return
 }
 
+func (c *Client) getMessageSenderId(message *client.Message) (senderId int64) {
+	if message.SenderId != nil {
+		senderId = c.GetSenderId(message.SenderId)
+	}
 	return
 }
 
 func (c *Client) formatSender(message *client.Message) string {
-	return c.formatContact(c.getSenderId(message))
+	return c.FormatContact(c.getMessageSenderId(message))
 }
 
 func (c *Client) messageToStub(message *client.Message, preview bool, text string) *messageStub {
@@ -392,7 +567,7 @@ func (c *Client) getMessageReply(message *client.Message, preview bool, noConten
 			}
 
 			gatewayReply = &gateway.Reply{
-				Author: fmt.Sprintf("%v@%s", c.getSenderId(replyMsg), gateway.Jid.Full()),
+				Author: fmt.Sprintf("%v@%s", c.getMessageSenderId(replyMsg), gateway.Jid.Full()),
 				Id:     replyId,
 			}
 		} else if !noContent {
@@ -409,7 +584,7 @@ func (c *Client) getMessageReply(message *client.Message, preview bool, noConten
 			}
 
 			tgReply = &messageStub{
-				Sender: c.formatOrigin(replyTo.Origin) + " @ " + c.formatContact(replyTo.ChatId),
+				Sender: c.formatOrigin(replyTo.Origin) + " @ " + c.FormatContact(replyTo.ChatId),
 				Date:   replyTo.OriginSendDate,
 				Text:   text,
 			}
@@ -479,14 +654,14 @@ func (c *Client) formatOrigin(origin client.MessageOrigin) string {
 	switch origin.MessageOriginType() {
 	case client.TypeMessageOriginUser:
 		originUser := origin.(*client.MessageOriginUser)
-		return c.formatContact(originUser.SenderUserId)
+		return c.FormatContact(originUser.SenderUserId)
 	case client.TypeMessageOriginChat:
 		originChat := origin.(*client.MessageOriginChat)
 		var signature string
 		if originChat.AuthorSignature != "" {
 			signature = fmt.Sprintf(" (%s)", originChat.AuthorSignature)
 		}
-		return c.formatContact(originChat.SenderChatId) + signature
+		return c.FormatContact(originChat.SenderChatId) + signature
 	case client.TypeMessageOriginHiddenUser:
 		originUser := origin.(*client.MessageOriginHiddenUser)
 		return originUser.SenderName
@@ -496,7 +671,7 @@ func (c *Client) formatOrigin(origin client.MessageOrigin) string {
 		if channel.AuthorSignature != "" {
 			signature = fmt.Sprintf(" (%s)", channel.AuthorSignature)
 		}
-		return c.formatContact(channel.ChatId) + signature
+		return c.FormatContact(channel.ChatId) + signature
 	}
 	return "Unknown origin type"
 }
@@ -665,13 +840,13 @@ func (c *Client) messageContentToText(content client.MessageContent, chatId int6
 
 		text := "invited "
 		if len(addMembers.MemberUserIds) > 0 {
-			text += c.formatContact(addMembers.MemberUserIds[0])
+			text += c.FormatContact(addMembers.MemberUserIds[0])
 		}
 
 		return text
 	case client.TypeMessageChatDeleteMember:
 		deleteMember, _ := content.(*client.MessageChatDeleteMember)
-		return "kicked " + c.formatContact(deleteMember.UserId)
+		return "kicked " + c.FormatContact(deleteMember.UserId)
 	case client.TypeMessagePinMessage:
 		pinMessage, _ := content.(*client.MessagePinMessage)
 		return "pinned message: " + c.formatMessage(chatId, pinMessage.MessageId, preview, nil)
@@ -821,7 +996,7 @@ func (c *Client) messageContentToText(content client.MessageContent, chatId int6
 		}
 	case client.TypeMessageChatSetMessageAutoDeleteTime:
 		ttl, _ := content.(*client.MessageChatSetMessageAutoDeleteTime)
-		name := c.formatContact(ttl.FromUserId)
+		name := c.FormatContact(ttl.FromUserId)
 		if name == "" {
 			if ttl.MessageAutoDeleteTime == 0 {
 				return "The self-destruct timer was disabled"
@@ -915,8 +1090,8 @@ func (c *Client) isCarbonsEnabled() bool {
 	return gateway.MessageOutgoingPermissionVersion > 0 && c.Session.Carbons
 }
 
-func (c *Client) messageToPrefix(message *client.Message, previewString string, fileString string) (string, *gateway.Reply) {
-	isPM, err := c.IsPM(message.ChatId)
+func (c *Client) messageToPrefix(message *client.Message, previewString string, fileString string, suppressReply bool) (string, *gateway.Reply) {
+	isPM, err := c.IsPM(message.ChatId, true)
 	if err != nil {
 		log.Errorf("Could not determine if chat is PM: %v", err)
 	}
@@ -941,7 +1116,8 @@ func (c *Client) messageToPrefix(message *client.Message, previewString string,
 			}
 		}
 	}
-	if !isPM || !c.Session.HideIds {
+	// with hideids options enabled, hide the id for everything but non-carbons in group chats
+	if (!isPM && !(c.isCarbonsEnabled() && message.IsOutgoing)) || !c.Session.HideIds {
 		prefix = append(prefix, directionChar+strconv.FormatInt(message.Id, 10))
 	}
 	// show sender in group chats
@@ -953,10 +1129,14 @@ func (c *Client) messageToPrefix(message *client.Message, previewString string,
 	}
 
 	// reply to
+	var reply *gateway.Reply
+	if !suppressReply {
 		preview := true
-	reply, tgReply := c.getMessageReply(message, preview, false)
+		gwReply, tgReply := c.getMessageReply(message, preview, false)
 
 		if tgReply != nil {
+			reply = gwReply
+
 			var replyStart, replyEnd int
 
 			if len(prefix) > 0 {
@@ -976,6 +1156,7 @@ func (c *Client) messageToPrefix(message *client.Message, previewString string,
 				reply.End = uint64(replyEnd)
 			}
 		}
+	}
 
 	if message.ForwardInfo != nil {
 		prefix = append(prefix, "fwd: "+c.formatOrigin(message.ForwardInfo.Origin))
@@ -1008,6 +1189,17 @@ func (c *Client) ensureDownloadFile(file *client.File) *client.File {
 	return file
 }
 
+// \n if it is groupchat and message is not empty
+func (c *Client) getPrefixSeparator(chatId int64) string {
+	var separator string
+	if chatId < 0 {
+		separator = "\n"
+	} else if chatId > 0 {
+		separator = " | "
+	}
+	return separator
+}
+
 // ProcessIncomingMessage transfers a message to XMPP side and marks it as read on Telegram side
 func (c *Client) ProcessIncomingMessage(chatId int64, message *client.Message) {
 	isCarbon := c.isCarbonsEnabled() && message.IsOutgoing
@@ -1023,9 +1215,28 @@ func (c *Client) ProcessIncomingMessage(chatId int64, message *client.Message) {
 			ChatId: chatId,
 		})
 		if err == nil {
-			c.cache.SetChat(chatId, chat)
-			go c.ProcessStatusUpdate(chatId, "", "", gateway.SPImmed(true))
+			c.cache.SetChat(chatId, chat, true)
+			go c.ProcessStatusUpdate(chatId, "", "", true, gateway.SPImmed(true))
 			text = "<Chat photo has changed>"
+
+			if chat.Photo == nil {
+				c.SetEmptyAvatarHash(chatId)
+			} else {
+				sha1 := c.GetPhotoSha1(chat.Photo.Small, chatId)
+				size := c.GetPhotoSize(chat.Photo.Small)
+
+				for resource := range c.resourcesRange() {
+					features, ok := c.XmppClientFeatures[resource]
+					if ok && features != nil {
+						for _, feature := range *features {
+							if feature == gateway.NodeAvatarMetadataNotify {
+								go gateway.SendPubSubAvatarNotification(c.xmpp, c.jid+"/"+resource, chatId, sha1, size)
+								break
+							}
+						}
+					}
+				}
+			}
 		}
 	} else {
 		text = c.messageToText(message, false)
@@ -1042,34 +1253,43 @@ func (c *Client) ProcessIncomingMessage(chatId int64, message *client.Message) {
 			fileName, link := c.formatFile(file, false)
 
 			oob = link
-			if c.Session.OOBMode && oob != "" {
-				typ := message.Content.MessageContentType()
-				if typ != client.TypeMessageSticker {
-					auxText = text
+			oobSwap := c.Session.OOBMode && oob != ""
+
+			var ignorePrefix bool
+			if oobSwap {
+				if text == "" || message.Content.MessageContentType() == client.TypeMessageSticker {
+					chatType, err := c.GetChatType(chatId, true)
+					ignorePrefix = err == nil && (chatType != ChatTypeBasicGroup && chatType != ChatTypeSupergroup) && c.isCarbonsEnabled()
 				}
-				text = oob
-			} else if !c.Session.RawMessages {
+			}
+
+			if !c.Session.RawMessages && !ignorePrefix {
 				var newText strings.Builder
 
-				prefix, prefixReply := c.messageToPrefix(message, previewName, fileName)
+				prefix, prefixReply := c.messageToPrefix(message, previewName, fileName, false)
 				reply = prefixReply
 				replyObtained = true
 				newText.WriteString(prefix)
 
 				if text != "" {
-					// \n if it is groupchat and message is not empty
 					if prefix != "" {
-						if chatId < 0 {
-							newText.WriteString("\n")
-						} else if chatId > 0 {
-							newText.WriteString(" | ")
+						separator := c.getPrefixSeparator(chatId)
+						newText.WriteString(separator)
+						if reply != nil {
+							reply.End += uint64(len(separator))
 						}
 					}
-
 					newText.WriteString(text)
 				}
 				text = newText.String()
 			}
+
+			if oobSwap {
+				if !ignorePrefix {
+					auxText = text
+				}
+				text = oob
+			}
 		}
 	}
 	if !replyObtained {
@@ -1109,15 +1329,15 @@ func (c *Client) PrepareOutgoingMessageContent(text string) client.InputMessageC
 }
 
 // ProcessOutgoingMessage executes commands or sends messages to mapped chats, returns message id
-func (c *Client) ProcessOutgoingMessage(chatID int64, text string, returnJid string, replyId int64, replaceId int64) int64 {
+func (c *Client) ProcessOutgoingMessage(chatID int64, text string, returnJid string, replyId int64, replaceId int64, raw bool) int64 {
 	if !c.Online() {
 		// we're offline
 		return 0
 	}
 
-	if replaceId == 0 && (strings.HasPrefix(text, "/") || strings.HasPrefix(text, "!")) {
+	if replaceId == 0 && !raw && (strings.HasPrefix(text, "/") || strings.HasPrefix(text, "!")) {
 		// try to execute commands
-		response, isCommand := c.ProcessChatCommand(chatID, text)
+		response, isCommand, _ := c.ProcessChatCommand(chatID, text)
 		if response != "" {
 			c.returnMessage(returnJid, chatID, response)
 		}
@@ -1240,6 +1460,11 @@ func (c *Client) prepareOutgoingMessageContent(text string, file *client.InputFi
 	return content
 }
 
+// OwnChatsKeys proxies the following function from unexported cache
+func (c *Client) OwnChatsKeys() []int64 {
+	return c.cache.OwnChatsKeys()
+}
+
 // StatusesRange proxies the following function from unexported cache
 func (c *Client) StatusesRange() chan *cache.Status {
 	return c.cache.StatusesRange()
@@ -1294,8 +1519,8 @@ func (c *Client) roster(resource string) {
 
 	log.Warnf("Sending roster for %v", resource)
 
-	for _, chat := range c.cache.ChatsKeys() {
-		c.ProcessStatusUpdate(chat, "", "")
+	for _, chat := range c.cache.OwnChatsKeys() {
+		c.ProcessStatusUpdate(chat, "", "", true)
 	}
 
 	c.sendPresence(gateway.SPStatus("Logged in as: " + c.Session.Login))
@@ -1314,6 +1539,13 @@ func (c *Client) getLastMessages(id int64, query string, from int64, count int32
 	})
 }
 
+// GetFile retrieves a file object by id given by TDlib
+func (c *Client) GetFile(id int32) (*client.File, error) {
+	return c.client.GetFile(&client.GetFileRequest{
+		FileId: id,
+	})
+}
+
 // DownloadFile actually obtains a file by id given by TDlib
 func (c *Client) DownloadFile(id int32, priority int32, synchronous bool) (*client.File, error) {
 	return c.client.DownloadFile(&client.DownloadFileRequest{
@@ -1402,7 +1634,7 @@ func (c *Client) subscribeToID(id int64, chat *client.Chat) {
 	args := gateway.SimplePresence(id, "subscribe")
 
 	if chat == nil {
-		chat, _, _ = c.GetContactByID(id, nil)
+		chat, _, _ = c.GetContactByID(id, nil, true)
 	}
 	if chat != nil {
 		args = append(args, gateway.SPNickname(chat.Title))
@@ -1432,7 +1664,7 @@ func (c *Client) prepareDiskSpace(size uint64) {
 
 func (c *Client) GetVcardInfo(toID int64) (VCardInfo, error) {
 	var info VCardInfo
-	chat, user, err := c.GetContactByID(toID, nil)
+	chat, user, err := c.GetContactByID(toID, nil, false)
 	if err != nil {
 		return info, err
 	}
@@ -1459,7 +1691,7 @@ func (c *Client) GetVcardInfo(toID int64) (VCardInfo, error) {
 }
 
 func (c *Client) UpdateChatNicknames() {
-	for _, id := range c.cache.ChatsKeys() {
+	for _, id := range c.cache.OwnChatsKeys() {
 		chat, ok := c.cache.GetChat(id)
 		if ok {
 			newArgs := []args.V{
@@ -1537,61 +1769,6 @@ func (c *Client) getCarbonFullJids(isOutgoing bool, ignoredResource string) []st
 	return jids
 }
 
-func (c *Client) calculateMessageHash(messageId int64, content client.MessageContent) uint64 {
-	var h maphash.Hash
-	h.SetSeed(c.msgHashSeed)
-
-	buf8 := make([]byte, 8)
-	binary.BigEndian.PutUint64(buf8, uint64(messageId))
-	h.Write(buf8)
-
-	if content != nil && content.MessageContentType() == client.TypeMessageText {
-		textContent, ok := content.(*client.MessageText)
-		if !ok {
-			uhOh()
-		}
-
-		if textContent.Text != nil {
-			h.WriteString(textContent.Text.Text)
-			for _, entity := range textContent.Text.Entities {
-				buf4 := make([]byte, 4)
-				binary.BigEndian.PutUint32(buf4, uint32(entity.Offset))
-				h.Write(buf4)
-				binary.BigEndian.PutUint32(buf4, uint32(entity.Length))
-				h.Write(buf4)
-				h.WriteString(entity.Type.TextEntityTypeType())
-			}
-		}
-	}
-
-	return h.Sum64()
-}
-
-func (c *Client) updateLastMessageHash(chatId, messageId int64, content client.MessageContent) {
-	c.locks.lastMsgHashesLock.Lock()
-	defer c.locks.lastMsgHashesLock.Unlock()
-
-	c.lastMsgHashes[chatId] = c.calculateMessageHash(messageId, content)
-}
-
-func (c *Client) hasLastMessageHashChanged(chatId, messageId int64, content client.MessageContent) bool {
-	c.locks.lastMsgHashesLock.Lock()
-	defer c.locks.lastMsgHashesLock.Unlock()
-
-	oldHash, ok := c.lastMsgHashes[chatId]
-	newHash := c.calculateMessageHash(messageId, content)
-
-	if !ok {
-		log.Warnf("Last message hash for chat %v does not exist", chatId)
-	}
-	log.WithFields(log.Fields{
-		"old hash": oldHash,
-		"new hash": newHash,
-	}).Info("Message hashes")
-
-	return !ok || oldHash != newHash
-}
-
 func (c *Client) UpdateLastChatMessageId(chatId int64, messageId string) {
 	c.locks.lastMsgIdsLock.Lock()
 	defer c.locks.lastMsgIdsLock.Unlock()
@@ -1618,3 +1795,83 @@ func (c *Client) usernamesToString(usernames []string) string {
 	}
 	return strings.Join(atUsernames, ", ")
 }
+
+func (c *Client) GetTZD() string {
+	if c.Session.Timezone == "" {
+		return "Z"
+	}
+	return c.Session.Timezone
+}
+
+// GetChatMembers retrieves a list of chat members. "Limited" mode works only if there are no more than 20 members at all
+func (c *Client) GetChatMembers(chatID int64, limited bool, query string, membersList MembersList) ([]*client.ChatMember, error) {
+	var filters []client.ChatMembersFilter
+	switch membersList {
+	case MembersListMembers:
+		filters = []client.ChatMembersFilter{&client.ChatMembersFilterMembers{}}
+	case MembersListRestricted:
+		filters = []client.ChatMembersFilter{&client.ChatMembersFilterRestricted{}}
+	case MembersListBanned:
+		filters = []client.ChatMembersFilter{&client.ChatMembersFilterBanned{}}
+	case MembersListBannedAndAdministrators:
+		filters = []client.ChatMembersFilter{&client.ChatMembersFilterBanned{}, &client.ChatMembersFilterAdministrators{}}
+	}
+
+	limit := int32(9999)
+	if limited {
+		limit = 20
+
+		chat, _, err := c.GetContactByID(chatID, nil, true)
+		if err != nil {
+			return nil, err
+		} else if chat == nil {
+			return nil, errors.New("Chat not found")
+		}
+
+		chatType := chat.Type.ChatTypeType()
+		if chatType == client.TypeChatTypeBasicGroup {
+			basicGroupType, _ := chat.Type.(*client.ChatTypeBasicGroup)
+			fullInfo, err := c.client.GetBasicGroupFullInfo(&client.GetBasicGroupFullInfoRequest{
+				BasicGroupId: basicGroupType.BasicGroupId,
+			})
+			if err != nil {
+				return nil, err
+			}
+
+			if len(fullInfo.Members) > int(limit) {
+				return nil, errOverLimit
+			}
+
+			return fullInfo.Members, nil
+		} else if chatType == client.TypeChatTypeSupergroup {
+			supergroupType, _ := chat.Type.(*client.ChatTypeSupergroup)
+			fullInfo, err := c.client.GetSupergroupFullInfo(&client.GetSupergroupFullInfoRequest{
+				SupergroupId: supergroupType.SupergroupId,
+			})
+			if err != nil {
+				return nil, err
+			}
+
+			if fullInfo.MemberCount > limit {
+				return nil, errOverLimit
+			}
+		} else {
+			return nil, errors.New("Inapplicable chat type")
+		}
+	}
+
+	var members []*client.ChatMember
+	for _, filter := range filters {
+		chatMembers, err := c.client.SearchChatMembers(&client.SearchChatMembersRequest{
+			ChatId: chatID,
+			Limit:  limit,
+			Query:  query,
+			Filter: filter,
+		})
+		if err != nil {
+			return nil, err
+		}
+		members = append(members, chatMembers.Members...)
+	}
+	return members, nil
+}

telegram/utils_test.go

@@ -436,7 +436,7 @@ func TestMessageToPrefix1(t *testing.T) {
 			},
 		},
 	}
-	prefix, gatewayReply := (&Client{Session: &persistence.Session{}}).messageToPrefix(&message, "", "")
+	prefix, gatewayReply := (&Client{Session: &persistence.Session{}}).messageToPrefix(&message, "", "", false)
 	if prefix != "➡ 42 | fwd: ziz" {
 		t.Errorf("Wrong prefix: %v", prefix)
 	}
@@ -454,7 +454,7 @@ func TestMessageToPrefix2(t *testing.T) {
 			},
 		},
 	}
-	prefix, gatewayReply := (&Client{Session: &persistence.Session{}}).messageToPrefix(&message, "y.jpg", "")
+	prefix, gatewayReply := (&Client{Session: &persistence.Session{}}).messageToPrefix(&message, "y.jpg", "", false)
 	if prefix != "⬅ 56 | fwd:  (zaz) | preview: y.jpg" {
 		t.Errorf("Wrong prefix: %v", prefix)
 	}
@@ -472,7 +472,7 @@ func TestMessageToPrefix3(t *testing.T) {
 			},
 		},
 	}
-	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "", "a.jpg")
+	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "", "a.jpg", false)
 	if prefix != "< 56 | fwd:  (zuz) | file: a.jpg" {
 		t.Errorf("Wrong prefix: %v", prefix)
 	}
@@ -486,7 +486,7 @@ func TestMessageToPrefix4(t *testing.T) {
 		Id:         23,
 		IsOutgoing: true,
 	}
-	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "", "")
+	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "", "", false)
 	if prefix != "> 23" {
 		t.Errorf("Wrong prefix: %v", prefix)
 	}
@@ -504,7 +504,7 @@ func TestMessageToPrefix5(t *testing.T) {
 			},
 		},
 	}
-	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "h.jpg", "a.jpg")
+	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "h.jpg", "a.jpg", false)
 	if prefix != "< 560 | fwd:  (zyz) | preview: h.jpg | file: a.jpg" {
 		t.Errorf("Wrong prefix: %v", prefix)
 	}
@@ -530,7 +530,7 @@ func TestMessageToPrefix6(t *testing.T) {
 			},
 		},
 	}
-	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "", "")
+	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "", "", false)
 	if prefix != "> 23 | reply: ziz @ unknown contact: TDlib instance is offline | tist uz iz" {
 		t.Errorf("Wrong prefix: %v", prefix)
 	}
@@ -556,7 +556,7 @@ func TestMessageToPrefix7(t *testing.T) {
 			},
 		},
 	}
-	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "", "")
+	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "", "", false)
 	if prefix != "> 23 | reply:  (zaz) @ unknown contact: TDlib instance is offline | tist" {
 		t.Errorf("Wrong prefix: %v", prefix)
 	}
@@ -565,9 +565,35 @@ func TestMessageToPrefix7(t *testing.T) {
 	}
 }
 
+func TestMessageToPrefix8(t *testing.T) {
+	message := client.Message{
+		Id:         23,
+		ChatId:     42,
+		IsOutgoing: true,
+		ReplyTo: &client.MessageReplyToMessage{
+			ChatId: 41,
+			Content: &client.MessageText{
+				Text: &client.FormattedText{
+					Text: "tist",
+				},
+			},
+			Origin: &client.MessageOriginChannel{
+				AuthorSignature: "zuz",
+			},
+		},
+	}
+	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "", "", true)
+	if prefix != "> 23" {
+		t.Errorf("Wrong prefix: %v", prefix)
+	}
+	if gatewayReply != nil {
+		t.Errorf("Reply is not nil: %v", gatewayReply)
+	}
+}
+
 func GetSenderIdEmpty(t *testing.T) {
 	message := client.Message{}
-	senderId := (&Client{}).getSenderId(&message)
+	senderId := (&Client{}).getMessageSenderId(&message)
 	if senderId != 0 {
 		t.Errorf("Wrong sender id: %v", senderId)
 	}
@@ -579,7 +605,7 @@ func GetSenderIdUser(t *testing.T) {
 			UserId: 42,
 		},
 	}
-	senderId := (&Client{}).getSenderId(&message)
+	senderId := (&Client{}).getMessageSenderId(&message)
 	if senderId != 42 {
 		t.Errorf("Wrong sender id: %v", senderId)
 	}
@@ -591,7 +617,7 @@ func GetSenderIdChat(t *testing.T) {
 			ChatId: -42,
 		},
 	}
-	senderId := (&Client{}).getSenderId(&message)
+	senderId := (&Client{}).getMessageSenderId(&message)
 	if senderId != -42 {
 		t.Errorf("Wrong sender id: %v", senderId)
 	}

xmpp/component.go

@@ -39,10 +39,11 @@ var sizeRegex = regexp.MustCompile("\\A([0-9]+) ?([KMGTPE]?B?)\\z")
 
 // NewComponent starts a new component and wraps it in
 // a stream manager that you should start yourself
-func NewComponent(conf config.XMPPConfig, tc config.TelegramConfig, idsPath string) (*xmpp.StreamManager, *xmpp.Component, error) {
+func NewComponent(conf config.XMPPConfig, tc config.TelegramConfig, idsPath string, version string) (*xmpp.StreamManager, *xmpp.Component, error) {
 	var err error
 
 	gateway.Jid, err = stanza.NewJid(conf.Jid)
+	gateway.Version = version
 	if err != nil {
 		return nil, nil, err
 	}
@@ -141,11 +142,24 @@ func heartbeat(component *xmpp.Component) {
 						chatID,
 						session.LastSeenStatus(delayedStatus.TimestampOnline),
 						"away",
+						true,
 					)
 					delete(session.DelayedStatuses, chatID)
 				}
 			}
 			session.DelayedStatusesLock.Unlock()
+
+			// shrink message id maps
+			session.MessageIdChangesLock.Lock()
+			for _, idsMap := range session.MessageIdChanges {
+				for oldMessageId, newId := range idsMap {
+					if newId.Ts < now - 60 {
+						newId.Unlock()
+						delete(idsMap, oldMessageId)
+					}
+				}
+			}
+			session.MessageIdChangesLock.Unlock()
 		}
 		sessionLock.Unlock()
 

xmpp/extensions/extensions.go

@@ -213,6 +213,20 @@ type QueryRegisterRemove struct {
 	XMLName xml.Name `xml:"remove"`
 }
 
+// EntityTime is from XEP-0202
+type EntityTime struct {
+	XMLName   xml.Name          `xml:"urn:xmpp:time time"`
+	Tzo       string            `xml:"tzo"`
+	Utc       string            `xml:"utc"`
+	ResultSet *stanza.ResultSet `xml:"set,omitempty"`
+}
+
+// AttachTo is from XEP-0367
+type AttachTo struct {
+	XMLName xml.Name `xml:"urn:xmpp:message-attaching:1 attach-to"`
+	Id      string   `xml:"id,attr"`
+}
+
 // Namespace is a namespace!
 func (c PresenceNickExtension) Namespace() string {
 	return c.XMLName.Space
@@ -278,11 +292,26 @@ func (c QueryRegister) GetSet() *stanza.ResultSet {
 	return c.ResultSet
 }
 
+// Namespace is a namespace!
+func (c EntityTime) Namespace() string {
+	return c.XMLName.Space
+}
+
+// GetSet getsets!
+func (c EntityTime) GetSet() *stanza.ResultSet {
+	return c.ResultSet
+}
+
 // Name is a packet name
 func (ClientMessage) Name() string {
 	return "message"
 }
 
+// Namespace is a namespace!
+func (c AttachTo) Namespace() string {
+	return c.XMLName.Space
+}
+
 // NewReplyFallback initializes a fallback range
 func NewReplyFallback(start uint64, end uint64) Fallback {
 	return Fallback{
@@ -362,4 +391,16 @@ func init() {
 		"jabber:iq:register",
 		"query",
 	}, QueryRegister{})
+
+	// entity time
+	stanza.TypeRegistry.MapExtension(stanza.PKTIQ, xml.Name{
+		"urn:xmpp:time",
+		"time",
+	}, EntityTime{})
+
+	// attach-to
+	stanza.TypeRegistry.MapExtension(stanza.PKTMessage, xml.Name{
+		"urn:xmpp:message-attaching:1",
+		"attach-to",
+	}, AttachTo{})
 }

xmpp/gateway/gateway.go

@@ -10,6 +10,7 @@ import (
 	"dev.narayana.im/narayana/telegabber/badger"
 	"dev.narayana.im/narayana/telegabber/xmpp/extensions"
 
+	"github.com/google/uuid"
 	log "github.com/sirupsen/logrus"
 	"github.com/soheilhy/args"
 	"gosrc.io/xmpp"
@@ -36,6 +37,11 @@ type marker struct {
 }
 
 const NSNick string = "http://jabber.org/protocol/nick"
+const NodeVCard4 string = "urn:xmpp:vcard4"
+const NodeAvatarMetadata string = "urn:xmpp:avatar:metadata"
+const NodeAvatarMetadataNotify string = NodeAvatarMetadata + "+notify"
+const NodeAvatarData string = "urn:xmpp:avatar:data"
+const NSCommand string = "http://jabber.org/protocol/commands"
 
 // Queue stores presences to send later
 var Queue = make(map[string]*stanza.Presence)
@@ -44,6 +50,9 @@ var QueueLock = sync.Mutex{}
 // Jid stores the component's JID object
 var Jid *stanza.Jid
 
+// Version stores this software's version
+var Version string
+
 // IdsDB provides a disk-backed bidirectional dictionary of Telegram and XMPP ids
 var IdsDB badger.IdsDB
 
@@ -61,12 +70,20 @@ func SendMessage(to string, from string, body string, id string, component *xmpp
 
 // SendServiceMessage creates and sends a simple message stanza from transport
 func SendServiceMessage(to string, body string, component *xmpp.Component) {
-	sendMessageWrapper(to, "", body, "", component, nil, nil, "", "", false, false)
+	var id string
+	if uuid, err := uuid.NewRandom(); err == nil {
+		id = uuid.String()
+	}
+	sendMessageWrapper(to, "", body, id, component, nil, nil, "", "", false, false)
 }
 
 // SendTextMessage creates and sends a simple message stanza
 func SendTextMessage(to string, from string, body string, component *xmpp.Component) {
-	sendMessageWrapper(to, from, body, "", component, nil, nil, "", "", false, false)
+	var id string
+	if uuid, err := uuid.NewRandom(); err == nil {
+		id = uuid.String()
+	}
+	sendMessageWrapper(to, from, body, id, component, nil, nil, "", "", false, false)
 }
 
 // SendMessageWithOOB creates and sends a message stanza with OOB URL
@@ -426,3 +443,46 @@ func SplitJID(from string) (string, string, bool) {
 	}
 	return fromJid.Bare(), fromJid.Resource, true
 }
+
+// SendPubSubAvatarNotification encourages clients to fetch an avatar
+func SendPubSubAvatarNotification(component *xmpp.Component, jid string, chatId int64, sha1 string, size int64) {
+	info := stanza.Node{
+		XMLName: xml.Name{Local: "info"},
+		Attrs: []xml.Attr{
+			xml.Attr{Name: xml.Name{Local: "bytes"}, Value: strconv.FormatInt(size, 10)},
+			xml.Attr{Name: xml.Name{Local: "height"}, Value: "160"},
+			xml.Attr{Name: xml.Name{Local: "id"}, Value: sha1},
+			xml.Attr{Name: xml.Name{Local: "type"}, Value: "image/jpeg"},
+			xml.Attr{Name: xml.Name{Local: "width"}, Value: "160"},
+		},
+	}
+	log.WithFields(log.Fields{
+		"chatId": chatId,
+	}).Debugf("%#v", info)
+
+	event := &stanza.PubSubEvent{
+		EventElement: &stanza.ItemsEvent{
+			Node: NodeAvatarMetadata,
+			Items: []stanza.ItemEvent{
+				stanza.ItemEvent{
+					Id: sha1,
+					Any: &stanza.Node{
+						XMLName: xml.Name{Local: "metadata", Space: NodeAvatarMetadata},
+						Nodes:   []stanza.Node{info},
+					},
+				},
+			},
+		},
+	}
+
+	message := stanza.Message{
+		Attrs: stanza.Attrs{
+			From: strconv.FormatInt(chatId, 10) + "@" + Jid.Bare(),
+			To:   jid,
+			Type: stanza.MessageTypeHeadline,
+		},
+		Extensions: []stanza.MsgExtension{event},
+	}
+
+	_ = ResumableSend(component, message)
+}

xmpp/loginwizard.go

@@ -0,0 +1,101 @@
+package xmpp
+
+import (
+	"fmt"
+
+	"dev.narayana.im/narayana/telegabber/telegram"
+	"dev.narayana.im/narayana/telegabber/xmpp/gateway"
+
+	log "github.com/sirupsen/logrus"
+	"gosrc.io/xmpp"
+	"gosrc.io/xmpp/stanza"
+)
+
+func loginWizardPayload(bare string, requestForm *stanza.Form, resource string) (payload *stanza.Command, cancelSend bool, returnSession *telegram.Client) {
+	payload = &stanza.Command{
+		SessionId: "loginwizard",
+		Node:      "loginwizard",
+	}
+
+	session, ok := sessions[bare]
+	if ok {
+		returnSession = session
+
+		if requestForm == nil {
+			session.StartLoginWizard(false)
+			cancelSend = true
+		} else {
+			if len(requestForm.Fields) != 1 {
+				setCommandPayloadError(payload, "Hey, don't tinker with the form!")
+				return
+			}
+			field := requestForm.Fields[0]
+			if field != nil {
+				if len(field.ValuesList) < 1 {
+					setCommandPayloadError(payload, "No value")
+					return
+				}
+				switch field.Var {
+				case "login", "code", "password":
+				default:
+					setCommandPayloadError(payload, "Unknown field")
+					return
+				}
+
+				session.StartLoginWizard(true)
+				response, success := session.ProcessTransportCommand(fmt.Sprintf("/%v %v", field.Var, field.ValuesList[0]), resource)
+				if !success {
+					setCommandPayloadError(payload, response)
+					session.StopLoginWizard()
+					return
+				}
+
+				cancelSend = true
+			}
+		}
+	} else {
+		setCommandPayloadError(payload, fmt.Sprintf("Session is not initialized, add the transport (%v) to contacts first", gateway.Jid.Bare()))
+	}
+
+	return
+}
+
+func sendLoginWizardResponse(component *xmpp.Component, answer *stanza.IQ, session *telegram.Client) {
+	payload := &stanza.Command{
+		SessionId: "loginwizard",
+		Node:      "loginwizard",
+	}
+
+	nextStage := "login"
+	if session != nil {
+		nextStage = session.GetLoginWizardNextStage()
+	}
+	log.Debugf("nextStage: %v", nextStage)
+
+	if nextStage == "cancel" {
+		setCommandPayloadError(payload, "Cancelled")
+		session.StopLoginWizard()
+	} else if nextStage == "success" {
+		payload.Status = stanza.CommandStatusCompleted
+		session.StopLoginWizard()
+	} else {
+		required := ""
+		form := stanza.Form{
+			Type:  stanza.FormTypeForm,
+			Title: "Login Wizard",
+			Fields: []*stanza.Field{
+				&stanza.Field{
+					Var:      nextStage,
+					Label:    nextStage,
+					Required: &required,
+				},
+			},
+		}
+		payload.Status = stanza.CommandStatusExecuting
+		payload.CommandElements = append(payload.CommandElements, &form)
+	}
+
+	answer.Payload = payload
+
+	gateway.ResumableSend(component, answer)
+}