Add whatsappmulti buildflag for whatsapp with multidevice support (whatsapp)

vendor/github.com/Rhymen/go-whatsapp/binary/decoder.go

@@ -0,0 +1,388 @@
+package binary
+
+import (
+	"fmt"
+	"github.com/Rhymen/go-whatsapp/binary/token"
+	"io"
+	"strconv"
+)
+
+type binaryDecoder struct {
+	data  []byte
+	index int
+}
+
+func NewDecoder(data []byte) *binaryDecoder {
+	return &binaryDecoder{data, 0}
+}
+
+func (r *binaryDecoder) checkEOS(length int) error {
+	if r.index+length > len(r.data) {
+		return io.EOF
+	}
+
+	return nil
+}
+
+func (r *binaryDecoder) readByte() (byte, error) {
+	if err := r.checkEOS(1); err != nil {
+		return 0, err
+	}
+
+	b := r.data[r.index]
+	r.index++
+
+	return b, nil
+}
+
+func (r *binaryDecoder) readIntN(n int, littleEndian bool) (int, error) {
+	if err := r.checkEOS(n); err != nil {
+		return 0, err
+	}
+
+	var ret int
+
+	for i := 0; i < n; i++ {
+		var curShift int
+		if littleEndian {
+			curShift = i
+		} else {
+			curShift = n - i - 1
+		}
+		ret |= int(r.data[r.index+i]) << uint(curShift*8)
+	}
+
+	r.index += n
+	return ret, nil
+}
+
+func (r *binaryDecoder) readInt8(littleEndian bool) (int, error) {
+	return r.readIntN(1, littleEndian)
+}
+
+func (r *binaryDecoder) readInt16(littleEndian bool) (int, error) {
+	return r.readIntN(2, littleEndian)
+}
+
+func (r *binaryDecoder) readInt20() (int, error) {
+	if err := r.checkEOS(3); err != nil {
+		return 0, err
+	}
+
+	ret := ((int(r.data[r.index]) & 15) << 16) + (int(r.data[r.index+1]) << 8) + int(r.data[r.index+2])
+	r.index += 3
+	return ret, nil
+}
+
+func (r *binaryDecoder) readInt32(littleEndian bool) (int, error) {
+	return r.readIntN(4, littleEndian)
+}
+
+func (r *binaryDecoder) readInt64(littleEndian bool) (int, error) {
+	return r.readIntN(8, littleEndian)
+}
+
+func (r *binaryDecoder) readPacked8(tag int) (string, error) {
+	startByte, err := r.readByte()
+	if err != nil {
+		return "", err
+	}
+
+	ret := ""
+
+	for i := 0; i < int(startByte&127); i++ {
+		currByte, err := r.readByte()
+		if err != nil {
+			return "", err
+		}
+
+		lower, err := unpackByte(tag, currByte&0xF0>>4)
+		if err != nil {
+			return "", err
+		}
+
+		upper, err := unpackByte(tag, currByte&0x0F)
+		if err != nil {
+			return "", err
+		}
+
+		ret += lower + upper
+	}
+
+	if startByte>>7 != 0 {
+		ret = ret[:len(ret)-1]
+	}
+	return ret, nil
+}
+
+func unpackByte(tag int, value byte) (string, error) {
+	switch tag {
+	case token.NIBBLE_8:
+		return unpackNibble(value)
+	case token.HEX_8:
+		return unpackHex(value)
+	default:
+		return "", fmt.Errorf("unpackByte with unknown tag %d", tag)
+	}
+}
+
+func unpackNibble(value byte) (string, error) {
+	switch {
+	case value < 0 || value > 15:
+		return "", fmt.Errorf("unpackNibble with value %d", value)
+	case value == 10:
+		return "-", nil
+	case value == 11:
+		return ".", nil
+	case value == 15:
+		return "\x00", nil
+	default:
+		return strconv.Itoa(int(value)), nil
+	}
+}
+
+func unpackHex(value byte) (string, error) {
+	switch {
+	case value < 0 || value > 15:
+		return "", fmt.Errorf("unpackHex with value %d", value)
+	case value < 10:
+		return strconv.Itoa(int(value)), nil
+	default:
+		return string('A' + value - 10), nil
+	}
+}
+
+func (r *binaryDecoder) readListSize(tag int) (int, error) {
+	switch tag {
+	case token.LIST_EMPTY:
+		return 0, nil
+	case token.LIST_8:
+		return r.readInt8(false)
+	case token.LIST_16:
+		return r.readInt16(false)
+	default:
+		return 0, fmt.Errorf("readListSize with unknown tag %d at position %d", tag, r.index)
+	}
+}
+
+func (r *binaryDecoder) readString(tag int) (string, error) {
+	switch {
+	case tag >= 3 && tag <= len(token.SingleByteTokens):
+		tok, err := token.GetSingleToken(tag)
+		if err != nil {
+			return "", err
+		}
+
+		if tok == "s.whatsapp.net" {
+			tok = "c.us"
+		}
+
+		return tok, nil
+	case tag == token.DICTIONARY_0 || tag == token.DICTIONARY_1 || tag == token.DICTIONARY_2 || tag == token.DICTIONARY_3:
+		i, err := r.readInt8(false)
+		if err != nil {
+			return "", err
+		}
+
+		return token.GetDoubleToken(tag-token.DICTIONARY_0, i)
+	case tag == token.LIST_EMPTY:
+		return "", nil
+	case tag == token.BINARY_8:
+		length, err := r.readInt8(false)
+		if err != nil {
+			return "", err
+		}
+
+		return r.readStringFromChars(length)
+	case tag == token.BINARY_20:
+		length, err := r.readInt20()
+		if err != nil {
+			return "", err
+		}
+
+		return r.readStringFromChars(length)
+	case tag == token.BINARY_32:
+		length, err := r.readInt32(false)
+		if err != nil {
+			return "", err
+		}
+
+		return r.readStringFromChars(length)
+	case tag == token.JID_PAIR:
+		b, err := r.readByte()
+		if err != nil {
+			return "", err
+		}
+		i, err := r.readString(int(b))
+		if err != nil {
+			return "", err
+		}
+
+		b, err = r.readByte()
+		if err != nil {
+			return "", err
+		}
+		j, err := r.readString(int(b))
+		if err != nil {
+			return "", err
+		}
+
+		if i == "" || j == "" {
+			return "", fmt.Errorf("invalid jid pair: %s - %s", i, j)
+		}
+
+		return i + "@" + j, nil
+	case tag == token.NIBBLE_8 || tag == token.HEX_8:
+		return r.readPacked8(tag)
+	default:
+		return "", fmt.Errorf("invalid string with tag %d", tag)
+	}
+}
+
+func (r *binaryDecoder) readStringFromChars(length int) (string, error) {
+	if err := r.checkEOS(length); err != nil {
+		return "", err
+	}
+
+	ret := r.data[r.index : r.index+length]
+	r.index += length
+
+	return string(ret), nil
+}
+
+func (r *binaryDecoder) readAttributes(n int) (map[string]string, error) {
+	if n == 0 {
+		return nil, nil
+	}
+
+	ret := make(map[string]string)
+	for i := 0; i < n; i++ {
+		idx, err := r.readInt8(false)
+		if err != nil {
+			return nil, err
+		}
+
+		index, err := r.readString(idx)
+		if err != nil {
+			return nil, err
+		}
+
+		idx, err = r.readInt8(false)
+		if err != nil {
+			return nil, err
+		}
+
+		ret[index], err = r.readString(idx)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return ret, nil
+}
+
+func (r *binaryDecoder) readList(tag int) ([]Node, error) {
+	size, err := r.readListSize(tag)
+	if err != nil {
+		return nil, err
+	}
+
+	ret := make([]Node, size)
+	for i := 0; i < size; i++ {
+		n, err := r.ReadNode()
+
+		if err != nil {
+			return nil, err
+		}
+
+		ret[i] = *n
+	}
+
+	return ret, nil
+}
+
+func (r *binaryDecoder) ReadNode() (*Node, error) {
+	ret := &Node{}
+
+	size, err := r.readInt8(false)
+	if err != nil {
+		return nil, err
+	}
+	listSize, err := r.readListSize(size)
+	if err != nil {
+		return nil, err
+	}
+
+	descrTag, err := r.readInt8(false)
+	if descrTag == token.STREAM_END {
+		return nil, fmt.Errorf("unexpected stream end")
+	}
+	ret.Description, err = r.readString(descrTag)
+	if err != nil {
+		return nil, err
+	}
+	if listSize == 0 || ret.Description == "" {
+		return nil, fmt.Errorf("invalid Node")
+	}
+
+	ret.Attributes, err = r.readAttributes((listSize - 1) >> 1)
+	if err != nil {
+		return nil, err
+	}
+
+	if listSize%2 == 1 {
+		return ret, nil
+	}
+
+	tag, err := r.readInt8(false)
+	if err != nil {
+		return nil, err
+	}
+
+	switch tag {
+	case token.LIST_EMPTY, token.LIST_8, token.LIST_16:
+		ret.Content, err = r.readList(tag)
+	case token.BINARY_8:
+		size, err = r.readInt8(false)
+		if err != nil {
+			return nil, err
+		}
+
+		ret.Content, err = r.readBytes(size)
+	case token.BINARY_20:
+		size, err = r.readInt20()
+		if err != nil {
+			return nil, err
+		}
+
+		ret.Content, err = r.readBytes(size)
+	case token.BINARY_32:
+		size, err = r.readInt32(false)
+		if err != nil {
+			return nil, err
+		}
+
+		ret.Content, err = r.readBytes(size)
+	default:
+		ret.Content, err = r.readString(tag)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	return ret, nil
+}
+
+func (r *binaryDecoder) readBytes(n int) ([]byte, error) {
+	ret := make([]byte, n)
+	var err error
+
+	for i := range ret {
+		ret[i], err = r.readByte()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return ret, nil
+}

vendor/github.com/Rhymen/go-whatsapp/binary/encoder.go

@@ -0,0 +1,351 @@
+package binary
+
+import (
+	"fmt"
+	"github.com/Rhymen/go-whatsapp/binary/token"
+	"math"
+	"strconv"
+	"strings"
+)
+
+type binaryEncoder struct {
+	data []byte
+}
+
+func NewEncoder() *binaryEncoder {
+	return &binaryEncoder{make([]byte, 0)}
+}
+
+func (w *binaryEncoder) GetData() []byte {
+	return w.data
+}
+
+func (w *binaryEncoder) pushByte(b byte) {
+	w.data = append(w.data, b)
+}
+
+func (w *binaryEncoder) pushBytes(bytes []byte) {
+	w.data = append(w.data, bytes...)
+}
+
+func (w *binaryEncoder) pushIntN(value, n int, littleEndian bool) {
+	for i := 0; i < n; i++ {
+		var curShift int
+		if littleEndian {
+			curShift = i
+		} else {
+			curShift = n - i - 1
+		}
+		w.pushByte(byte((value >> uint(curShift*8)) & 0xFF))
+	}
+}
+
+func (w *binaryEncoder) pushInt20(value int) {
+	w.pushBytes([]byte{byte((value >> 16) & 0x0F), byte((value >> 8) & 0xFF), byte(value & 0xFF)})
+}
+
+func (w *binaryEncoder) pushInt8(value int) {
+	w.pushIntN(value, 1, false)
+}
+
+func (w *binaryEncoder) pushInt16(value int) {
+	w.pushIntN(value, 2, false)
+}
+
+func (w *binaryEncoder) pushInt32(value int) {
+	w.pushIntN(value, 4, false)
+}
+
+func (w *binaryEncoder) pushInt64(value int) {
+	w.pushIntN(value, 8, false)
+}
+
+func (w *binaryEncoder) pushString(value string) {
+	w.pushBytes([]byte(value))
+}
+
+func (w *binaryEncoder) writeByteLength(length int) error {
+	if length > math.MaxInt32 {
+		return fmt.Errorf("length is too large: %d", length)
+	} else if length >= (1 << 20) {
+		w.pushByte(token.BINARY_32)
+		w.pushInt32(length)
+	} else if length >= 256 {
+		w.pushByte(token.BINARY_20)
+		w.pushInt20(length)
+	} else {
+		w.pushByte(token.BINARY_8)
+		w.pushInt8(length)
+	}
+
+	return nil
+}
+
+func (w *binaryEncoder) WriteNode(n Node) error {
+	numAttributes := 0
+	if n.Attributes != nil {
+		numAttributes = len(n.Attributes)
+	}
+
+	hasContent := 0
+	if n.Content != nil {
+		hasContent = 1
+	}
+
+	w.writeListStart(2*numAttributes + 1 + hasContent)
+	if err := w.writeString(n.Description, false); err != nil {
+		return err
+	}
+
+	if err := w.writeAttributes(n.Attributes); err != nil {
+		return err
+	}
+
+	if err := w.writeChildren(n.Content); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (w *binaryEncoder) writeString(tok string, i bool) error {
+	if !i && tok == "c.us" {
+		if err := w.writeToken(token.IndexOfSingleToken("s.whatsapp.net")); err != nil {
+			return err
+		}
+		return nil
+	}
+
+	tokenIndex := token.IndexOfSingleToken(tok)
+	if tokenIndex == -1 {
+		jidSepIndex := strings.Index(tok, "@")
+		if jidSepIndex < 1 {
+			w.writeStringRaw(tok)
+		} else {
+			w.writeJid(tok[:jidSepIndex], tok[jidSepIndex+1:])
+		}
+	} else {
+		if tokenIndex < token.SINGLE_BYTE_MAX {
+			if err := w.writeToken(tokenIndex); err != nil {
+				return err
+			}
+		} else {
+			singleByteOverflow := tokenIndex - token.SINGLE_BYTE_MAX
+			dictionaryIndex := singleByteOverflow >> 8
+			if dictionaryIndex < 0 || dictionaryIndex > 3 {
+				return fmt.Errorf("double byte dictionary token out of range: %v", tok)
+			}
+			if err := w.writeToken(token.DICTIONARY_0 + dictionaryIndex); err != nil {
+				return err
+			}
+			if err := w.writeToken(singleByteOverflow % 256); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (w *binaryEncoder) writeStringRaw(value string) error {
+	if err := w.writeByteLength(len(value)); err != nil {
+		return err
+	}
+
+	w.pushString(value)
+
+	return nil
+}
+
+func (w *binaryEncoder) writeJid(jidLeft, jidRight string) error {
+	w.pushByte(token.JID_PAIR)
+
+	if jidLeft != "" {
+		if err := w.writePackedBytes(jidLeft); err != nil {
+			return err
+		}
+	} else {
+		if err := w.writeToken(token.LIST_EMPTY); err != nil {
+			return err
+		}
+	}
+
+	if err := w.writeString(jidRight, false); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (w *binaryEncoder) writeToken(tok int) error {
+	if tok < len(token.SingleByteTokens) {
+		w.pushByte(byte(tok))
+	} else if tok <= 500 {
+		return fmt.Errorf("invalid token: %d", tok)
+	}
+
+	return nil
+}
+
+func (w *binaryEncoder) writeAttributes(attributes map[string]string) error {
+	if attributes == nil {
+		return nil
+	}
+
+	for key, val := range attributes {
+		if val == "" {
+			continue
+		}
+
+		if err := w.writeString(key, false); err != nil {
+			return err
+		}
+
+		if err := w.writeString(val, false); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (w *binaryEncoder) writeChildren(children interface{}) error {
+	if children == nil {
+		return nil
+	}
+
+	switch childs := children.(type) {
+	case string:
+		if err := w.writeString(childs, true); err != nil {
+			return err
+		}
+	case []byte:
+		if err := w.writeByteLength(len(childs)); err != nil {
+			return err
+		}
+
+		w.pushBytes(childs)
+	case []Node:
+		w.writeListStart(len(childs))
+		for _, n := range childs {
+			if err := w.WriteNode(n); err != nil {
+				return err
+			}
+		}
+	default:
+		return fmt.Errorf("cannot write child of type: %T", children)
+	}
+
+	return nil
+}
+
+func (w *binaryEncoder) writeListStart(listSize int) {
+	if listSize == 0 {
+		w.pushByte(byte(token.LIST_EMPTY))
+	} else if listSize < 256 {
+		w.pushByte(byte(token.LIST_8))
+		w.pushInt8(listSize)
+	} else {
+		w.pushByte(byte(token.LIST_16))
+		w.pushInt16(listSize)
+	}
+}
+
+func (w *binaryEncoder) writePackedBytes(value string) error {
+	if err := w.writePackedBytesImpl(value, token.NIBBLE_8); err != nil {
+		if err := w.writePackedBytesImpl(value, token.HEX_8); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (w *binaryEncoder) writePackedBytesImpl(value string, dataType int) error {
+	numBytes := len(value)
+	if numBytes > token.PACKED_MAX {
+		return fmt.Errorf("too many bytes to pack: %d", numBytes)
+	}
+
+	w.pushByte(byte(dataType))
+
+	x := 0
+	if numBytes%2 != 0 {
+		x = 128
+	}
+	w.pushByte(byte(x | int(math.Ceil(float64(numBytes)/2.0))))
+	for i, l := 0, numBytes/2; i < l; i++ {
+		b, err := w.packBytePair(dataType, value[2*i:2*i+1], value[2*i+1:2*i+2])
+		if err != nil {
+			return err
+		}
+
+		w.pushByte(byte(b))
+	}
+
+	if (numBytes % 2) != 0 {
+		b, err := w.packBytePair(dataType, value[numBytes-1:], "\x00")
+		if err != nil {
+			return err
+		}
+
+		w.pushByte(byte(b))
+	}
+
+	return nil
+}
+
+func (w *binaryEncoder) packBytePair(packType int, part1, part2 string) (int, error) {
+	if packType == token.NIBBLE_8 {
+		n1, err := packNibble(part1)
+		if err != nil {
+			return 0, err
+		}
+
+		n2, err := packNibble(part2)
+		if err != nil {
+			return 0, err
+		}
+
+		return (n1 << 4) | n2, nil
+	} else if packType == token.HEX_8 {
+		n1, err := packHex(part1)
+		if err != nil {
+			return 0, err
+		}
+
+		n2, err := packHex(part2)
+		if err != nil {
+			return 0, err
+		}
+
+		return (n1 << 4) | n2, nil
+	} else {
+		return 0, fmt.Errorf("invalid pack type (%d) for byte pair: %s / %s", packType, part1, part2)
+	}
+}
+
+func packNibble(value string) (int, error) {
+	if value >= "0" && value <= "9" {
+		return strconv.Atoi(value)
+	} else if value == "-" {
+		return 10, nil
+	} else if value == "." {
+		return 11, nil
+	} else if value == "\x00" {
+		return 15, nil
+	}
+
+	return 0, fmt.Errorf("invalid string to pack as nibble: %v", value)
+}
+
+func packHex(value string) (int, error) {
+	if (value >= "0" && value <= "9") || (value >= "A" && value <= "F") || (value >= "a" && value <= "f") {
+		d, err := strconv.ParseInt(value, 16, 0)
+		return int(d), err
+	} else if value == "\x00" {
+		return 15, nil
+	}
+
+	return 0, fmt.Errorf("invalid string to pack as hex: %v", value)
+}

vendor/github.com/Rhymen/go-whatsapp/binary/node.go

@@ -0,0 +1,106 @@
+package binary
+
+import (
+	"fmt"
+	pb "github.com/Rhymen/go-whatsapp/binary/proto"
+	"github.com/golang/protobuf/proto"
+)
+
+type Node struct {
+	Description string
+	Attributes  map[string]string
+	Content     interface{}
+}
+
+func Marshal(n Node) ([]byte, error) {
+	if n.Attributes != nil && n.Content != nil {
+		a, err := marshalMessageArray(n.Content.([]interface{}))
+		if err != nil {
+			return nil, err
+		}
+		n.Content = a
+	}
+
+	w := NewEncoder()
+	if err := w.WriteNode(n); err != nil {
+		return nil, err
+	}
+
+	return w.GetData(), nil
+}
+
+func marshalMessageArray(messages []interface{}) ([]Node, error) {
+	ret := make([]Node, len(messages))
+
+	for i, m := range messages {
+		if wmi, ok := m.(*pb.WebMessageInfo); ok {
+			b, err := marshalWebMessageInfo(wmi)
+			if err != nil {
+				return nil, nil
+			}
+			ret[i] = Node{"message", nil, b}
+		} else {
+			ret[i], ok = m.(Node)
+			if !ok {
+				return nil, fmt.Errorf("invalid Node")
+			}
+		}
+	}
+
+	return ret, nil
+}
+
+func marshalWebMessageInfo(p *pb.WebMessageInfo) ([]byte, error) {
+	b, err := proto.Marshal(p)
+	if err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+func Unmarshal(data []byte) (*Node, error) {
+	r := NewDecoder(data)
+	n, err := r.ReadNode()
+	if err != nil {
+		return nil, err
+	}
+
+	if n != nil && n.Attributes != nil && n.Content != nil {
+		nContent, ok := n.Content.([]Node)
+		if ok {
+			n.Content, err = unmarshalMessageArray(nContent)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return n, nil
+}
+
+func unmarshalMessageArray(messages []Node) ([]interface{}, error) {
+	ret := make([]interface{}, len(messages))
+
+	for i, msg := range messages {
+		if msg.Description == "message" {
+			info, err := unmarshalWebMessageInfo(msg.Content.([]byte))
+			if err != nil {
+				return nil, err
+			}
+			ret[i] = info
+		} else {
+			ret[i] = msg
+		}
+	}
+
+	return ret, nil
+}
+
+func unmarshalWebMessageInfo(msg []byte) (*pb.WebMessageInfo, error) {
+	message := &pb.WebMessageInfo{}
+	err := proto.Unmarshal(msg, message)
+	if err != nil {
+		return nil, err
+	}
+	return message, nil
+}

vendor/github.com/Rhymen/go-whatsapp/binary/token/token.go

@@ -0,0 +1,81 @@
+package token
+
+import "fmt"
+
+var SingleByteTokens = [...]string{"", "", "", "200", "400", "404", "500", "501", "502", "action", "add",
+	"after", "archive", "author", "available", "battery", "before", "body",
+	"broadcast", "chat", "clear", "code", "composing", "contacts", "count",
+	"create", "debug", "delete", "demote", "duplicate", "encoding", "error",
+	"false", "filehash", "from", "g.us", "group", "groups_v2", "height", "id",
+	"image", "in", "index", "invis", "item", "jid", "kind", "last", "leave",
+	"live", "log", "media", "message", "mimetype", "missing", "modify", "name",
+	"notification", "notify", "out", "owner", "participant", "paused",
+	"picture", "played", "presence", "preview", "promote", "query", "raw",
+	"read", "receipt", "received", "recipient", "recording", "relay",
+	"remove", "response", "resume", "retry", "s.whatsapp.net", "seconds",
+	"set", "size", "status", "subject", "subscribe", "t", "text", "to", "true",
+	"type", "unarchive", "unavailable", "url", "user", "value", "web", "width",
+	"mute", "read_only", "admin", "creator", "short", "update", "powersave",
+	"checksum", "epoch", "block", "previous", "409", "replaced", "reason",
+	"spam", "modify_tag", "message_info", "delivery", "emoji", "title",
+	"description", "canonical-url", "matched-text", "star", "unstar",
+	"media_key", "filename", "identity", "unread", "page", "page_count",
+	"search", "media_message", "security", "call_log", "profile", "ciphertext",
+	"invite", "gif", "vcard", "frequent", "privacy", "blacklist", "whitelist",
+	"verify", "location", "document", "elapsed", "revoke_invite", "expiration",
+	"unsubscribe", "disable", "vname", "old_jid", "new_jid", "announcement",
+	"locked", "prop", "label", "color", "call", "offer", "call-id",
+	"quick_reply", "sticker", "pay_t", "accept", "reject", "sticker_pack",
+	"invalid", "canceled", "missed", "connected", "result", "audio",
+	"video", "recent"}
+
+var doubleByteTokens = [...]string{}
+
+func GetSingleToken(i int) (string, error) {
+	if i < 3 || i >= len(SingleByteTokens) {
+		return "", fmt.Errorf("index out of single byte token bounds %d", i)
+	}
+
+	return SingleByteTokens[i], nil
+}
+
+func GetDoubleToken(index1 int, index2 int) (string, error) {
+	n := 256*index1 + index2
+	if n < 0 || n >= len(doubleByteTokens) {
+		return "", fmt.Errorf("index out of double byte token bounds %d", n)
+	}
+
+	return doubleByteTokens[n], nil
+}
+
+func IndexOfSingleToken(token string) int {
+	for i, t := range SingleByteTokens {
+		if t == token {
+			return i
+		}
+	}
+
+	return -1
+}
+
+const (
+	LIST_EMPTY   = 0
+	STREAM_END   = 2
+	DICTIONARY_0 = 236
+	DICTIONARY_1 = 237
+	DICTIONARY_2 = 238
+	DICTIONARY_3 = 239
+	LIST_8       = 248
+	LIST_16      = 249
+	JID_PAIR     = 250
+	HEX_8        = 251
+	BINARY_8     = 252
+	BINARY_20    = 253
+	BINARY_32    = 254
+	NIBBLE_8     = 255
+)
+
+const (
+	PACKED_MAX      = 254
+	SINGLE_BYTE_MAX = 256
+)