Release v1.14.4

Revert "Fix typo"

This reverts commit dffd67eb31.

Revert "Handle quit message relay better on gateways with one channel on the irc bridge #722"

This reverts commit 240559581a.

Revert "Support quits from irc correctly. Fixes #722 (#724)"

This reverts commit d76a04bd0a.

.golangci.yaml

@@ -7,7 +7,7 @@ run:
   # concurrency: 4
 
   # timeout for analysis, e.g. 30s, 5m, default is 1m
-  deadline: 1m
+  deadline: 2m
 
   # exit code when at least one issue was found, default is 1
   issues-exit-code: 1
@@ -105,10 +105,6 @@ linters-settings:
     # with golangci-lint call it on a directory with the changed file.
     check-exported: false
   unparam:
-    # call graph construction algorithm (cha, rta). In general, use cha for libraries,
-    # and rta for programs with main packages. Default is cha.
-    algo: rta
-
     # Inspect exported functions, default is false. Set to true if no external program/library imports your code.
     # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors:
     # if it's called for subdir of a project it can't find external interfaces. All text editor integrations
@@ -158,7 +154,6 @@ linters-settings:
       - regexpMust
       - singleCaseSwitch
       - sloppyLen
-      - sloppyReassign
       - switchTrue
       - typeSwitchVar
       - typeUnparen

.goreleaser.yml

@@ -0,0 +1,34 @@
+release:
+  prerelease: auto
+  name_template: "{{.ProjectName}} v{{.Version}}"
+
+builds:
+- env:
+    - CGO_ENABLED=0
+  goos:
+    - freebsd
+    - windows
+    - darwin
+    - linux
+    - dragonfly
+    - netbsd
+    - openbsd
+  goarch:
+    - amd64
+    - arm
+    - arm64
+    - 386
+  ldflags:
+    - -s -w -X main.githash={{.ShortCommit}}
+
+archive:
+  name_template: "{{ .Binary }}-{{ .Version }}-{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
+  format: binary
+  files:
+    - none*
+  replacements:
+    386: 32bit
+    amd64: 64bit
+
+checksum:
+  name_template: 'checksums.txt'

.travis.yml

@@ -1,57 +1,55 @@
 language: go
-go:
-    - 1.11.x
 go_import_path: github.com/42wim/matterbridge
 
-# we have everything vendored
+# We have everything vendored so this helps TravisCI not run `go get ...`.
 install: true
 
 git:
   depth: 200
 
-env:
-  global:
-    - GOOS=linux GOARCH=amd64
-    - GOLANGCI_VERSION="v1.14.0"
-
-matrix:
-  # It's ok if our code fails on unstable development versions of Go.
-  allow_failures:
-    - go: tip
-  # Don't wait for tip tests to finish. Mark the test run green if the
-  # tests pass on the stable versions of Go.
-  fast_finish: true
-
 notifications:
-      email: false
+  email: false
 
-before_script:
-  # Get version info from tags.
-  - MY_VERSION="$(git describe --tags)"
-  # Retrieve the golangci-lint linter binary.
-  - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b ${GOPATH}/bin ${GOLANGCI_VERSION}
-  # Retrieve and prepare CodeClimate's test coverage reporter.
-  - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
-  - chmod +x ./cc-test-reporter
-  - ./cc-test-reporter before-build
+branches:
+  only:
+  - master
 
-script:
-  # Run the linter.
-  - golangci-lint run
-  # Run all the tests with the race detector and generate coverage.
-  - go test -v -race -coverprofile c.out ./...
-  # Run the build script to generate the necessary binaries and images.
-  - /bin/bash ci/bintray.sh
+jobs:
+  include:
+  - stage: lint
+    # Run linting in one Go environment only.
+    script: ./ci/lint.sh
+    go: 1.12.x
+    env:
+    - GO111MODULE=on
+    - GOLANGCI_VERSION="v1.16.0"
+  - stage: test
+    # Run tests in a combination of Go environments.
+    script: ./ci/test.sh
+    go: 1.11.x
+    env:
+    - GO111MODULE=off
+  - script: ./ci/test.sh
+    go: 1.11.x
+    env:
+    - GO111MODULE=on
+  - script: ./ci/test.sh
+    go: 1.12.x
+    env:
+    - GO111MODULE=on
+    - REPORT_COVERAGE=1
+    - BINDEPLOY=1
 
-after_script:
-  # Upload test coverage to CodeClimate.
-  - ./cc-test-reporter after-build --exit-code ${TRAVIS_TEST_RESULT}
+before_deploy: /bin/bash ci/bintray.sh
 
 deploy:
+  on:
+    all_branches: true
+    condition: $BINDEPLOY = 1
   provider: bintray
   edge:
     branch: v1.8.47
   file: ci/deploy.json
   user: 42wim
   key:
-     secure: "CeXXe6JOmt7HYR81MdWLua0ltQHhDdkIeRGBFbgd7hkb1wi8eF9DgpAcQrTso8NIlHNZmSAP46uhFgsRvkuezzX0ygalZ7DCJyAyn3sAMEh+UQSHV1WGThRehTtidqRGjetzsIGSwdrJOWil+XTfbO1Z8DGzfakhSuAZka8CM4BAoe3YeP9rYK8h+84x0GHfczvsLtXZ3mWLvQuwe4pK6+ItBCUg0ae7O7ZUpWHy0xQQkkWztY/6RAzXfaG7DuGjIw+20fhx3WOXRNpHCtZ6Bc3qERCpk0s1HhlQWlrN9wDaFTBWYwlvSnNgvxxMbNXJ6RrRJ0l0bA7FUswYwyroxhzrGLdzWDg8dHaQkypocngdalfhpsnoO9j3ApJhomUFJ3UoEq5nOGRUrKn8MPi+dP0zE4kNQ3e4VNa1ufNrvfpWolMg3xh8OXuhQdD5wIM5zFAbRJLqWSCVAjPq4DDPecmvXBOlIial7oa312lN5qnBnUjvAcxszZ+FUyDHT1Grxzna4tMwxY9obPzZUzm7359AOCCwIQFVB8GLqD2nwIstcXS0zGRz+fhviPipHuBa02q5bGUZwmkvrSNab0s8Jo7pCrel2Rz3nWPKaiCfq2WjbW1CLheSMkOQrjsdUd1hhbqNWFPUjJPInTc77NAKCfm5runv5uyowRLh4NNd0sI="
+    secure: "CeXXe6JOmt7HYR81MdWLua0ltQHhDdkIeRGBFbgd7hkb1wi8eF9DgpAcQrTso8NIlHNZmSAP46uhFgsRvkuezzX0ygalZ7DCJyAyn3sAMEh+UQSHV1WGThRehTtidqRGjetzsIGSwdrJOWil+XTfbO1Z8DGzfakhSuAZka8CM4BAoe3YeP9rYK8h+84x0GHfczvsLtXZ3mWLvQuwe4pK6+ItBCUg0ae7O7ZUpWHy0xQQkkWztY/6RAzXfaG7DuGjIw+20fhx3WOXRNpHCtZ6Bc3qERCpk0s1HhlQWlrN9wDaFTBWYwlvSnNgvxxMbNXJ6RrRJ0l0bA7FUswYwyroxhzrGLdzWDg8dHaQkypocngdalfhpsnoO9j3ApJhomUFJ3UoEq5nOGRUrKn8MPi+dP0zE4kNQ3e4VNa1ufNrvfpWolMg3xh8OXuhQdD5wIM5zFAbRJLqWSCVAjPq4DDPecmvXBOlIial7oa312lN5qnBnUjvAcxszZ+FUyDHT1Grxzna4tMwxY9obPzZUzm7359AOCCwIQFVB8GLqD2nwIstcXS0zGRz+fhviPipHuBa02q5bGUZwmkvrSNab0s8Jo7pCrel2Rz3nWPKaiCfq2WjbW1CLheSMkOQrjsdUd1hhbqNWFPUjJPInTc77NAKCfm5runv5uyowRLh4NNd0sI="

README.md

@@ -35,6 +35,12 @@
 
 **Note:** Matter<em>most</em> isn't required to run matter<em>bridge</em>.</sup></div>
 
+<p>
+  <a href="https://www.digitalocean.com/">
+    <img src="https://opensource.nyc3.cdn.digitaloceanspaces.com/attribution/assets/PoweredByDO/DO_Powered_by_Badge_blue.svg" width="201px">
+  </a>
+</p>
+
 ### Table of Contents
  * [Features](https://github.com/42wim/matterbridge/wiki/Features)
    * [Natively supported](#natively-supported)
@@ -88,6 +94,7 @@
 * [Minecraft](https://github.com/elytra/MatterLink)
 * [Reddit](https://github.com/bonehurtingjuice/mattereddit)
 * [Facebook messenger](https://github.com/VictorNine/fbridge)
+* [Discourse](https://github.com/DeclanHoare/matterbabble)
 
 ### API
 The API is very basic at the moment.   
@@ -99,6 +106,7 @@ Used by the projects below. Feel free to make a PR to add your project to this l
 * [pyCord](https://github.com/NikkyAI/pyCord) (crossplatform chatbot)
 * [Mattereddit](https://github.com/bonehurtingjuice/mattereddit) (Reddit chat support)
 * [fbridge](https://github.com/VictorNine/fbridge) (Facebook messenger support)
+* [matterbabble](https://github.com/DeclanHoare/matterbabble) (Discourse support)
 
 ## Chat with us
 
@@ -121,7 +129,7 @@ See https://github.com/42wim/matterbridge/wiki
 
 ## Installing
 ### Binaries
-* Latest stable release [v1.13.1](https://github.com/42wim/matterbridge/releases/latest)
+* Latest stable release [v1.14.2](https://github.com/42wim/matterbridge/releases/latest)
 * Development releases (follows master) can be downloaded [here](https://dl.bintray.com/42wim/nightly/)
 
 ### Packages
@@ -237,10 +245,6 @@ See [changelog.md](https://github.com/42wim/matterbridge/blob/master/changelog.m
 
 See [FAQ](https://github.com/42wim/matterbridge/wiki/FAQ)
 
-Want to tip ?
-* eth: 0xb3f9b5387c66ad6be892bcb7bbc67862f3abc16f
-* btc: 1N7cKHj5SfqBHBzDJ6kad4BzeqUBBS2zhs
-
 ## Related projects
 * [FOSSRIT/infrastructure - roles/matterbridge](https://github.com/FOSSRIT/infrastructure/tree/master/roles/matterbridge) (Ansible role used to automate deployments of Matterbridge)
 * [matterbridge autoconfig](https://github.com/patcon/matterbridge-autoconfig)
@@ -251,6 +255,8 @@ Want to tip ?
 * [mattermost-plugin](https://github.com/matterbridge/mattermost-plugin) - Run matterbridge as a plugin in mattermost
 * [pyCord](https://github.com/NikkyAI/pyCord) (crossplatform chatbot)
 * [fbridge](https://github.com/VictorNine/fbridge) (Facebook messenger support)
+* [isla](https://github.com/alphachung/isla) (Bot for Discord-Telegram groups used alongside matterbridge)
+* [matterbabble](https://github.com/DeclanHoare/matterbabble) (Connect Discourse threads to Matterbridge)
 
 ## Articles
 * [matterbridge on kubernetes](https://medium.freecodecamp.org/using-kubernetes-to-deploy-a-chat-gateway-or-when-technology-works-like-its-supposed-to-a169a8cd69a3)
@@ -264,7 +270,13 @@ Want to tip ?
 * https://daniele.tech/2019/02/how-to-use-matterbridge-to-connect-2-different-slack-workspaces/
 
 ## Thanks
-[![Digitalocean](https://snag.gy/3LVifX.jpg)](https://www.digitalocean.com/) for sponsoring demo/testing droplets.
+
+<p>This project is supported by:</p>
+<p>
+  <a href="https://www.digitalocean.com/">
+    <img src="https://opensource.nyc3.cdn.digitaloceanspaces.com/attribution/assets/SVG/DO_Logo_horizontal_blue.svg" width="201px">
+  </a>
+</p>
 
 Matterbridge wouldn't exist without these libraries:
 * discord - https://github.com/bwmarrin/discordgo

bridge/config/config.go

@@ -93,6 +93,7 @@ type Protocol struct {
 	MediaDownloadSize      int    // all protocols
 	MediaServerDownload    string
 	MediaServerUpload      string
+	MediaConvertWebPToPNG  bool       // telegram
 	MessageDelay           int        // IRC, time in millisecond to wait between messages
 	MessageFormat          string     // telegram
 	MessageLength          int        // IRC, max length of a message allowed

bridge/discord/discord.go

@@ -246,7 +246,7 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 			b.Log.Debugf("Setting webhook channel to \"%s\"", msg.Channel)
 			_, err := b.c.WebhookEdit(wID, "", "", channelID)
 			if err != nil {
-				b.Log.Errorf("Could not set webhook channel: %v", err)
+				b.Log.Errorf("Could not set webhook channel: %s", err)
 				return "", err
 			}
 		}
@@ -278,7 +278,7 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
 			rmsg.Text = helper.ClipMessage(rmsg.Text, MessageLength)
 			if _, err := b.c.ChannelMessageSend(channelID, rmsg.Username+rmsg.Text); err != nil {
-				b.Log.Errorf("Could not send message %#v: %v", rmsg, err)
+				b.Log.Errorf("Could not send message %#v: %s", rmsg, err)
 			}
 		}
 		// check if we have files to upload (from slack, telegram or mattermost)
@@ -360,7 +360,7 @@ func (b *Bdiscord) handleUploadFile(msg *config.Message, channelID string) (stri
 		}
 		_, err = b.c.ChannelMessageSendComplex(channelID, &m)
 		if err != nil {
-			return "", fmt.Errorf("file upload failed: %#v", err)
+			return "", fmt.Errorf("file upload failed: %s", err)
 		}
 	}
 	return "", nil

bridge/discord/handlers.go

@@ -36,7 +36,7 @@ func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreat
 		return
 	}
 	// if using webhooks, do not relay if it's ours
-	if b.useWebhook() && m.Author.Bot && b.isWebhookID(m.Author.ID) {
+	if b.useWebhook() && m.Author.Bot { // && b.isWebhookID(m.Author.ID) {
 		return
 	}
 

bridge/discord/helpers.go

@@ -25,7 +25,7 @@ func (b *Bdiscord) getNick(user *discordgo.User) string {
 	// If we didn't find nick, search for it.
 	member, err := b.c.GuildMember(b.guildID, user.ID)
 	if err != nil {
-		b.Log.Warnf("Failed to fetch information for member %#v: %#v", user, err)
+		b.Log.Warnf("Failed to fetch information for member %#v: %s", user, err)
 		return user.Username
 	} else if member == nil {
 		b.Log.Warnf("Got no information for member %#v", user)

bridge/helper/helper.go

@@ -3,6 +3,7 @@ package helper
 import (
 	"bytes"
 	"fmt"
+	"image/png"
 	"io"
 	"net/http"
 	"regexp"
@@ -10,6 +11,8 @@ import (
 	"time"
 	"unicode/utf8"
 
+	"golang.org/x/image/webp"
+
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/sirupsen/logrus"
 	"gitlab.com/golang-commonmark/markdown"
@@ -175,5 +178,24 @@ func ClipMessage(text string, length int) string {
 
 func ParseMarkdown(input string) string {
 	md := markdown.New(markdown.XHTMLOutput(true), markdown.Breaks(true))
-	return (md.RenderToString([]byte(input)))
+	res := md.RenderToString([]byte(input))
+	res = strings.TrimPrefix(res, "<p>")
+	res = strings.TrimSuffix(res, "</p>\n")
+	return res
+}
+
+// ConvertWebPToPNG convert input data (which should be WebP format to PNG format)
+func ConvertWebPToPNG(data *[]byte) error {
+	r := bytes.NewReader(*data)
+	m, err := webp.Decode(r)
+	if err != nil {
+		return err
+	}
+	var output []byte
+	w := bytes.NewBuffer(output)
+	if err := png.Encode(w, m); err != nil {
+		return err
+	}
+	*data = w.Bytes()
+	return nil
 }

bridge/helper/helper_test.go

@@ -1,6 +1,8 @@
 package helper
 
 import (
+	"io/ioutil"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -103,3 +105,22 @@ func TestGetSubLines(t *testing.T) {
 		assert.Equalf(t, testcase.nonSplitOutput, nonSplitLines, "'%s' testcase should give expected lines without splitting.", testname)
 	}
 }
+
+func TestConvertWebPToPNG(t *testing.T) {
+	if os.Getenv("LOCAL_TEST") == "" {
+		t.Skip()
+	}
+	input, err := ioutil.ReadFile("test.webp")
+	if err != nil {
+		t.Fail()
+	}
+	d := &input
+	err = ConvertWebPToPNG(d)
+	if err != nil {
+		t.Fail()
+	}
+	err = ioutil.WriteFile("test.png", *d, 0644)
+	if err != nil {
+		t.Fail()
+	}
+}

bridge/irc/handlers.go

@@ -92,10 +92,6 @@ func (b *Birc) handleJoinPart(client *girc.Client, event girc.Event) {
 			return
 		}
 		b.Log.Debugf("<= Sending JOIN_LEAVE event from %s to gateway", b.Account)
-		// QUIT isn't channel bound, happens for all channels on the bridge
-		if event.Command == "QUIT" {
-			channel = ""
-		}
 		msg := config.Message{Username: "system", Text: event.Source.Name + " " + strings.ToLower(event.Command) + "s", Channel: channel, Account: b.Account, Event: config.EventJoinLeave}
 		b.Log.Debugf("<= Message is %#v", msg)
 		b.Remote <- msg
@@ -160,7 +156,10 @@ func (b *Birc) handleOtherAuth(client *girc.Client, event girc.Event) {
 	b.handleNickServ()
 	b.handleRunCommands()
 	// we are now fully connected
-	b.connected <- nil
+	// only send on first connection
+	if b.FirstConnection {
+		b.connected <- nil
+	}
 }
 
 func (b *Birc) handlePrivMsg(client *girc.Client, event girc.Event) {

bridge/irc/irc.go

@@ -137,6 +137,7 @@ func (b *Birc) Send(msg config.Message) (string, error) {
 	// we can be in between reconnects #385
 	if !b.i.IsConnected() {
 		b.Log.Error("Not connected to server, dropping message")
+		return "", nil
 	}
 
 	// Execute a command
@@ -231,7 +232,7 @@ func (b *Birc) getClient() (*girc.Client, error) {
 	// fix strict user handling of girc
 	user := b.GetString("Nick")
 	for !girc.IsValidUser(user) {
-		if len(user) == 1 {
+		if len(user) == 1 || len(user) == 0 {
 			user = "matterbridge"
 			break
 		}

bridge/mattermost/helpers.go

@@ -186,6 +186,12 @@ func (b *Bmattermost) skipMessage(message *matterclient.Message) bool {
 		return true
 	}
 
+	// Ignore non-post messages
+	if message.Post == nil {
+		b.Log.Debugf("ignoring nil message.Post: %#v", message)
+		return true
+	}
+
 	// Ignore messages sent from matterbridge
 	if message.Post.Props != nil {
 		if _, ok := message.Post.Props["matterbridge_"+b.uuid].(bool); ok {

bridge/mattermost/mattermost.go

@@ -121,6 +121,12 @@ func (b *Bmattermost) Send(msg config.Message) (string, error) {
 		return msg.ID, b.mc.DeleteMessage(msg.ID)
 	}
 
+	// Handle prefix hint for unthreaded messages.
+	if msg.ParentID == "msg-parent-not-found" {
+		msg.ParentID = ""
+		msg.Text = fmt.Sprintf("[thread]: %s", msg.Text)
+	}
+
 	// Upload a file if it exists
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {

bridge/rocketchat/handlers.go

@@ -40,6 +40,11 @@ func (b *Brocketchat) handleRocketHook(messages chan *config.Message) {
 
 func (b *Brocketchat) handleRocketClient(messages chan *config.Message) {
 	for message := range b.messageChan {
+		// skip messages with same ID, apparently messages get duplicated for an unknown reason
+		if _, ok := b.cache.Get(message.ID); ok {
+			continue
+		}
+		b.cache.Add(message.ID, true)
 		b.Log.Debugf("message %#v", message)
 		m := message
 		if b.skipMessage(&m) {

bridge/rocketchat/helpers.go

@@ -95,7 +95,7 @@ func (b *Brocketchat) getChannelID(name string) string {
 	b.RLock()
 	defer b.RUnlock()
 	for k, v := range b.channelMap {
-		if v == name {
+		if v == name || v == "#"+name {
 			return k
 		}
 	}

bridge/rocketchat/rocketchat.go

@@ -2,6 +2,7 @@ package brocketchat
 
 import (
 	"errors"
+	"strings"
 	"sync"
 
 	"github.com/42wim/matterbridge/bridge"
@@ -9,16 +10,18 @@ import (
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/42wim/matterbridge/hook/rockethook"
 	"github.com/42wim/matterbridge/matterhook"
+	lru "github.com/hashicorp/golang-lru"
 	"github.com/matterbridge/Rocket.Chat.Go.SDK/models"
 	"github.com/matterbridge/Rocket.Chat.Go.SDK/realtime"
 	"github.com/matterbridge/Rocket.Chat.Go.SDK/rest"
 )
 
 type Brocketchat struct {
-	mh *matterhook.Client
-	rh *rockethook.Client
-	c  *realtime.Client
-	r  *rest.Client
+	mh    *matterhook.Client
+	rh    *rockethook.Client
+	c     *realtime.Client
+	r     *rest.Client
+	cache *lru.Cache
 	*bridge.Config
 	messageChan chan models.Message
 	channelMap  map[string]string
@@ -27,9 +30,16 @@ type Brocketchat struct {
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
-	b := &Brocketchat{Config: cfg}
-	b.messageChan = make(chan models.Message)
-	b.channelMap = make(map[string]string)
+	newCache, err := lru.New(100)
+	if err != nil {
+		cfg.Log.Fatalf("Could not create LRU cache for rocketchat bridge: %v", err)
+	}
+	b := &Brocketchat{
+		Config:      cfg,
+		messageChan: make(chan models.Message),
+		channelMap:  make(map[string]string),
+		cache:       newCache,
+	}
 	b.Log.Debugf("enabling rocketchat")
 	return b
 }
@@ -76,14 +86,14 @@ func (b *Brocketchat) JoinChannel(channel config.ChannelInfo) error {
 	if b.c == nil {
 		return nil
 	}
-	id, err := b.c.GetChannelId(channel.Name)
+	id, err := b.c.GetChannelId(strings.TrimPrefix(channel.Name, "#"))
 	if err != nil {
 		return err
 	}
 	b.Lock()
 	b.channelMap[id] = channel.Name
 	b.Unlock()
-	mychannel := &models.Channel{ID: id, Name: channel.Name}
+	mychannel := &models.Channel{ID: id, Name: strings.TrimPrefix(channel.Name, "#")}
 	if err := b.c.JoinChannel(id); err != nil {
 		return err
 	}
@@ -94,6 +104,8 @@ func (b *Brocketchat) JoinChannel(channel config.ChannelInfo) error {
 }
 
 func (b *Brocketchat) Send(msg config.Message) (string, error) {
+	// strip the # if people has set this
+	msg.Channel = strings.TrimPrefix(msg.Channel, "#")
 	channel := &models.Channel{ID: b.getChannelID(msg.Channel), Name: msg.Channel}
 
 	// Delete message
@@ -122,6 +134,8 @@ func (b *Brocketchat) Send(msg config.Message) (string, error) {
 	// Upload a file if it exists
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
+			// strip the # if people has set this
+			rmsg.Channel = strings.TrimPrefix(rmsg.Channel, "#")
 			smsg := &models.Message{
 				RoomID: b.getChannelID(rmsg.Channel),
 				Msg:    rmsg.Username + rmsg.Text,

bridge/slack/handlers.go

@@ -22,20 +22,20 @@ func (b *Bslack) handleSlack() {
 	time.Sleep(time.Second)
 	b.Log.Debug("Start listening for Slack messages")
 	for message := range messages {
-		if message.Event != config.EventUserTyping {
+		// don't do any action on deleted/typing messages
+		if message.Event != config.EventUserTyping && message.Event != config.EventMsgDelete {
 			b.Log.Debugf("<= Sending message from %s on %s to gateway", message.Username, b.Account)
+			// cleanup the message
+			message.Text = b.replaceMention(message.Text)
+			message.Text = b.replaceVariable(message.Text)
+			message.Text = b.replaceChannel(message.Text)
+			message.Text = b.replaceURL(message.Text)
+			message.Text = html.UnescapeString(message.Text)
+
+			// Add the avatar
+			message.Avatar = b.users.getAvatar(message.UserID)
 		}
 
-		// cleanup the message
-		message.Text = b.replaceMention(message.Text)
-		message.Text = b.replaceVariable(message.Text)
-		message.Text = b.replaceChannel(message.Text)
-		message.Text = b.replaceURL(message.Text)
-		message.Text = html.UnescapeString(message.Text)
-
-		// Add the avatar
-		message.Avatar = b.getAvatar(message.UserID)
-
 		b.Log.Debugf("<= Message is %#v", message)
 		b.Remote <- *message
 	}
@@ -75,20 +75,17 @@ func (b *Bslack) handleSlackClient(messages chan *config.Message) {
 			// When we join a channel we update the full list of users as
 			// well as the information for the channel that we joined as this
 			// should now tell that we are a member of it.
-			b.channelsMutex.Lock()
-			b.channelsByID[ev.Channel.ID] = &ev.Channel
-			b.channelsByName[ev.Channel.Name] = &ev.Channel
-			b.channelsMutex.Unlock()
+			b.channels.registerChannel(ev.Channel)
 		case *slack.ConnectedEvent:
 			b.si = ev.Info
-			b.populateChannels(true)
-			b.populateUsers(true)
+			b.channels.populateChannels(true)
+			b.users.populateUsers(true)
 		case *slack.InvalidAuthEvent:
 			b.Log.Fatalf("Invalid Token %#v", ev)
 		case *slack.ConnectionErrorEvent:
 			b.Log.Errorf("Connection failed %#v %#v", ev.Error(), ev.ErrorObj)
 		case *slack.MemberJoinedChannelEvent:
-			b.populateUser(ev.User)
+			b.users.populateUser(ev.User)
 		case *slack.LatencyReport:
 			continue
 		default:
@@ -133,12 +130,18 @@ func (b *Bslack) skipMessageEvent(ev *slack.MessageEvent) bool {
 		return true
 	}
 
-	// It seems ev.SubMessage.Edited == nil when slack unfurls.
-	// Do not forward these messages. See Github issue #266.
-	if ev.SubMessage != nil &&
-		ev.SubMessage.ThreadTimestamp != ev.SubMessage.Timestamp &&
-		ev.SubMessage.Edited == nil {
-		return true
+	if ev.SubMessage != nil {
+		// It seems ev.SubMessage.Edited == nil when slack unfurls.
+		// Do not forward these messages. See Github issue #266.
+		if ev.SubMessage.ThreadTimestamp != ev.SubMessage.Timestamp &&
+			ev.SubMessage.Edited == nil {
+			return true
+		}
+		// see hidden subtypes at https://api.slack.com/events/message
+		// these messages are sent when we add a message to a thread #709
+		if ev.SubType == "message_replied" && ev.Hidden {
+			return true
+		}
 	}
 
 	if len(ev.Files) > 0 {
@@ -210,7 +213,7 @@ func (b *Bslack) handleStatusEvent(ev *slack.MessageEvent, rmsg *config.Message)
 		rmsg.Username = sSystemUser
 		rmsg.Event = config.EventJoinLeave
 	case sChannelTopic, sChannelPurpose:
-		b.populateChannels(false)
+		b.channels.populateChannels(false)
 		rmsg.Event = config.EventTopicChange
 	case sMessageChanged:
 		rmsg.Text = ev.SubMessage.Text
@@ -266,7 +269,7 @@ func (b *Bslack) handleAttachments(ev *slack.MessageEvent, rmsg *config.Message)
 }
 
 func (b *Bslack) handleTypingEvent(ev *slack.UserTypingEvent) (*config.Message, error) {
-	channelInfo, err := b.getChannelByID(ev.Channel)
+	channelInfo, err := b.channels.getChannelByID(ev.Channel)
 	if err != nil {
 		return nil, err
 	}
@@ -316,36 +319,7 @@ func (b *Bslack) handleGetChannelMembers(rmsg *config.Message) bool {
 		return false
 	}
 
-	cMembers := config.ChannelMembers{}
-
-	b.channelMembersMutex.RLock()
-
-	for channelID, members := range b.channelMembers {
-		for _, member := range members {
-			channelName := ""
-			userName := ""
-			userNick := ""
-			user := b.getUser(member)
-			if user != nil {
-				userName = user.Name
-				userNick = user.Profile.DisplayName
-			}
-			channel, _ := b.getChannelByID(channelID)
-			if channel != nil {
-				channelName = channel.Name
-			}
-			cMember := config.ChannelMember{
-				Username:    userName,
-				Nick:        userNick,
-				UserID:      member,
-				ChannelID:   channelID,
-				ChannelName: channelName,
-			}
-			cMembers = append(cMembers, cMember)
-		}
-	}
-
-	b.channelMembersMutex.RUnlock()
+	cMembers := b.channels.getChannelMembers(b.users)
 
 	extra := make(map[string][]interface{})
 	extra[config.EventGetChannelMembers] = append(extra[config.EventGetChannelMembers], cMembers)

bridge/slack/helpers.go

@@ -1,7 +1,6 @@
 package bslack
 
 import (
-	"context"
 	"fmt"
 	"regexp"
 	"strings"
@@ -9,225 +8,14 @@ import (
 
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/nlopes/slack"
+	"github.com/sirupsen/logrus"
 )
 
-func (b *Bslack) getUser(id string) *slack.User {
-	b.usersMutex.RLock()
-	user, ok := b.users[id]
-	b.usersMutex.RUnlock()
-	if ok {
-		return user
-	}
-	b.populateUser(id)
-	b.usersMutex.RLock()
-	defer b.usersMutex.RUnlock()
-
-	return b.users[id]
-}
-
-func (b *Bslack) getUsername(id string) string {
-	if user := b.getUser(id); user != nil {
-		if user.Profile.DisplayName != "" {
-			return user.Profile.DisplayName
-		}
-		return user.Name
-	}
-	b.Log.Warnf("Could not find user with ID '%s'", id)
-	return ""
-}
-
-func (b *Bslack) getAvatar(id string) string {
-	if user := b.getUser(id); user != nil {
-		return user.Profile.Image48
-	}
-	return ""
-}
-
-func (b *Bslack) getChannel(channel string) (*slack.Channel, error) {
-	if strings.HasPrefix(channel, "ID:") {
-		return b.getChannelByID(strings.TrimPrefix(channel, "ID:"))
-	}
-	return b.getChannelByName(channel)
-}
-
-func (b *Bslack) getChannelByName(name string) (*slack.Channel, error) {
-	return b.getChannelBy(name, b.channelsByName)
-}
-
-func (b *Bslack) getChannelByID(ID string) (*slack.Channel, error) {
-	return b.getChannelBy(ID, b.channelsByID)
-}
-
-func (b *Bslack) getChannelBy(lookupKey string, lookupMap map[string]*slack.Channel) (*slack.Channel, error) {
-	b.channelsMutex.RLock()
-	defer b.channelsMutex.RUnlock()
-
-	if channel, ok := lookupMap[lookupKey]; ok {
-		return channel, nil
-	}
-	return nil, fmt.Errorf("%s: channel %s not found", b.Account, lookupKey)
-}
-
-const minimumRefreshInterval = 10 * time.Second
-
-func (b *Bslack) populateUser(userID string) {
-	b.usersMutex.RLock()
-	_, exists := b.users[userID]
-	b.usersMutex.RUnlock()
-	if exists {
-		// already in cache
-		return
-	}
-
-	user, err := b.sc.GetUserInfo(userID)
-	if err != nil {
-		b.Log.Debugf("GetUserInfo failed for %v: %v", userID, err)
-		return
-	}
-
-	b.usersMutex.Lock()
-	b.users[userID] = user
-	b.usersMutex.Unlock()
-}
-
-func (b *Bslack) populateUsers(wait bool) {
-	b.refreshMutex.Lock()
-	if !wait && (time.Now().Before(b.earliestUserRefresh) || b.refreshInProgress) {
-		b.Log.Debugf("Not refreshing user list as it was done less than %v ago.",
-			minimumRefreshInterval)
-		b.refreshMutex.Unlock()
-
-		return
-	}
-	for b.refreshInProgress {
-		b.refreshMutex.Unlock()
-		time.Sleep(time.Second)
-		b.refreshMutex.Lock()
-	}
-	b.refreshInProgress = true
-	b.refreshMutex.Unlock()
-
-	newUsers := map[string]*slack.User{}
-	pagination := b.sc.GetUsersPaginated(slack.GetUsersOptionLimit(200))
-	count := 0
-	for {
-		var err error
-		pagination, err = pagination.Next(context.Background())
-		time.Sleep(time.Second)
-		if err != nil {
-			if pagination.Done(err) {
-				break
-			}
-
-			if err = b.handleRateLimit(err); err != nil {
-				b.Log.Errorf("Could not retrieve users: %#v", err)
-				return
-			}
-			continue
-		}
-
-		for i := range pagination.Users {
-			newUsers[pagination.Users[i].ID] = &pagination.Users[i]
-		}
-		b.Log.Debugf("getting %d users", len(pagination.Users))
-		count++
-		// more > 2000 users, slack will complain and ratelimit. break
-		if count > 10 {
-			b.Log.Info("Large slack detected > 2000 users, skipping loading complete userlist.")
-			break
-		}
-	}
-
-	b.usersMutex.Lock()
-	defer b.usersMutex.Unlock()
-	b.users = newUsers
-
-	b.refreshMutex.Lock()
-	defer b.refreshMutex.Unlock()
-	b.earliestUserRefresh = time.Now().Add(minimumRefreshInterval)
-	b.refreshInProgress = false
-}
-
-func (b *Bslack) populateChannels(wait bool) {
-	b.refreshMutex.Lock()
-	if !wait && (time.Now().Before(b.earliestChannelRefresh) || b.refreshInProgress) {
-		b.Log.Debugf("Not refreshing channel list as it was done less than %v seconds ago.",
-			minimumRefreshInterval)
-		b.refreshMutex.Unlock()
-		return
-	}
-	for b.refreshInProgress {
-		b.refreshMutex.Unlock()
-		time.Sleep(time.Second)
-		b.refreshMutex.Lock()
-	}
-	b.refreshInProgress = true
-	b.refreshMutex.Unlock()
-
-	newChannelsByID := map[string]*slack.Channel{}
-	newChannelsByName := map[string]*slack.Channel{}
-	newChannelMembers := make(map[string][]string)
-
-	// We only retrieve public and private channels, not IMs
-	// and MPIMs as those do not have a channel name.
-	queryParams := &slack.GetConversationsParameters{
-		ExcludeArchived: "true",
-		Types:           []string{"public_channel,private_channel"},
-	}
-	for {
-		channels, nextCursor, err := b.sc.GetConversations(queryParams)
-		if err != nil {
-			if err = b.handleRateLimit(err); err != nil {
-				b.Log.Errorf("Could not retrieve channels: %#v", err)
-				return
-			}
-			continue
-		}
-
-		for i := range channels {
-			newChannelsByID[channels[i].ID] = &channels[i]
-			newChannelsByName[channels[i].Name] = &channels[i]
-			// also find all the members in every channel
-			// comment for now, issues on big slacks
-			/*
-				members, err := b.getUsersInConversation(channels[i].ID)
-				if err != nil {
-					if err = b.handleRateLimit(err); err != nil {
-						b.Log.Errorf("Could not retrieve channel members: %#v", err)
-						return
-					}
-					continue
-				}
-				newChannelMembers[channels[i].ID] = members
-			*/
-		}
-
-		if nextCursor == "" {
-			break
-		}
-		queryParams.Cursor = nextCursor
-	}
-
-	b.channelsMutex.Lock()
-	defer b.channelsMutex.Unlock()
-	b.channelsByID = newChannelsByID
-	b.channelsByName = newChannelsByName
-
-	b.channelMembersMutex.Lock()
-	defer b.channelMembersMutex.Unlock()
-	b.channelMembers = newChannelMembers
-
-	b.refreshMutex.Lock()
-	defer b.refreshMutex.Unlock()
-	b.earliestChannelRefresh = time.Now().Add(minimumRefreshInterval)
-	b.refreshInProgress = false
-}
-
 // populateReceivedMessage shapes the initial Matterbridge message that we will forward to the
 // router before we apply message-dependent modifications.
 func (b *Bslack) populateReceivedMessage(ev *slack.MessageEvent) (*config.Message, error) {
 	// Use our own func because rtm.GetChannelInfo doesn't work for private channels.
-	channel, err := b.getChannelByID(ev.Channel)
+	channel, err := b.channels.getChannelByID(ev.Channel)
 	if err != nil {
 		return nil, err
 	}
@@ -289,7 +77,7 @@ func (b *Bslack) populateMessageWithUserInfo(ev *slack.MessageEvent, rmsg *confi
 		return nil
 	}
 
-	user := b.getUser(userID)
+	user := b.users.getUser(userID)
 	if user == nil {
 		return fmt.Errorf("could not find information for user with id %s", ev.User)
 	}
@@ -315,7 +103,7 @@ func (b *Bslack) populateMessageWithBotInfo(ev *slack.MessageEvent, rmsg *config
 			break
 		}
 
-		if err = b.handleRateLimit(err); err != nil {
+		if err = handleRateLimit(b.Log, err); err != nil {
 			b.Log.Errorf("Could not retrieve bot information: %#v", err)
 			return err
 		}
@@ -360,7 +148,7 @@ func (b *Bslack) extractTopicOrPurpose(text string) (string, string) {
 func (b *Bslack) replaceMention(text string) string {
 	replaceFunc := func(match string) string {
 		userID := strings.Trim(match, "@<>")
-		if username := b.getUsername(userID); userID != "" {
+		if username := b.users.getUsername(userID); userID != "" {
 			return "@" + username
 		}
 		return match
@@ -404,16 +192,6 @@ func (b *Bslack) replaceCodeFence(text string) string {
 	return codeFenceRE.ReplaceAllString(text, "```")
 }
 
-func (b *Bslack) handleRateLimit(err error) error {
-	rateLimit, ok := err.(*slack.RateLimitedError)
-	if !ok {
-		return err
-	}
-	b.Log.Infof("Rate-limited by Slack. Sleeping for %v", rateLimit.RetryAfter)
-	time.Sleep(rateLimit.RetryAfter)
-	return nil
-}
-
 // getUsersInConversation returns an array of userIDs that are members of channelID
 func (b *Bslack) getUsersInConversation(channelID string) ([]string, error) {
 	channelMembers := []string{}
@@ -424,7 +202,7 @@ func (b *Bslack) getUsersInConversation(channelID string) ([]string, error) {
 
 		members, nextCursor, err := b.sc.GetUsersInConversation(queryParams)
 		if err != nil {
-			if err = b.handleRateLimit(err); err != nil {
+			if err = handleRateLimit(b.Log, err); err != nil {
 				return channelMembers, fmt.Errorf("Could not retrieve users in channels: %#v", err)
 			}
 			continue
@@ -439,3 +217,13 @@ func (b *Bslack) getUsersInConversation(channelID string) ([]string, error) {
 	}
 	return channelMembers, nil
 }
+
+func handleRateLimit(log *logrus.Entry, err error) error {
+	rateLimit, ok := err.(*slack.RateLimitedError)
+	if !ok {
+		return err
+	}
+	log.Infof("Rate-limited by Slack. Sleeping for %v", rateLimit.RetryAfter)
+	time.Sleep(rateLimit.RetryAfter)
+	return nil
+}

bridge/slack/legacy.go

@@ -55,14 +55,18 @@ func (b *BLegacy) Connect() error {
 		})
 		if b.GetString(tokenConfig) != "" {
 			b.Log.Info("Connecting using token (receiving)")
-			b.sc = slack.New(b.GetString(tokenConfig))
+			b.sc = slack.New(b.GetString(tokenConfig), slack.OptionDebug(b.GetBool("debug")))
+			b.channels = newChannelManager(b.Log, b.sc)
+			b.users = newUserManager(b.Log, b.sc)
 			b.rtm = b.sc.NewRTM()
 			go b.rtm.ManageConnection()
 			go b.handleSlack()
 		}
 	} else if b.GetString(tokenConfig) != "" {
 		b.Log.Info("Connecting using token (sending and receiving)")
-		b.sc = slack.New(b.GetString(tokenConfig))
+		b.sc = slack.New(b.GetString(tokenConfig), slack.OptionDebug(b.GetBool("debug")))
+		b.channels = newChannelManager(b.Log, b.sc)
+		b.users = newUserManager(b.Log, b.sc)
 		b.rtm = b.sc.NewRTM()
 		go b.rtm.ManageConnection()
 		go b.handleSlack()

bridge/slack/slack.go

@@ -30,20 +30,8 @@ type Bslack struct {
 	uuid         string
 	useChannelID bool
 
-	users      map[string]*slack.User
-	usersMutex sync.RWMutex
-
-	channelsByID   map[string]*slack.Channel
-	channelsByName map[string]*slack.Channel
-	channelsMutex  sync.RWMutex
-
-	channelMembers      map[string][]string
-	channelMembersMutex sync.RWMutex
-
-	refreshInProgress      bool
-	earliestChannelRefresh time.Time
-	earliestUserRefresh    time.Time
-	refreshMutex           sync.Mutex
+	channels *channels
+	users    *users
 }
 
 const (
@@ -94,14 +82,9 @@ func newBridge(cfg *bridge.Config) *Bslack {
 		cfg.Log.Fatalf("Could not create LRU cache for Slack bridge: %v", err)
 	}
 	b := &Bslack{
-		Config:                 cfg,
-		uuid:                   xid.New().String(),
-		cache:                  newCache,
-		users:                  map[string]*slack.User{},
-		channelsByID:           map[string]*slack.Channel{},
-		channelsByName:         map[string]*slack.Channel{},
-		earliestChannelRefresh: time.Now(),
-		earliestUserRefresh:    time.Now(),
+		Config: cfg,
+		uuid:   xid.New().String(),
+		cache:  newCache,
 	}
 	return b
 }
@@ -121,7 +104,12 @@ func (b *Bslack) Connect() error {
 	// If we have a token we use the Slack websocket-based RTM for both sending and receiving.
 	if token := b.GetString(tokenConfig); token != "" {
 		b.Log.Info("Connecting using token")
+
 		b.sc = slack.New(token, slack.OptionDebug(b.GetBool("Debug")))
+
+		b.channels = newChannelManager(b.Log, b.sc)
+		b.users = newUserManager(b.Log, b.sc)
+
 		b.rtm = b.sc.NewRTM()
 		go b.rtm.ManageConnection()
 		go b.handleSlack()
@@ -163,9 +151,9 @@ func (b *Bslack) JoinChannel(channel config.ChannelInfo) error {
 		return nil
 	}
 
-	b.populateChannels(false)
+	b.channels.populateChannels(false)
 
-	channelInfo, err := b.getChannel(channel.Name)
+	channelInfo, err := b.channels.getChannel(channel.Name)
 	if err != nil {
 		return fmt.Errorf("could not join channel: %#v", err)
 	}
@@ -275,7 +263,7 @@ func (b *Bslack) sendRTM(msg config.Message) (string, error) {
 		return "", nil
 	}
 
-	channelInfo, err := b.getChannel(msg.Channel)
+	channelInfo, err := b.channels.getChannel(msg.Channel)
 	if err != nil {
 		return "", fmt.Errorf("could not send message: %v", err)
 	}
@@ -351,7 +339,7 @@ func (b *Bslack) updateTopicOrPurpose(msg *config.Message, channelInfo *slack.Ch
 		if err == nil {
 			return nil
 		}
-		if err = b.handleRateLimit(err); err != nil {
+		if err = handleRateLimit(b.Log, err); err != nil {
 			return err
 		}
 	}
@@ -392,7 +380,7 @@ func (b *Bslack) deleteMessage(msg *config.Message, channelInfo *slack.Channel)
 			return true, nil
 		}
 
-		if err = b.handleRateLimit(err); err != nil {
+		if err = handleRateLimit(b.Log, err); err != nil {
 			b.Log.Errorf("Failed to delete user message from Slack: %#v", err)
 			return true, err
 		}
@@ -411,7 +399,7 @@ func (b *Bslack) editMessage(msg *config.Message, channelInfo *slack.Channel) (b
 			return true, nil
 		}
 
-		if err = b.handleRateLimit(err); err != nil {
+		if err = handleRateLimit(b.Log, err); err != nil {
 			b.Log.Errorf("Failed to edit user message on Slack: %#v", err)
 			return true, err
 		}
@@ -424,14 +412,18 @@ func (b *Bslack) postMessage(msg *config.Message, channelInfo *slack.Channel) (s
 		return "", nil
 	}
 	messageOptions := b.prepareMessageOptions(msg)
-	messageOptions = append(messageOptions, slack.MsgOptionText(msg.Text, false))
+	messageOptions = append(
+		messageOptions,
+		slack.MsgOptionText(msg.Text, false),
+		slack.MsgOptionEnableLinkUnfurl(),
+	)
 	for {
 		_, id, err := b.rtm.PostMessage(channelInfo.ID, messageOptions...)
 		if err == nil {
 			return id, nil
 		}
 
-		if err = b.handleRateLimit(err); err != nil {
+		if err = handleRateLimit(b.Log, err); err != nil {
 			b.Log.Errorf("Failed to sent user message to Slack: %#v", err)
 			return "", err
 		}

bridge/slack/users_channels.go

@@ -0,0 +1,336 @@
+package bslack
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/nlopes/slack"
+	"github.com/sirupsen/logrus"
+)
+
+const minimumRefreshInterval = 10 * time.Second
+
+type users struct {
+	log *logrus.Entry
+	sc  *slack.Client
+
+	users           map[string]*slack.User
+	usersMutex      sync.RWMutex
+	usersSyncPoints map[string]chan struct{}
+
+	refreshInProgress bool
+	earliestRefresh   time.Time
+	refreshMutex      sync.Mutex
+}
+
+func newUserManager(log *logrus.Entry, sc *slack.Client) *users {
+	return &users{
+		log:             log,
+		sc:              sc,
+		users:           make(map[string]*slack.User),
+		usersSyncPoints: make(map[string]chan struct{}),
+		earliestRefresh: time.Now(),
+	}
+}
+
+func (b *users) getUser(id string) *slack.User {
+	b.usersMutex.RLock()
+	user, ok := b.users[id]
+	b.usersMutex.RUnlock()
+	if ok {
+		return user
+	}
+	b.populateUser(id)
+	b.usersMutex.RLock()
+	defer b.usersMutex.RUnlock()
+
+	return b.users[id]
+}
+
+func (b *users) getUsername(id string) string {
+	if user := b.getUser(id); user != nil {
+		if user.Profile.DisplayName != "" {
+			return user.Profile.DisplayName
+		}
+		return user.Name
+	}
+	b.log.Warnf("Could not find user with ID '%s'", id)
+	return ""
+}
+
+func (b *users) getAvatar(id string) string {
+	if user := b.getUser(id); user != nil {
+		return user.Profile.Image48
+	}
+	return ""
+}
+
+func (b *users) populateUser(userID string) {
+	for {
+		b.usersMutex.Lock()
+		_, exists := b.users[userID]
+		if exists {
+			// already in cache
+			b.usersMutex.Unlock()
+			return
+		}
+
+		if syncPoint, ok := b.usersSyncPoints[userID]; ok {
+			// Another goroutine is already populating this user for us so wait on it to finish.
+			b.usersMutex.Unlock()
+			<-syncPoint
+			// We do not return and iterate again to check that the entry does indeed exist
+			// in case the previous query failed for some reason.
+		} else {
+			b.usersSyncPoints[userID] = make(chan struct{})
+			defer func() {
+				// Wake up any waiting goroutines and remove the synchronization point.
+				close(b.usersSyncPoints[userID])
+				delete(b.usersSyncPoints, userID)
+			}()
+			break
+		}
+	}
+
+	// Do not hold the lock while fetching information from Slack
+	// as this might take an unbounded amount of time.
+	b.usersMutex.Unlock()
+
+	user, err := b.sc.GetUserInfo(userID)
+	if err != nil {
+		b.log.Debugf("GetUserInfo failed for %v: %v", userID, err)
+		return
+	}
+
+	b.usersMutex.Lock()
+	defer b.usersMutex.Unlock()
+
+	// Register user information.
+	b.users[userID] = user
+}
+
+func (b *users) populateUsers(wait bool) {
+	b.refreshMutex.Lock()
+	if !wait && (time.Now().Before(b.earliestRefresh) || b.refreshInProgress) {
+		b.log.Debugf("Not refreshing user list as it was done less than %v ago.", minimumRefreshInterval)
+		b.refreshMutex.Unlock()
+
+		return
+	}
+	for b.refreshInProgress {
+		b.refreshMutex.Unlock()
+		time.Sleep(time.Second)
+		b.refreshMutex.Lock()
+	}
+	b.refreshInProgress = true
+	b.refreshMutex.Unlock()
+
+	newUsers := map[string]*slack.User{}
+	pagination := b.sc.GetUsersPaginated(slack.GetUsersOptionLimit(200))
+	count := 0
+	for {
+		var err error
+		pagination, err = pagination.Next(context.Background())
+		time.Sleep(time.Second)
+		if err != nil {
+			if pagination.Done(err) {
+				break
+			}
+
+			if err = handleRateLimit(b.log, err); err != nil {
+				b.log.Errorf("Could not retrieve users: %#v", err)
+				return
+			}
+			continue
+		}
+
+		for i := range pagination.Users {
+			newUsers[pagination.Users[i].ID] = &pagination.Users[i]
+		}
+		b.log.Debugf("getting %d users", len(pagination.Users))
+		count++
+		// more > 2000 users, slack will complain and ratelimit. break
+		if count > 10 {
+			b.log.Info("Large slack detected > 2000 users, skipping loading complete userlist.")
+			break
+		}
+	}
+
+	b.usersMutex.Lock()
+	defer b.usersMutex.Unlock()
+	b.users = newUsers
+
+	b.refreshMutex.Lock()
+	defer b.refreshMutex.Unlock()
+	b.earliestRefresh = time.Now().Add(minimumRefreshInterval)
+	b.refreshInProgress = false
+}
+
+type channels struct {
+	log *logrus.Entry
+	sc  *slack.Client
+
+	channelsByID   map[string]*slack.Channel
+	channelsByName map[string]*slack.Channel
+	channelsMutex  sync.RWMutex
+
+	channelMembers      map[string][]string
+	channelMembersMutex sync.RWMutex
+
+	refreshInProgress bool
+	earliestRefresh   time.Time
+	refreshMutex      sync.Mutex
+}
+
+func newChannelManager(log *logrus.Entry, sc *slack.Client) *channels {
+	return &channels{
+		log:             log,
+		sc:              sc,
+		channelsByID:    make(map[string]*slack.Channel),
+		channelsByName:  make(map[string]*slack.Channel),
+		earliestRefresh: time.Now(),
+	}
+}
+
+func (b *channels) getChannel(channel string) (*slack.Channel, error) {
+	if strings.HasPrefix(channel, "ID:") {
+		return b.getChannelByID(strings.TrimPrefix(channel, "ID:"))
+	}
+	return b.getChannelByName(channel)
+}
+
+func (b *channels) getChannelByName(name string) (*slack.Channel, error) {
+	return b.getChannelBy(name, b.channelsByName)
+}
+
+func (b *channels) getChannelByID(id string) (*slack.Channel, error) {
+	return b.getChannelBy(id, b.channelsByID)
+}
+
+func (b *channels) getChannelBy(lookupKey string, lookupMap map[string]*slack.Channel) (*slack.Channel, error) {
+	b.channelsMutex.RLock()
+	defer b.channelsMutex.RUnlock()
+
+	if channel, ok := lookupMap[lookupKey]; ok {
+		return channel, nil
+	}
+	return nil, fmt.Errorf("channel %s not found", lookupKey)
+}
+
+func (b *channels) getChannelMembers(users *users) config.ChannelMembers {
+	b.channelMembersMutex.RLock()
+	defer b.channelMembersMutex.RUnlock()
+
+	membersInfo := config.ChannelMembers{}
+	for channelID, members := range b.channelMembers {
+		for _, member := range members {
+			channelName := ""
+			userName := ""
+			userNick := ""
+			user := users.getUser(member)
+			if user != nil {
+				userName = user.Name
+				userNick = user.Profile.DisplayName
+			}
+			channel, _ := b.getChannelByID(channelID)
+			if channel != nil {
+				channelName = channel.Name
+			}
+			memberInfo := config.ChannelMember{
+				Username:    userName,
+				Nick:        userNick,
+				UserID:      member,
+				ChannelID:   channelID,
+				ChannelName: channelName,
+			}
+			membersInfo = append(membersInfo, memberInfo)
+		}
+	}
+	return membersInfo
+}
+
+func (b *channels) registerChannel(channel slack.Channel) {
+	b.channelsMutex.Lock()
+	defer b.channelsMutex.Unlock()
+
+	b.channelsByID[channel.ID] = &channel
+	b.channelsByName[channel.Name] = &channel
+}
+
+func (b *channels) populateChannels(wait bool) {
+	b.refreshMutex.Lock()
+	if !wait && (time.Now().Before(b.earliestRefresh) || b.refreshInProgress) {
+		b.log.Debugf("Not refreshing channel list as it was done less than %v seconds ago.", minimumRefreshInterval)
+		b.refreshMutex.Unlock()
+		return
+	}
+	for b.refreshInProgress {
+		b.refreshMutex.Unlock()
+		time.Sleep(time.Second)
+		b.refreshMutex.Lock()
+	}
+	b.refreshInProgress = true
+	b.refreshMutex.Unlock()
+
+	newChannelsByID := map[string]*slack.Channel{}
+	newChannelsByName := map[string]*slack.Channel{}
+	newChannelMembers := make(map[string][]string)
+
+	// We only retrieve public and private channels, not IMs
+	// and MPIMs as those do not have a channel name.
+	queryParams := &slack.GetConversationsParameters{
+		ExcludeArchived: "true",
+		Types:           []string{"public_channel,private_channel"},
+	}
+	for {
+		channels, nextCursor, err := b.sc.GetConversations(queryParams)
+		if err != nil {
+			if err = handleRateLimit(b.log, err); err != nil {
+				b.log.Errorf("Could not retrieve channels: %#v", err)
+				return
+			}
+			continue
+		}
+
+		for i := range channels {
+			newChannelsByID[channels[i].ID] = &channels[i]
+			newChannelsByName[channels[i].Name] = &channels[i]
+			// also find all the members in every channel
+			// comment for now, issues on big slacks
+			/*
+				members, err := b.getUsersInConversation(channels[i].ID)
+				if err != nil {
+					if err = b.handleRateLimit(err); err != nil {
+						b.Log.Errorf("Could not retrieve channel members: %#v", err)
+						return
+					}
+					continue
+				}
+				newChannelMembers[channels[i].ID] = members
+			*/
+		}
+
+		if nextCursor == "" {
+			break
+		}
+		queryParams.Cursor = nextCursor
+	}
+
+	b.channelsMutex.Lock()
+	defer b.channelsMutex.Unlock()
+	b.channelsByID = newChannelsByID
+	b.channelsByName = newChannelsByName
+
+	b.channelMembersMutex.Lock()
+	defer b.channelMembersMutex.Unlock()
+	b.channelMembers = newChannelMembers
+
+	b.refreshMutex.Lock()
+	defer b.refreshMutex.Unlock()
+	b.earliestRefresh = time.Now().Add(minimumRefreshInterval)
+	b.refreshInProgress = false
+}

bridge/telegram/handlers.go

@@ -125,6 +125,11 @@ func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
 		// handle groups
 		message = b.handleGroups(&rmsg, message, update)
 
+		if message == nil {
+			b.Log.Error("message is nil, this shouldn't happen.")
+			continue
+		}
+
 		// set the ID's from the channel or group message
 		rmsg.ID = strconv.Itoa(message.MessageID)
 		rmsg.Channel = strconv.FormatInt(message.Chat.ID, 10)
@@ -144,6 +149,9 @@ func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
 		// quote the previous message
 		b.handleQuoting(&rmsg, message)
 
+		// handle entities (adding URLs)
+		b.handleEntities(&rmsg, message)
+
 		if rmsg.Text != "" || len(rmsg.Extra) > 0 {
 			rmsg.Text = helper.RemoveEmptyNewLines(rmsg.Text)
 			// channels don't have (always?) user information. see #410
@@ -245,6 +253,15 @@ func (b *Btelegram) handleDownload(rmsg *config.Message, message *tgbotapi.Messa
 	if err != nil {
 		return err
 	}
+	if strings.HasSuffix(name, ".webp") && b.GetBool("MediaConvertWebPToPNG") {
+		b.Log.Debugf("WebP to PNG conversion enabled, converting %s", name)
+		err := helper.ConvertWebPToPNG(data)
+		if err != nil {
+			b.Log.Errorf("conversion failed: %s", err)
+		} else {
+			name = strings.Replace(name, ".webp", ".png", 1)
+		}
+	}
 	helper.HandleDownloadData(b.Log, rmsg, name, message.Caption, "", data, b.General)
 	return nil
 }
@@ -344,3 +361,22 @@ func (b *Btelegram) handleQuote(message, quoteNick, quoteMessage string) string
 	format = strings.Replace(format, "{QUOTEMESSAGE}", quoteMessage, -1)
 	return format
 }
+
+// handleEntities handles messageEntities
+func (b *Btelegram) handleEntities(rmsg *config.Message, message *tgbotapi.Message) {
+	if message.Entities == nil {
+		return
+	}
+	// for now only do URL replacements
+	for _, e := range *message.Entities {
+		if e.Type == "text_link" {
+			url, err := e.ParseURL()
+			if err != nil {
+				b.Log.Errorf("entity text_link url parse failed: %s", err)
+				continue
+			}
+			link := rmsg.Text[e.Offset : e.Offset+e.Length]
+			rmsg.Text = strings.Replace(rmsg.Text, link, url.String(), 1)
+		}
+	}
+}

bridge/telegram/html.go

@@ -3,7 +3,6 @@ package btelegram
 import (
 	"bytes"
 	"html"
-	"io"
 
 	"github.com/russross/blackfriday"
 )
@@ -33,7 +32,7 @@ func (options *customHTML) Header(out *bytes.Buffer, text func() bool, level int
 	options.Paragraph(out, text)
 }
 
-func (options *customHTML) HRule(out io.ByteWriter) {
+func (options *customHTML) HRule(out *bytes.Buffer) {
 	out.WriteByte('\n') //nolint:errcheck
 }
 
@@ -54,16 +53,13 @@ func (options *customHTML) ListItem(out *bytes.Buffer, text []byte, flags int) {
 }
 
 func makeHTML(input string) string {
-	extensions := blackfriday.NoIntraEmphasis |
-		blackfriday.FencedCode |
-		blackfriday.Autolink |
-		blackfriday.SpaceHeadings |
-		blackfriday.HeadingIDs |
-		blackfriday.BackslashLineBreak |
-		blackfriday.DefinitionLists
-
-	renderer := &customHTML{blackfriday.NewHTMLRenderer(blackfriday.HTMLRendererParameters{
-		Flags: blackfriday.UseXHTML | blackfriday.SkipImages,
-	})}
-	return string(blackfriday.Run([]byte(input), blackfriday.WithExtensions(extensions), blackfriday.WithRenderer(renderer)))
+	return string(blackfriday.Markdown([]byte(input),
+		&customHTML{blackfriday.HtmlRenderer(blackfriday.HTML_USE_XHTML|blackfriday.HTML_SKIP_IMAGES, "", "")},
+		blackfriday.EXTENSION_NO_INTRA_EMPHASIS|
+			blackfriday.EXTENSION_FENCED_CODE|
+			blackfriday.EXTENSION_AUTOLINK|
+			blackfriday.EXTENSION_SPACE_HEADERS|
+			blackfriday.EXTENSION_HEADER_IDS|
+			blackfriday.EXTENSION_BACKSLASH_LINE_BREAK|
+			blackfriday.EXTENSION_DEFINITION_LISTS))
 }

bridge/whatsapp/handlers.go

@@ -6,9 +6,9 @@ import (
 
 	"github.com/42wim/matterbridge/bridge/config"
 
-	"github.com/Rhymen/go-whatsapp"
+	"github.com/matterbridge/go-whatsapp"
 
-	whatsappExt "maunium.net/go/mautrix-whatsapp/whatsapp-ext"
+	whatsappExt "github.com/matterbridge/mautrix-whatsapp/whatsapp-ext"
 )
 
 /*

bridge/whatsapp/helpers.go

@@ -6,7 +6,7 @@ import (
 	"os"
 
 	qrcodeTerminal "github.com/Baozisoftware/qrcode-terminal-go"
-	"github.com/Rhymen/go-whatsapp"
+	"github.com/matterbridge/go-whatsapp"
 )
 
 func qrFromTerminal(invert bool) chan string {

bridge/whatsapp/whatsapp.go

@@ -12,9 +12,9 @@ import (
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 
-	"github.com/Rhymen/go-whatsapp"
+	"github.com/matterbridge/go-whatsapp"
 
-	whatsappExt "maunium.net/go/mautrix-whatsapp/whatsapp-ext"
+	whatsappExt "github.com/matterbridge/mautrix-whatsapp/whatsapp-ext"
 )
 
 const (

bridge/zulip/zulip.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"io/ioutil"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -18,12 +19,11 @@ type Bzulip struct {
 	bot     *gzb.Bot
 	streams map[int]string
 	*bridge.Config
-	channelToTopic map[string]string
 	sync.RWMutex
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
-	return &Bzulip{Config: cfg, streams: make(map[int]string), channelToTopic: make(map[string]string)}
+	return &Bzulip{Config: cfg, streams: make(map[int]string)}
 }
 
 func (b *Bzulip) Connect() error {
@@ -48,9 +48,6 @@ func (b *Bzulip) Disconnect() error {
 }
 
 func (b *Bzulip) JoinChannel(channel config.ChannelInfo) error {
-	b.Lock()
-	defer b.Unlock()
-	b.channelToTopic[channel.Name] = channel.Options.Topic
 	return nil
 }
 
@@ -116,11 +113,13 @@ func (b *Bzulip) handleQueue() error {
 		case gzb.BadEventQueueError:
 			b.Log.Info("got a bad event queue id error, reconnecting")
 			b.bot.Queues = nil
-			b.q, err = b.bot.RegisterAll()
-			if err != nil {
-				b.Log.Errorf("reconnecting failed: %s. Sleeping 10 seconds", err)
-				time.Sleep(time.Second * 10)
-				continue
+			for {
+				b.q, err = b.bot.RegisterAll()
+				if err != nil {
+					b.Log.Errorf("reconnecting failed: %s. Sleeping 10 seconds", err)
+					time.Sleep(time.Second * 10)
+				}
+				break
 			}
 		case gzb.HeartbeatError:
 			b.Log.Debug("heartbeat received.")
@@ -136,7 +135,14 @@ func (b *Bzulip) handleQueue() error {
 			if m.SenderEmail == b.GetString("login") {
 				continue
 			}
-			rmsg := config.Message{Username: m.SenderFullName, Text: m.Content, Channel: b.getChannel(m.StreamID), Account: b.Account, UserID: strconv.Itoa(m.SenderID), Avatar: m.AvatarURL}
+			rmsg := config.Message{
+				Username: m.SenderFullName,
+				Text:     m.Content,
+				Channel:  b.getChannel(m.StreamID) + "/topic:" + m.Subject,
+				Account:  b.Account,
+				UserID:   strconv.Itoa(m.SenderID),
+				Avatar:   m.AvatarURL,
+			}
 			b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
 			b.Log.Debugf("<= Message is %#v", rmsg)
 			b.Remote <- rmsg
@@ -147,12 +153,11 @@ func (b *Bzulip) handleQueue() error {
 }
 
 func (b *Bzulip) sendMessage(msg config.Message) (string, error) {
-	topic := "matterbridge"
-	if b.GetString("topic") != "" {
-		topic = b.GetString("topic")
-	}
-	if res := b.getTopic(msg.Channel); res != "" {
-		topic = res
+	topic := ""
+	if strings.Contains(msg.Channel, "/topic:") {
+		res := strings.Split(msg.Channel, "/topic:")
+		topic = res[1]
+		msg.Channel = res[0]
 	}
 	m := gzb.Message{
 		Stream:  msg.Channel,
@@ -200,9 +205,3 @@ func (b *Bzulip) handleUploadFile(msg *config.Message) (string, error) {
 	}
 	return "", nil
 }
-
-func (b *Bzulip) getTopic(channel string) string {
-	b.RLock()
-	defer b.RUnlock()
-	return b.channelToTopic[channel]
-}

changelog.md

@@ -1,4 +1,41 @@
-# v1.14.0-rc1
+# v1.14.4
+
+## Bugfix
+* mattermost: Add Id to EditMessage (mattermost). Fixes #802
+* mattermost: Fix panic on nil message.Post (mattermost). Fixes #804
+* mattermost: Handle unthreaded messages (mattermost). Fixes #803
+* mattermost: Use paging in initUser and UpdateUsers (mattermost)
+* slack: Add lacking clean-up in Slack synchronisation (#811)
+* slack: Disable user lookups on delete messages (slack) (#812)
+
+# v1.14.3
+
+## Bugfix
+* irc: Fix deadlock on reconnect (irc). Closes #757
+
+# v1.14.2
+
+## Bugfix
+* general: Update tengo vendor and load the stdlib. Fixes #789 (#792)
+* rocketchat: Look up #channel too (rocketchat). Fix #773 (#775)
+* slack: Ignore messagereplied and hidden messages (slack). Fixes #709 (#779)
+* telegram: Handle nil message (telegram). Fixes #777
+* irc: Use default nick if none specified (irc). Fixes #785
+* irc: Return when not connected and drop a message (irc). Fixes #786
+* irc: Revert fix for #722 (Support quits from irc correctly). Closes #781
+
+## Contributors
+This release couldn't exist without the following contributors:
+@42wim, @Helcaraxan, @dajohi
+
+# v1.14.1
+## Bugfix
+* slack: Fix crash double unlock (slack) (#771)
+
+# v1.14.0
+
+## Breaking
+* zulip: Need to specify /topic:mytopic for channel configuration (zulip). (#751)
 
 ## New features
 * whatsapp: new protocol added. Add initial WhatsApp support (#711) Thanks to @KrzysztofMadejski
@@ -10,18 +47,29 @@
 * rocketchat: add support for the rocketchat API. Sending to rocketchat now supports uploading of files, editing and deleting of messages.
 * discord: Support join/leaves from discord. Closes #654 (#721)
 * discord: Allow sending discriminator with Discord username (#726). See `UseDiscriminator` in matterbridge.toml.sample
-* zulip: Allow zulip bridge to specify topic per channel. Closes #701 (#723). See `Topic` in matterbridge.toml.sample
 * slack: Add extra debug option (slack). See `Debug` in the slack section in matterbridge.toml.sample
+* telegram: Add support for URL in messageEntities (telegram). Fixes #735 (#736)
+* telegram: Add MediaConvertWebPToPNG option (telegram). (#741). See `MediaConvertWebPToPNG` in matterbridge.toml.sample
+
+## Enhancements
+* general: Fail gracefully on incorrect human input. Fixes #739 (#740)
+* matrix: Detect html nicks in RemoteNickFormat (matrix). Fixes #696 (#719)
+* matrix: Send notices on join/parts (matrix). Fixes #712 (#716)
 
 ## Bugfix
+* general: Handle file upload/download only once for each message (#742)
 * zulip: Fix error handling on bad event queue id (zulip). Closes #694
+* zulip: Keep reconnecting until succeed (zulip) (#737)
 * irc: add support for (older) unrealircd versions. #708
 * irc: Support quits from irc correctly. Fixes #722 (#724)
 * matrix: Send username when uploading video/images (matrix). Fixes #715 (#717)
-* matrix: Send notices on join/parts (matrix). Fixes #712 (#716)
-* matrix: Detect html nicks in RemoteNickFormat (matrix). Fixes #696 (#719)
+* matrix: Trim <p> and </p> tags (matrix). Closes #686 (#753)
 * slack: Hint at thread replies when messages are unthreaded (slack) (#684)
+* slack: Fix race-condition in populateUser() (#767)
 * xmpp: Do not send topic changes on connect (xmpp). Fixes #732 (#733)
+* telegram: Fix regression in HTML handling (telegram). Closes #734
+* discord: Do not relay any bot messages (discord) (#743)
+* rocketchat: Do not send duplicate messages (rocketchat). Fixes #745 (#752)
 
 ## Contributors
 This release couldn't exist without the following contributors:

ci/bintray.sh

@@ -1,5 +1,8 @@
-#!/bin/bash
-go version | grep go1.11 || exit
+#!/usr/bin/env bash
+set -u -e -x -o pipefail
+
+go version | grep go1.12 || exit
+
 VERSION=$(git describe --tags)
 mkdir ci/binaries
 GOOS=windows GOARCH=amd64 go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-windows-amd64.exe

ci/lint.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -u -e -x -o pipefail
+
+if [[ -n "${GOLANGCI_VERSION-}" ]]; then
+  # Retrieve the golangci-lint linter binary.
+  curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b ${GOPATH}/bin ${GOLANGCI_VERSION}
+fi
+
+# Run the linter.
+golangci-lint run
+
+if [[ "${GO111MODULE-off}" == "on" ]]; then
+  # If Go modules are active then check that dependencies are correctly maintained.
+  go mod tidy
+  go mod vendor
+  git diff --exit-code --quiet || (echo "Please run 'go mod tidy' to clean up the 'go.mod' and 'go.sum' files."; false)
+fi

ci/test.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -u -e -x -o pipefail
+
+if [[ -n "${REPORT_COVERAGE+cover}" ]]; then
+  # Retrieve and prepare CodeClimate's test coverage reporter.
+  curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+  chmod +x ./cc-test-reporter
+  ./cc-test-reporter before-build
+fi
+
+# Run all the tests with the race detector and generate coverage.
+go test -v -race -coverprofile c.out ./...
+
+if [[ -n "${REPORT_COVERAGE+cover}" && "${TRAVIS_SECURE_ENV_VARS}" == "true" ]]; then
+  # Upload test coverage to CodeClimate.
+  ./cc-test-reporter after-build
+fi

gateway/gateway.go

@@ -10,6 +10,7 @@ import (
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/d5/tengo/script"
+	"github.com/d5/tengo/stdlib"
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/peterhellberg/emojilib"
 	"github.com/sirupsen/logrus"
@@ -92,6 +93,9 @@ func (gw *Gateway) AddBridge(cfg *config.Bridge) error {
 			Bridge: br,
 		}
 		// add the actual bridger for this protocol to this bridge using the bridgeMap
+		if _, ok := gw.Router.BridgeMap[br.Protocol]; !ok {
+			gw.logger.Fatalf("Incorrect protocol %s specified in gateway configuration %s, exiting.", br.Protocol, cfg.Account)
+		}
 		br.Bridger = gw.Router.BridgeMap[br.Protocol](brconfig)
 	}
 	gw.mapChannelsToBridge(br)
@@ -156,6 +160,10 @@ func (gw *Gateway) mapChannelConfig(cfg []config.Bridge, direction string) {
 			gw.logger.Errorf("Mattermost channels do not start with a #: remove the # in %s", br.Channel)
 			os.Exit(1)
 		}
+		if strings.HasPrefix(br.Account, "zulip.") && !strings.Contains(br.Channel, "/topic:") {
+			gw.logger.Errorf("Breaking change, since matterbridge 1.14.0 zulip channels need to specify the topic with channel/topic:mytopic in %s of %s", br.Channel, br.Account)
+			os.Exit(1)
+		}
 		ID := br.Channel + br.Account
 		if _, ok := gw.Channels[ID]; !ok {
 			channel := &config.ChannelInfo{
@@ -204,23 +212,6 @@ func (gw *Gateway) getDestChannel(msg *config.Message, dest bridge.Bridge) []con
 		return channels
 	}
 
-	// irc quit is for the whole bridge, isn't a per channel quit.
-	// channel is empty when we quit
-	if msg.Event == config.EventJoinLeave && getProtocol(msg) == "irc" && msg.Channel == "" {
-		// if we only have one channel on this irc bridge it's got to be the sending one.
-		// don't send it back
-		if dest.Account == msg.Account && len(dest.Channels) == 1 && dest.Protocol == "irc" {
-			return channels
-		}
-		for _, channel := range gw.Channels {
-			if channel.Account == dest.Account && strings.Contains(channel.Direction, "out") &&
-				gw.validGatewayDest(msg) {
-				channels = append(channels, *channel)
-			}
-		}
-		return channels
-	}
-
 	// if source channel is in only, do nothing
 	for _, channel := range gw.Channels {
 		// lookup the channel from the message
@@ -496,6 +487,7 @@ func modifyMessageTengo(filename string, msg *config.Message) error {
 		return err
 	}
 	s := script.New(res)
+	s.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
 	_ = s.Add("msgText", msg.Text)
 	_ = s.Add("msgUsername", msg.Username)
 	_ = s.Add("msgAccount", msg.Account)

gateway/router.go

@@ -125,6 +125,7 @@ func (r *Router) handleReceive() {
 		r.handleEventGetChannelMembers(&msg)
 		r.handleEventFailure(&msg)
 		r.handleEventRejoinChannels(&msg)
+		idx := 0
 		for _, gw := range r.Gateways {
 			// record all the message ID's of the different bridges
 			var msgIDs []*BrMsgID
@@ -133,7 +134,9 @@ func (r *Router) handleReceive() {
 			}
 			msg.Timestamp = time.Now()
 			gw.modifyMessage(&msg)
-			gw.handleFiles(&msg)
+			if idx == 0 {
+				gw.handleFiles(&msg)
+			}
 			for _, br := range gw.Bridges {
 				msgIDs = append(msgIDs, gw.handleMessage(&msg, br)...)
 			}
@@ -141,6 +144,7 @@ func (r *Router) handleReceive() {
 			if _, ok := gw.Messages.Get(msg.Protocol + " " + msg.ID); !ok && msg.ID != "" {
 				gw.Messages.Add(msg.Protocol+" "+msg.ID, msgIDs)
 			}
+			idx++
 		}
 	}
 }

go.mod

@@ -6,9 +6,8 @@ require (
 	github.com/BurntSushi/toml v0.0.0-20170318202913-d94612f9fc14 // indirect
 	github.com/Jeffail/gabs v1.1.1 // indirect
 	github.com/Philipp15b/go-steam v1.0.1-0.20180818081528-681bd9573329
-	github.com/Rhymen/go-whatsapp v0.0.0-20190208184307-c9a81e957884
 	github.com/bwmarrin/discordgo v0.19.0
-	github.com/d5/tengo v1.9.2
+	github.com/d5/tengo v1.20.0
 	github.com/dfordsoft/golib v0.0.0-20180902042739-76ee6ab99bec
 	github.com/fsnotify/fsnotify v1.4.7
 	github.com/go-telegram-bot-api/telegram-bot-api v4.6.5-0.20181225215658-ec221ba9ea45+incompatible
@@ -28,10 +27,12 @@ require (
 	github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 // indirect
 	github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 // indirect
 	github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20190210153444-cc9d05784d5d
+	github.com/matterbridge/go-whatsapp v0.0.1-0.20190301204034-f2f1b29d441b
 	github.com/matterbridge/go-xmpp v0.0.0-20180529212104-cd19799fba91
 	github.com/matterbridge/gomatrix v0.0.0-20190102230110-6f9631ca6dea
 	github.com/matterbridge/gozulipbot v0.0.0-20190212232658-7aa251978a18
 	github.com/matterbridge/logrus-prefixed-formatter v0.0.0-20180806162718-01618749af61
+	github.com/matterbridge/mautrix-whatsapp v0.0.0-20190301210046-3539cf52ed6e
 	github.com/mattermost/mattermost-server v5.5.0+incompatible
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
 	github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474 // indirect
@@ -46,7 +47,7 @@ require (
 	github.com/peterhellberg/emojilib v0.0.0-20190124112554-c18758d55320
 	github.com/pkg/errors v0.8.0 // indirect
 	github.com/rs/xid v1.2.1
-	github.com/russross/blackfriday v2.0.0+incompatible
+	github.com/russross/blackfriday v1.5.2
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca
 	github.com/shazow/ssh-chat v0.0.0-20190125184227-81d7e1686296
 	github.com/sirupsen/logrus v1.3.0
@@ -66,8 +67,8 @@ require (
 	go.uber.org/atomic v1.3.2 // indirect
 	go.uber.org/multierr v1.1.0 // indirect
 	go.uber.org/zap v1.9.1 // indirect
+	golang.org/x/image v0.0.0-20190220214146-31aff87c08e9
 	gopkg.in/fsnotify.v1 v1.4.7 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
-	maunium.net/go/mautrix-whatsapp v0.0.0-20190127121751-281b3e8f77f3
 )

go.sum

@@ -8,11 +8,6 @@ github.com/Jeffail/gabs v1.1.1 h1:V0uzR08Hj22EX8+8QMhyI9sX2hwRu+/RJhJUmnwda/E=
 github.com/Jeffail/gabs v1.1.1/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc=
 github.com/Philipp15b/go-steam v1.0.1-0.20180818081528-681bd9573329 h1:xZBoq249G9MSt+XuY7sVQzcfONJ6IQuwpCK+KAaOpnY=
 github.com/Philipp15b/go-steam v1.0.1-0.20180818081528-681bd9573329/go.mod h1:HuVM+sZFzumUdKPWiz+IlCMb4RdsKdT3T+nQBKL+sYg=
-github.com/Rhymen/go-whatsapp v0.0.0-20181218094654-2ca6af00572c h1:ldRXgMEfKmzBomrZusl3edG9AGEeztA7jovLEQy62us=
-github.com/Rhymen/go-whatsapp v0.0.0-20181218094654-2ca6af00572c/go.mod h1:MSDmePOOkbFFbVW2WRRppBcbA+aabwpXRgyIIG7jDFQ=
-github.com/Rhymen/go-whatsapp v0.0.0-20190208184307-c9a81e957884 h1:2AxfzkQi2L4QGBvUCZoWD6hQuUJa5MG54wiYyNqJlf4=
-github.com/Rhymen/go-whatsapp v0.0.0-20190208184307-c9a81e957884/go.mod h1:rdQr95g2C1xcOfM7QGOhza58HeI3I+tZ/bbluv7VazA=
-github.com/alexcesaro/log v0.0.0-20150915221235-61e686294e58 h1:MkpmYfld/S8kXqTYI68DfL8/hHXjHogL120Dy00TIxc=
 github.com/alexcesaro/log v0.0.0-20150915221235-61e686294e58/go.mod h1:YNfsMyWSs+h+PaYkxGeMVmVCX75Zj/pqdjbu12ciCYE=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/bwmarrin/discordgo v0.19.0 h1:kMED/DB0NR1QhRcalb85w0Cu3Ep2OrGAqZH1R5awQiY=
@@ -20,8 +15,8 @@ github.com/bwmarrin/discordgo v0.19.0/go.mod h1:O9S4p+ofTFwB02em7jkpkV8M3R0/PUVO
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/d5/tengo v1.9.2 h1:UE/X8PYl7bLS4Ww2zGeh91nq5PTnkhe8ncgNeA5PK7k=
-github.com/d5/tengo v1.9.2/go.mod h1:gsbjo7lBXzBIWBd6NQp1lRKqqiDDANqBOyhW8rTlFsY=
+github.com/d5/tengo v1.20.0 h1:lFmktzEGR6khlZu2MHUWJ5oDWS4l3jNRV/OhclZgcYc=
+github.com/d5/tengo v1.20.0/go.mod h1:gsbjo7lBXzBIWBd6NQp1lRKqqiDDANqBOyhW8rTlFsY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -43,6 +38,7 @@ github.com/gopackage/ddp v0.0.0-20170117053602-652027933df4/go.mod h1:lEO7XoHJ/x
 github.com/gopherjs/gopherjs v0.0.0-20180628210949-0892b62f0d9f h1:FDM3EtwZLyhW48YRiyqjivNlNZjAObv4xt4NnJaU+NQ=
 github.com/gopherjs/gopherjs v0.0.0-20180628210949-0892b62f0d9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/schema v1.0.2 h1:sAgNfOcNYvdDSrzGHVy9nzCQahG+qmsg+nE8dK85QRA=
 github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
 github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
@@ -82,6 +78,8 @@ github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDe
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20190210153444-cc9d05784d5d h1:F+Sr+C0ojSlYQ37BLylQtSFmyQULe3jbAygcyXQ9mVs=
 github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20190210153444-cc9d05784d5d/go.mod h1:c6MxwqHD+0HvtAJjsHMIdPCiAwGiQwPRPTp69ACMg8A=
+github.com/matterbridge/go-whatsapp v0.0.1-0.20190301204034-f2f1b29d441b h1:cO6Z+yj4Ivq/ay/IxSrV90oSIW/SSXWLa+XHsiLKMrw=
+github.com/matterbridge/go-whatsapp v0.0.1-0.20190301204034-f2f1b29d441b/go.mod h1:dW19fYkkdUZsBAx7zv9fDh0n6NRqYIaKwB2JEBw8d0U=
 github.com/matterbridge/go-xmpp v0.0.0-20180529212104-cd19799fba91 h1:KzDEcy8eDbTx881giW8a6llsAck3e2bJvMyKvh1IK+k=
 github.com/matterbridge/go-xmpp v0.0.0-20180529212104-cd19799fba91/go.mod h1:ECDRehsR9TYTKCAsRS8/wLeOk6UUqDydw47ln7wG41Q=
 github.com/matterbridge/gomatrix v0.0.0-20190102230110-6f9631ca6dea h1:kaADGqpK4gGO2BpzEyJrBxq2Jc57Rsar4i2EUxcACUc=
@@ -90,12 +88,16 @@ github.com/matterbridge/gozulipbot v0.0.0-20190212232658-7aa251978a18 h1:fLhwXtW
 github.com/matterbridge/gozulipbot v0.0.0-20190212232658-7aa251978a18/go.mod h1:yAjnZ34DuDyPHMPHHjOsTk/FefW4JJjoMMCGt/8uuQA=
 github.com/matterbridge/logrus-prefixed-formatter v0.0.0-20180806162718-01618749af61 h1:R/MgM/eUyRBQx2FiH6JVmXck8PaAuKfe2M1tWIzW7nE=
 github.com/matterbridge/logrus-prefixed-formatter v0.0.0-20180806162718-01618749af61/go.mod h1:iXGEotOvwI1R1SjLxRc+BF5rUORTMtE0iMZBT2lxqAU=
+github.com/matterbridge/mautrix-whatsapp v0.0.0-20190301210046-3539cf52ed6e h1:1NqciL8sz+0UYeFrd/UQlL8tJPhFxOBmg+a94DN2sJU=
+github.com/matterbridge/mautrix-whatsapp v0.0.0-20190301210046-3539cf52ed6e/go.mod h1:DrIFGcFumRlEW5k3PJjWGKPd4+w37d3SwOxlh1ZAL+4=
 github.com/mattermost/mattermost-server v5.5.0+incompatible h1:0wcLGgYtd+YImtLDPf2AOfpBHxbU4suATx+6XKw1XbU=
 github.com/mattermost/mattermost-server v5.5.0+incompatible/go.mod h1:5L6MjAec+XXQwMIt791Ganu45GKsSiM+I0tLR9wUj8Y=
-github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
+github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
@@ -129,8 +131,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
-github.com/russross/blackfriday v2.0.0+incompatible h1:cBXrhZNUf9C+La9/YpS+UHpUT8YD6Td9ZMSU9APFcsk=
-github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
@@ -138,11 +140,9 @@ github.com/shazow/rateio v0.0.0-20150116013248-e8e00881e5c1 h1:Lx3BlDGFElJt4u/zK
 github.com/shazow/rateio v0.0.0-20150116013248-e8e00881e5c1/go.mod h1:vt2jWY/3Qw1bIzle5thrJWucsLuuX9iUNnp20CqCciI=
 github.com/shazow/ssh-chat v0.0.0-20190125184227-81d7e1686296 h1:8RLq547MSVc6vhOuCl4Ca0TsAQknj6NX6ZLSZ3+xmio=
 github.com/shazow/ssh-chat v0.0.0-20190125184227-81d7e1686296/go.mod h1:1GLXsL4esywkpNId3v4QWuMf3THtWGitWvtQ/L3aSA4=
-github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME=
 github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/skip2/go-qrcode v0.0.0-20171229120447-cf5f9fa2f0d8/go.mod h1:PLPIyL7ikehBD1OAjmKKiOEhbvWyHGaNDjquXMcYABo=
 github.com/skip2/go-qrcode v0.0.0-20190110000554-dc11ecdae0a9 h1:lpEzuenPuO1XNTeikEmvqYFcU37GVLl8SRNblzyvGBE=
 github.com/skip2/go-qrcode v0.0.0-20190110000554-dc11ecdae0a9/go.mod h1:PLPIyL7ikehBD1OAjmKKiOEhbvWyHGaNDjquXMcYABo=
 github.com/smartystreets/assertions v0.0.0-20180803164922-886ec427f6b9 h1:lXQ+j+KwZcbwrbgU0Rp4Eglg3EJLHbuZU3BbOqAGBmg=
@@ -160,9 +160,7 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38=
 github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -197,25 +195,24 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/
 go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o=
 go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 h1:y6ce7gCWtnH+m3dCjzQ1PCuwl28DDIc3VNnvY29DlIA=
 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190130090550-b01c7a725664 h1:YbZJ76lQ1BqNhVe7dKTSB67wDrc2VPRR75IyGyyPDX8=
 golang.org/x/crypto v0.0.0-20190130090550-b01c7a725664/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc=
 golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190222235706-ffb98f73852f h1:qWFY9ZxP3tfI37wYIs/MnIAqK0vlXp1xnYEa5HxFSSY=
+golang.org/x/crypto v0.0.0-20190222235706-ffb98f73852f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/image v0.0.0-20190220214146-31aff87c08e9 h1:+vH8qNweCrORN49012OX3h0oWEXO3p+rRnpAGQinddk=
+golang.org/x/image v0.0.0-20190220214146-31aff87c08e9/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/net v0.0.0-20190110200230-915654e7eabc h1:Yx9JGxI1SBhVLFjpAkWMaO1TF+xyqtHLjZpvQboJGiM=
 golang.org/x/net v0.0.0-20190110200230-915654e7eabc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181212120007-b05ddf57801d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc h1:WiYx1rIFmx8c0mXAFtv5D/mHyKe1+jmuP7PViuwqwuQ=
 golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222171317-cd391775e71e h1:oF7qaQxUH6KzFdKN4ww7NpPdo53SZi4UlcksLrb2y/o=
+golang.org/x/sys v0.0.0-20190222171317-cd391775e71e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -233,5 +230,3 @@ maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfk
 maunium.net/go/maulogger/v2 v2.0.0/go.mod h1:Hbbkq3NV6jvJodByZu1mgEF3fpT7Kz9z0MjEZ3/BusI=
 maunium.net/go/mautrix v0.1.0-alpha.3/go.mod h1:GTVu6WDHR+98DKOrYetWsXorvUeKQV3jsSWO6ScbuFI=
 maunium.net/go/mautrix-appservice v0.1.0-alpha.3/go.mod h1:wOnWOIuprYad7ly12rHIo3JLCPh4jwvx1prVrAB9RhM=
-maunium.net/go/mautrix-whatsapp v0.0.0-20190127121751-281b3e8f77f3 h1:A18t5Lp7I3aK0V7B7zdpb0hb/PBlu0X/Ai2AyU/XEk4=
-maunium.net/go/mautrix-whatsapp v0.0.0-20190127121751-281b3e8f77f3/go.mod h1:r5E3J4urDEsjfui9OYZYMLBfCliaAqcCwM2xeczta6k=

matterbridge.go

@@ -15,7 +15,7 @@ import (
 )
 
 var (
-	version = "1.14.0-rc1"
+	version = "1.14.4"
 	githash string
 
 	flagConfig  = flag.String("conf", "matterbridge.toml", "config file")

matterbridge.toml.sample

@@ -27,7 +27,7 @@ UseTLS=false
 #OPTIONAL (default false)
 UseSASL=false
 
-#Enable to not verify the certificate on your irc server. i
+#Enable to not verify the certificate on your irc server.
 #e.g. when using selfsigned certificates
 #OPTIONAL (default false)
 SkipTLSVerify=true
@@ -913,6 +913,11 @@ QuoteDisable=false
 #OPTIONAL (default "{MESSAGE} (re @{QUOTENICK}: {QUOTEMESSAGE})")
 QuoteFormat="{MESSAGE} (re @{QUOTENICK}: {QUOTEMESSAGE})"
 
+#Convert WebP images to PNG before upload.
+#https://github.com/42wim/matterbridge/issues/398
+#OPTIONAL (default false)
+MediaConvertWebPToPNG=false
+
 #Disable sending of edits to other bridges
 #OPTIONAL (default false)
 EditDisable=false
@@ -1002,9 +1007,10 @@ ShowTopicChange=false
 Server="https://yourrocketchatserver.domain.com:443"
 
 #login/pass of your bot. 
+#login needs to be the login with email address! user@domain.com
 #Use a dedicated user for this and not your own! 
 #REQUIRED (when not using webhooks)
-Login="yourlogin"
+Login="yourlogin@domain.com"
 Password="yourpass"
 
 #### Settings for webhook matterbridge.
@@ -1045,6 +1051,8 @@ SkipTLSVerify=true
 #Useful if username overrides for incoming webhooks isn't enabled on the 
 #rocketchat server. If you set PrefixMessagesWithNick to true, each message 
 #from bridge to rocketchat will by default be prefixed by the RemoteNickFormat setting. i
+#if you're using login/pass you can better enable because of this bug:
+#https://github.com/RocketChat/Rocket.Chat/issues/7549
 #OPTIONAL (default false)
 PrefixMessagesWithNick=false
 
@@ -1358,12 +1366,6 @@ Login="yourbot-bot@yourserver.zulipchat.com"
 #REQUIRED 
 Server="https://yourserver.zulipchat.com"
 
-#Topic of the messages matterbridge will use
-#OPTIONAL (default "matterbridge")
-#You can specify a specific topic for each channel using [gateway.inout.options]
-#See more information below at the gateway configuration
-Topic="matterbridge"
-
 ## RELOADABLE SETTINGS
 ## Settings below can be reloaded by editing the file
 
@@ -1603,7 +1605,7 @@ enable=true
     #              if you specify an empty string bridge will list all the possibilities
     #            - "Group Name" if you specify a group name the bridge will hint its JID to specify
     #              as group names might change in time and contain weird emoticons
-    # zulip      - stream (without the #)
+    # zulip      - stream/topic:topicname (without the #)
     #                  
     # REQUIRED
     channel="#testing"
@@ -1645,10 +1647,7 @@ enable=true
 
     [[gateway.inout]]
     account="zulip.streamchat"
-    channel="general"
-        #OPTIONAL - topic only works for zulip
-        [gateway.inout.options]
-        topic="topic1"
+    channel="general/topic:mytopic"
 
     #API example
     #[[gateway.inout]]

matterclient/channels.go

@@ -51,8 +51,9 @@ func (m *MMClient) GetChannelId(name string, teamId string) string { //nolint:go
 				if res == name {
 					return channel.Id
 				}
+			} else if channel.Name == name {
+				return channel.Id
 			}
-
 		}
 	}
 	return ""

matterclient/helpers.go

@@ -132,14 +132,25 @@ func (m *MMClient) initUser() error {
 		return resp.Error
 	}
 	for _, team := range teams {
-		mmusers, resp := m.Client.GetUsersInTeam(team.Id, 0, 50000, "")
+		idx := 0
+		max := 200
+		usermap := make(map[string]*model.User)
+		mmusers, resp := m.Client.GetUsersInTeam(team.Id, idx, max, "")
 		if resp.Error != nil {
 			return errors.New(resp.Error.DetailedError)
 		}
-		usermap := make(map[string]*model.User)
-		for _, user := range mmusers {
-			usermap[user.Id] = user
+		for len(mmusers) > 0 {
+			for _, user := range mmusers {
+				usermap[user.Id] = user
+			}
+			mmusers, resp = m.Client.GetUsersInTeam(team.Id, idx, max, "")
+			if resp.Error != nil {
+				return errors.New(resp.Error.DetailedError)
+			}
+			idx++
+			time.Sleep(time.Millisecond * 200)
 		}
+		m.logger.Infof("found %d users in team %s", len(usermap), team.Name)
 
 		t := &Team{Team: team, Users: usermap, Id: team.Id}
 

matterclient/matterclient.go

@@ -216,9 +216,14 @@ func (m *MMClient) WsReceiver() {
 			if msg.Post != nil {
 				if msg.Text != "" || len(msg.Post.FileIds) > 0 || msg.Post.Type == "slack_attachment" {
 					m.MessageChan <- msg
+					continue
 				}
 			}
-			continue
+			switch msg.Raw.Event {
+			case model.WEBSOCKET_EVENT_USER_ADDED, model.WEBSOCKET_EVENT_USER_REMOVED:
+				m.MessageChan <- msg
+				continue
+			}
 		}
 
 		var response model.WebSocketResponse

matterclient/messages.go

@@ -83,7 +83,7 @@ func (m *MMClient) DeleteMessage(postId string) error { //nolint:golint
 }
 
 func (m *MMClient) EditMessage(postId string, text string) (string, error) { //nolint:golint
-	post := &model.Post{Message: text}
+	post := &model.Post{Message: text, Id: postId}
 	res, resp := m.Client.UpdatePost(postId, post)
 	if resp.Error != nil {
 		return "", resp.Error

matterclient/users.go

@@ -2,6 +2,7 @@ package matterclient
 
 import (
 	"errors"
+	"time"
 
 	"github.com/mattermost/mattermost-server/model"
 )
@@ -99,15 +100,25 @@ func (m *MMClient) GetUsers() map[string]*model.User {
 }
 
 func (m *MMClient) UpdateUsers() error {
-	mmusers, resp := m.Client.GetUsers(0, 50000, "")
+	idx := 0
+	max := 200
+	mmusers, resp := m.Client.GetUsers(idx, max, "")
 	if resp.Error != nil {
 		return errors.New(resp.Error.DetailedError)
 	}
-	m.Lock()
-	for _, user := range mmusers {
-		m.Users[user.Id] = user
+	for len(mmusers) > 0 {
+		m.Lock()
+		for _, user := range mmusers {
+			m.Users[user.Id] = user
+		}
+		m.Unlock()
+		mmusers, resp = m.Client.GetUsers(idx, max, "")
+		time.Sleep(time.Millisecond * 300)
+		if resp.Error != nil {
+			return errors.New(resp.Error.DetailedError)
+		}
+		idx++
 	}
-	m.Unlock()
 	return nil
 }
 

vendor/github.com/d5/tengo/.gitignore

@@ -0,0 +1 @@
+dist/

vendor/github.com/d5/tengo/.goreleaser.yml

@@ -0,0 +1,23 @@
+builds:
+  - env:
+      - CGO_ENABLED=0
+    main: ./cmd/tengo/main.go
+    goos:
+      - darwin
+      - linux
+      - windows
+  - env:
+      - CGO_ENABLED=0
+    main: ./cmd/tengomin/main.go
+    binary: tengomin
+    goos:
+      - darwin
+      - linux
+      - windows
+archive:
+  files:
+    - none*
+checksum:
+  name_template: 'checksums.txt'
+changelog:
+  sort: asc

vendor/github.com/d5/tengo/.travis.yml

@@ -0,0 +1,17 @@
+language: go
+
+go:
+  - 1.9
+
+install:
+  - go get -u golang.org/x/lint/golint
+
+script:
+  - make test
+
+deploy:
+  - provider: script
+    skip_cleanup: true
+    script: curl -sL https://git.io/goreleaser | bash
+    on:
+      tags: true

vendor/github.com/d5/tengo/Makefile

@@ -0,0 +1,14 @@
+vet:
+	go vet ./...
+
+generate:
+	go generate ./...
+
+lint:
+	golint -set_exit_status ./...
+
+test: generate vet lint
+	go test -race -cover ./...
+
+fmt:
+	go fmt ./...

vendor/github.com/d5/tengo/README.md

@@ -0,0 +1,76 @@
+<p align="center">
+  <img src="https://raw.githubusercontent.com/d5/tengolang.com/master/logo_400.png" width="200" height="200">
+</p>
+
+# The Tengo Language
+
+[![GoDoc](https://godoc.org/github.com/d5/tengo?status.svg)](https://godoc.org/github.com/d5/tengo/script)
+[![Go Report Card](https://goreportcard.com/badge/github.com/d5/tengo)](https://goreportcard.com/report/github.com/d5/tengo)
+[![Build Status](https://travis-ci.org/d5/tengo.svg?branch=master)](https://travis-ci.org/d5/tengo)
+
+**Tengo is a small, dynamic, fast, secure script language for Go.** 
+
+Tengo is **[fast](#benchmark)** and secure because it's compiled/executed as bytecode on stack-based VM that's written in native Go.
+
+```golang
+/* The Tengo Language */
+
+fmt := import("fmt")
+
+each := func(seq, fn) {
+    for x in seq { fn(x) }
+}
+
+sum := func(init, seq) {
+    each(seq, func(x) { init += x })
+    return init
+}
+
+fmt.println(sum(0, [1, 2, 3]))   // "6"
+fmt.println(sum("", [1, 2, 3]))  // "123"
+```
+
+> Run this code in the [Playground](https://tengolang.com/?s=0c8d5d0d88f2795a7093d7f35ae12c3afa17bea3)
+
+## Features
+
+- Simple and highly readable [Syntax](https://github.com/d5/tengo/blob/master/docs/tutorial.md)
+  - Dynamic typing with type coercion
+  - Higher-order functions and closures
+  - Immutable values
+  - Garbage collection
+- [Securely Embeddable](https://github.com/d5/tengo/blob/master/docs/interoperability.md) and [Extensible](https://github.com/d5/tengo/blob/master/docs/objects.md)
+- Compiler/runtime written in native Go _(no external deps or cgo)_
+- Executable as a [standalone](https://github.com/d5/tengo/blob/master/docs/tengo-cli.md) language / REPL
+- Use cases: rules engine, [state machine](https://github.com/d5/go-fsm), [gaming](https://github.com/d5/pbr), data pipeline, [transpiler](https://github.com/d5/tengo2lua)
+
+## Benchmark
+
+| | fib(35) | fibt(35) |  Type  |
+| :--- |    ---: |     ---: |  :---: |
+| Go | `48ms` | `3ms` | Go (native) |
+| [**Tengo**](https://github.com/d5/tengo) | `2,349ms` | `5ms` | VM on Go |
+| Lua | `1,416ms` | `3ms` | Lua (native) |
+| [go-lua](https://github.com/Shopify/go-lua) | `4,402ms` | `5ms` | Lua VM on Go |
+| [GopherLua](https://github.com/yuin/gopher-lua) | `4,023ms` | `5ms` | Lua VM on Go |
+| Python | `2,588ms` | `26ms` | Python (native) |
+| [starlark-go](https://github.com/google/starlark-go) | `11,126ms` | `6ms` | Python-like Interpreter on Go |
+| [gpython](https://github.com/go-python/gpython) | `15,035ms` | `4ms` | Python Interpreter on Go |
+| [goja](https://github.com/dop251/goja) | `5,089ms` | `5ms` | JS VM on Go |
+| [otto](https://github.com/robertkrimen/otto) | `68,377ms` | `11ms` | JS Interpreter on Go |
+| [Anko](https://github.com/mattn/anko) | `92,579ms` | `18ms` | Interpreter on Go |
+
+_* [fib(35)](https://github.com/d5/tengobench/blob/master/code/fib.tengo): Fibonacci(35)_  
+_* [fibt(35)](https://github.com/d5/tengobench/blob/master/code/fibtc.tengo): [tail-call](https://en.wikipedia.org/wiki/Tail_call) version of Fibonacci(35)_  
+_* **Go** does not read the source code from file, while all other cases do_  
+_* See [here](https://github.com/d5/tengobench) for commands/codes used_
+
+## References
+
+- [Language Syntax](https://github.com/d5/tengo/blob/master/docs/tutorial.md)
+- [Object Types](https://github.com/d5/tengo/blob/master/docs/objects.md)
+- [Runtime Types](https://github.com/d5/tengo/blob/master/docs/runtime-types.md) and [Operators](https://github.com/d5/tengo/blob/master/docs/operators.md)
+- [Builtin Functions](https://github.com/d5/tengo/blob/master/docs/builtins.md)
+- [Interoperability](https://github.com/d5/tengo/blob/master/docs/interoperability.md)
+- [Tengo CLI](https://github.com/d5/tengo/blob/master/docs/tengo-cli.md)
+- [Standard Library](https://github.com/d5/tengo/blob/master/docs/stdlib.md)

vendor/github.com/d5/tengo/compiler/bytecode.go

@@ -17,32 +17,6 @@ type Bytecode struct {
 	Constants    []objects.Object
 }
 
-// Decode reads Bytecode data from the reader.
-func (b *Bytecode) Decode(r io.Reader) error {
-	dec := gob.NewDecoder(r)
-
-	if err := dec.Decode(&b.FileSet); err != nil {
-		return err
-	}
-	// TODO: files in b.FileSet.File does not have their 'set' field properly set to b.FileSet
-	// as it's private field and not serialized by gob encoder/decoder.
-
-	if err := dec.Decode(&b.MainFunction); err != nil {
-		return err
-	}
-
-	if err := dec.Decode(&b.Constants); err != nil {
-		return err
-	}
-
-	// replace Bool and Undefined with known value
-	for i, v := range b.Constants {
-		b.Constants[i] = cleanupObjects(v)
-	}
-
-	return nil
-}
-
 // Encode writes Bytecode data to the writer.
 func (b *Bytecode) Encode(w io.Writer) error {
 	enc := gob.NewEncoder(w)
@@ -59,6 +33,17 @@ func (b *Bytecode) Encode(w io.Writer) error {
 	return enc.Encode(b.Constants)
 }
 
+// CountObjects returns the number of objects found in Constants.
+func (b *Bytecode) CountObjects() int {
+	n := 0
+
+	for _, c := range b.Constants {
+		n += objects.CountObjects(c)
+	}
+
+	return n
+}
+
 // FormatInstructions returns human readable string representations of
 // compiled instructions.
 func (b *Bytecode) FormatInstructions() []string {
@@ -83,51 +68,22 @@ func (b *Bytecode) FormatConstants() (output []string) {
 	return
 }
 
-func cleanupObjects(o objects.Object) objects.Object {
-	switch o := o.(type) {
-	case *objects.Bool:
-		if o.IsFalsy() {
-			return objects.FalseValue
-		}
-		return objects.TrueValue
-	case *objects.Undefined:
-		return objects.UndefinedValue
-	case *objects.Array:
-		for i, v := range o.Value {
-			o.Value[i] = cleanupObjects(v)
-		}
-	case *objects.Map:
-		for k, v := range o.Value {
-			o.Value[k] = cleanupObjects(v)
-		}
-	}
-
-	return o
-}
-
 func init() {
 	gob.Register(&source.FileSet{})
 	gob.Register(&source.File{})
 	gob.Register(&objects.Array{})
-	gob.Register(&objects.ArrayIterator{})
 	gob.Register(&objects.Bool{})
-	gob.Register(&objects.Break{})
-	gob.Register(&objects.BuiltinFunction{})
 	gob.Register(&objects.Bytes{})
 	gob.Register(&objects.Char{})
 	gob.Register(&objects.Closure{})
 	gob.Register(&objects.CompiledFunction{})
-	gob.Register(&objects.Continue{})
 	gob.Register(&objects.Error{})
 	gob.Register(&objects.Float{})
 	gob.Register(&objects.ImmutableArray{})
 	gob.Register(&objects.ImmutableMap{})
 	gob.Register(&objects.Int{})
 	gob.Register(&objects.Map{})
-	gob.Register(&objects.MapIterator{})
-	gob.Register(&objects.ReturnValue{})
 	gob.Register(&objects.String{})
-	gob.Register(&objects.StringIterator{})
 	gob.Register(&objects.Time{})
 	gob.Register(&objects.Undefined{})
 	gob.Register(&objects.UserFunction{})

vendor/github.com/d5/tengo/compiler/bytecode_decode.go

@@ -0,0 +1,97 @@
+package compiler
+
+import (
+	"encoding/gob"
+	"fmt"
+	"io"
+
+	"github.com/d5/tengo/objects"
+)
+
+// Decode reads Bytecode data from the reader.
+func (b *Bytecode) Decode(r io.Reader, modules *objects.ModuleMap) error {
+	if modules == nil {
+		modules = objects.NewModuleMap()
+	}
+
+	dec := gob.NewDecoder(r)
+
+	if err := dec.Decode(&b.FileSet); err != nil {
+		return err
+	}
+	// TODO: files in b.FileSet.File does not have their 'set' field properly set to b.FileSet
+	// as it's private field and not serialized by gob encoder/decoder.
+
+	if err := dec.Decode(&b.MainFunction); err != nil {
+		return err
+	}
+
+	if err := dec.Decode(&b.Constants); err != nil {
+		return err
+	}
+	for i, v := range b.Constants {
+		fv, err := fixDecoded(v, modules)
+		if err != nil {
+			return err
+		}
+		b.Constants[i] = fv
+	}
+
+	return nil
+}
+
+func fixDecoded(o objects.Object, modules *objects.ModuleMap) (objects.Object, error) {
+	switch o := o.(type) {
+	case *objects.Bool:
+		if o.IsFalsy() {
+			return objects.FalseValue, nil
+		}
+		return objects.TrueValue, nil
+	case *objects.Undefined:
+		return objects.UndefinedValue, nil
+	case *objects.Array:
+		for i, v := range o.Value {
+			fv, err := fixDecoded(v, modules)
+			if err != nil {
+				return nil, err
+			}
+			o.Value[i] = fv
+		}
+	case *objects.ImmutableArray:
+		for i, v := range o.Value {
+			fv, err := fixDecoded(v, modules)
+			if err != nil {
+				return nil, err
+			}
+			o.Value[i] = fv
+		}
+	case *objects.Map:
+		for k, v := range o.Value {
+			fv, err := fixDecoded(v, modules)
+			if err != nil {
+				return nil, err
+			}
+			o.Value[k] = fv
+		}
+	case *objects.ImmutableMap:
+		modName := moduleName(o)
+		if mod := modules.GetBuiltinModule(modName); mod != nil {
+			return mod.AsImmutableMap(modName), nil
+		}
+
+		for k, v := range o.Value {
+			// encoding of user function not supported
+			if _, isUserFunction := v.(*objects.UserFunction); isUserFunction {
+				return nil, fmt.Errorf("user function not decodable")
+			}
+
+			fv, err := fixDecoded(v, modules)
+			if err != nil {
+				return nil, err
+			}
+			o.Value[k] = fv
+		}
+	}
+
+	return o, nil
+}

vendor/github.com/d5/tengo/compiler/bytecode_optimize.go

@@ -0,0 +1,129 @@
+package compiler
+
+import (
+	"fmt"
+
+	"github.com/d5/tengo/objects"
+)
+
+// RemoveDuplicates finds and remove the duplicate values in Constants.
+// Note this function mutates Bytecode.
+func (b *Bytecode) RemoveDuplicates() {
+	var deduped []objects.Object
+
+	indexMap := make(map[int]int) // mapping from old constant index to new index
+	ints := make(map[int64]int)
+	strings := make(map[string]int)
+	floats := make(map[float64]int)
+	chars := make(map[rune]int)
+	immutableMaps := make(map[string]int) // for modules
+
+	for curIdx, c := range b.Constants {
+		switch c := c.(type) {
+		case *objects.CompiledFunction:
+			// add to deduped list
+			indexMap[curIdx] = len(deduped)
+			deduped = append(deduped, c)
+		case *objects.ImmutableMap:
+			modName := moduleName(c)
+			newIdx, ok := immutableMaps[modName]
+			if modName != "" && ok {
+				indexMap[curIdx] = newIdx
+			} else {
+				newIdx = len(deduped)
+				immutableMaps[modName] = newIdx
+				indexMap[curIdx] = newIdx
+				deduped = append(deduped, c)
+			}
+		case *objects.Int:
+			if newIdx, ok := ints[c.Value]; ok {
+				indexMap[curIdx] = newIdx
+			} else {
+				newIdx = len(deduped)
+				ints[c.Value] = newIdx
+				indexMap[curIdx] = newIdx
+				deduped = append(deduped, c)
+			}
+		case *objects.String:
+			if newIdx, ok := strings[c.Value]; ok {
+				indexMap[curIdx] = newIdx
+			} else {
+				newIdx = len(deduped)
+				strings[c.Value] = newIdx
+				indexMap[curIdx] = newIdx
+				deduped = append(deduped, c)
+			}
+		case *objects.Float:
+			if newIdx, ok := floats[c.Value]; ok {
+				indexMap[curIdx] = newIdx
+			} else {
+				newIdx = len(deduped)
+				floats[c.Value] = newIdx
+				indexMap[curIdx] = newIdx
+				deduped = append(deduped, c)
+			}
+		case *objects.Char:
+			if newIdx, ok := chars[c.Value]; ok {
+				indexMap[curIdx] = newIdx
+			} else {
+				newIdx = len(deduped)
+				chars[c.Value] = newIdx
+				indexMap[curIdx] = newIdx
+				deduped = append(deduped, c)
+			}
+		default:
+			panic(fmt.Errorf("unsupported top-level constant type: %s", c.TypeName()))
+		}
+	}
+
+	// replace with de-duplicated constants
+	b.Constants = deduped
+
+	// update CONST instructions with new indexes
+	// main function
+	updateConstIndexes(b.MainFunction.Instructions, indexMap)
+	// other compiled functions in constants
+	for _, c := range b.Constants {
+		switch c := c.(type) {
+		case *objects.CompiledFunction:
+			updateConstIndexes(c.Instructions, indexMap)
+		}
+	}
+}
+
+func updateConstIndexes(insts []byte, indexMap map[int]int) {
+	i := 0
+	for i < len(insts) {
+		op := insts[i]
+		numOperands := OpcodeOperands[op]
+		_, read := ReadOperands(numOperands, insts[i+1:])
+
+		switch op {
+		case OpConstant:
+			curIdx := int(insts[i+2]) | int(insts[i+1])<<8
+			newIdx, ok := indexMap[curIdx]
+			if !ok {
+				panic(fmt.Errorf("constant index not found: %d", curIdx))
+			}
+			copy(insts[i:], MakeInstruction(op, newIdx))
+		case OpClosure:
+			curIdx := int(insts[i+2]) | int(insts[i+1])<<8
+			numFree := int(insts[i+3])
+			newIdx, ok := indexMap[curIdx]
+			if !ok {
+				panic(fmt.Errorf("constant index not found: %d", curIdx))
+			}
+			copy(insts[i:], MakeInstruction(op, newIdx, numFree))
+		}
+
+		i += 1 + read
+	}
+}
+
+func moduleName(mod *objects.ImmutableMap) string {
+	if modName, ok := mod.Value["__module_name__"].(*objects.String); ok {
+		return modName.Value
+	}
+
+	return ""
+}

vendor/github.com/d5/tengo/compiler/compilation_scope.go

@@ -5,8 +5,7 @@ import "github.com/d5/tengo/compiler/source"
 // CompilationScope represents a compiled instructions
 // and the last two instructions that were emitted.
 type CompilationScope struct {
-	instructions     []byte
-	lastInstructions [2]EmittedInstruction
-	symbolInit       map[string]bool
-	sourceMap        map[int]source.Pos
+	instructions []byte
+	symbolInit   map[string]bool
+	sourceMap    map[int]source.Pos
 }

vendor/github.com/d5/tengo/compiler/compiler.go

@@ -3,27 +3,30 @@ package compiler
 import (
 	"fmt"
 	"io"
+	"io/ioutil"
+	"path/filepath"
 	"reflect"
+	"strings"
 
+	"github.com/d5/tengo"
 	"github.com/d5/tengo/compiler/ast"
 	"github.com/d5/tengo/compiler/source"
 	"github.com/d5/tengo/compiler/token"
 	"github.com/d5/tengo/objects"
-	"github.com/d5/tengo/stdlib"
 )
 
 // Compiler compiles the AST into a bytecode.
 type Compiler struct {
 	file            *source.File
 	parent          *Compiler
-	moduleName      string
+	modulePath      string
 	constants       []objects.Object
 	symbolTable     *SymbolTable
 	scopes          []CompilationScope
 	scopeIndex      int
-	moduleLoader    ModuleLoader
-	builtinModules  map[string]bool
+	modules         *objects.ModuleMap
 	compiledModules map[string]*objects.CompiledFunction
+	allowFileImport bool
 	loops           []*Loop
 	loopIndex       int
 	trace           io.Writer
@@ -31,12 +34,7 @@ type Compiler struct {
 }
 
 // NewCompiler creates a Compiler.
-// User can optionally provide the symbol table if one wants to add or remove
-// some global- or builtin- scope symbols. If not (nil), Compile will create
-// a new symbol table and use the default builtin functions. Likewise, standard
-// modules can be explicitly provided if user wants to add or remove some modules.
-// By default, Compile will use all the standard modules otherwise.
-func NewCompiler(file *source.File, symbolTable *SymbolTable, constants []objects.Object, builtinModules map[string]bool, trace io.Writer) *Compiler {
+func NewCompiler(file *source.File, symbolTable *SymbolTable, constants []objects.Object, modules *objects.ModuleMap, trace io.Writer) *Compiler {
 	mainScope := CompilationScope{
 		symbolInit: make(map[string]bool),
 		sourceMap:  make(map[int]source.Pos),
@@ -45,18 +43,16 @@ func NewCompiler(file *source.File, symbolTable *SymbolTable, constants []object
 	// symbol table
 	if symbolTable == nil {
 		symbolTable = NewSymbolTable()
+	}
 
-		for idx, fn := range objects.Builtins {
-			symbolTable.DefineBuiltin(idx, fn.Name)
-		}
+	// add builtin functions to the symbol table
+	for idx, fn := range objects.Builtins {
+		symbolTable.DefineBuiltin(idx, fn.Name)
 	}
 
 	// builtin modules
-	if builtinModules == nil {
-		builtinModules = make(map[string]bool)
-		for name := range stdlib.Modules {
-			builtinModules[name] = true
-		}
+	if modules == nil {
+		modules = objects.NewModuleMap()
 	}
 
 	return &Compiler{
@@ -67,7 +63,7 @@ func NewCompiler(file *source.File, symbolTable *SymbolTable, constants []object
 		scopeIndex:      0,
 		loopIndex:       -1,
 		trace:           trace,
-		builtinModules:  builtinModules,
+		modules:         modules,
 		compiledModules: make(map[string]*objects.CompiledFunction),
 	}
 }
@@ -123,7 +119,7 @@ func (c *Compiler) Compile(node ast.Node) error {
 				return err
 			}
 
-			c.emit(node, OpGreaterThan)
+			c.emit(node, OpBinaryOp, int(token.Greater))
 
 			return nil
 		} else if node.Token == token.LessEq {
@@ -134,7 +130,7 @@ func (c *Compiler) Compile(node ast.Node) error {
 				return err
 			}
 
-			c.emit(node, OpGreaterThanEqual)
+			c.emit(node, OpBinaryOp, int(token.GreaterEq))
 
 			return nil
 		}
@@ -148,35 +144,35 @@ func (c *Compiler) Compile(node ast.Node) error {
 
 		switch node.Token {
 		case token.Add:
-			c.emit(node, OpAdd)
+			c.emit(node, OpBinaryOp, int(token.Add))
 		case token.Sub:
-			c.emit(node, OpSub)
+			c.emit(node, OpBinaryOp, int(token.Sub))
 		case token.Mul:
-			c.emit(node, OpMul)
+			c.emit(node, OpBinaryOp, int(token.Mul))
 		case token.Quo:
-			c.emit(node, OpDiv)
+			c.emit(node, OpBinaryOp, int(token.Quo))
 		case token.Rem:
-			c.emit(node, OpRem)
+			c.emit(node, OpBinaryOp, int(token.Rem))
 		case token.Greater:
-			c.emit(node, OpGreaterThan)
+			c.emit(node, OpBinaryOp, int(token.Greater))
 		case token.GreaterEq:
-			c.emit(node, OpGreaterThanEqual)
+			c.emit(node, OpBinaryOp, int(token.GreaterEq))
 		case token.Equal:
 			c.emit(node, OpEqual)
 		case token.NotEqual:
 			c.emit(node, OpNotEqual)
 		case token.And:
-			c.emit(node, OpBAnd)
+			c.emit(node, OpBinaryOp, int(token.And))
 		case token.Or:
-			c.emit(node, OpBOr)
+			c.emit(node, OpBinaryOp, int(token.Or))
 		case token.Xor:
-			c.emit(node, OpBXor)
+			c.emit(node, OpBinaryOp, int(token.Xor))
 		case token.AndNot:
-			c.emit(node, OpBAndNot)
+			c.emit(node, OpBinaryOp, int(token.AndNot))
 		case token.Shl:
-			c.emit(node, OpBShiftLeft)
+			c.emit(node, OpBinaryOp, int(token.Shl))
 		case token.Shr:
-			c.emit(node, OpBShiftRight)
+			c.emit(node, OpBinaryOp, int(token.Shr))
 		default:
 			return c.errorf(node, "invalid binary operator: %s", node.Token.String())
 		}
@@ -195,6 +191,10 @@ func (c *Compiler) Compile(node ast.Node) error {
 		}
 
 	case *ast.StringLit:
+		if len(node.Value) > tengo.MaxStringLen {
+			return c.error(node, objects.ErrStringLimit)
+		}
+
 		c.emit(node, OpConstant, c.addConstant(&objects.String{Value: node.Value}))
 
 	case *ast.CharLit:
@@ -292,6 +292,15 @@ func (c *Compiler) Compile(node ast.Node) error {
 		}
 
 	case *ast.BlockStmt:
+		if len(node.Stmts) == 0 {
+			return nil
+		}
+
+		c.symbolTable = c.symbolTable.Fork(true)
+		defer func() {
+			c.symbolTable = c.symbolTable.Parent(false)
+		}()
+
 		for _, stmt := range node.Stmts {
 			if err := c.Compile(stmt); err != nil {
 				return err
@@ -332,6 +341,9 @@ func (c *Compiler) Compile(node ast.Node) error {
 	case *ast.MapLit:
 		for _, elt := range node.Elements {
 			// key
+			if len(elt.Key) > tengo.MaxStringLen {
+				return c.error(node, objects.ErrStringLimit)
+			}
 			c.emit(node, OpConstant, c.addConstant(&objects.String{Value: elt.Key}))
 
 			// value
@@ -401,10 +413,8 @@ func (c *Compiler) Compile(node ast.Node) error {
 			return err
 		}
 
-		// add OpReturn if function returns nothing
-		if !c.lastInstructionIs(OpReturnValue) && !c.lastInstructionIs(OpReturn) {
-			c.emit(node, OpReturn)
-		}
+		// code optimization
+		c.optimizeFunc(node)
 
 		freeSymbols := c.symbolTable.FreeSymbols()
 		numLocals := c.symbolTable.MaxSymbols()
@@ -457,9 +467,9 @@ func (c *Compiler) Compile(node ast.Node) error {
 					s.LocalAssigned = true
 				}
 
-				c.emit(node, OpGetLocal, s.Index)
+				c.emit(node, OpGetLocalPtr, s.Index)
 			case ScopeFree:
-				c.emit(node, OpGetFree, s.Index)
+				c.emit(node, OpGetFreePtr, s.Index)
 			}
 		}
 
@@ -483,13 +493,13 @@ func (c *Compiler) Compile(node ast.Node) error {
 		}
 
 		if node.Result == nil {
-			c.emit(node, OpReturn)
+			c.emit(node, OpReturn, 0)
 		} else {
 			if err := c.Compile(node.Result); err != nil {
 				return err
 			}
 
-			c.emit(node, OpReturnValue)
+			c.emit(node, OpReturn, 1)
 		}
 
 	case *ast.CallExpr:
@@ -506,17 +516,57 @@ func (c *Compiler) Compile(node ast.Node) error {
 		c.emit(node, OpCall, len(node.Args))
 
 	case *ast.ImportExpr:
-		if c.builtinModules[node.ModuleName] {
-			c.emit(node, OpConstant, c.addConstant(&objects.String{Value: node.ModuleName}))
-			c.emit(node, OpGetBuiltinModule)
-		} else {
-			userMod, err := c.compileModule(node)
+		if node.ModuleName == "" {
+			return c.errorf(node, "empty module name")
+		}
+
+		if mod := c.modules.Get(node.ModuleName); mod != nil {
+			v, err := mod.Import(node.ModuleName)
 			if err != nil {
 				return err
 			}
 
-			c.emit(node, OpConstant, c.addConstant(userMod))
+			switch v := v.(type) {
+			case []byte: // module written in Tengo
+				compiled, err := c.compileModule(node, node.ModuleName, node.ModuleName, v)
+				if err != nil {
+					return err
+				}
+				c.emit(node, OpConstant, c.addConstant(compiled))
+				c.emit(node, OpCall, 0)
+			case objects.Object: // builtin module
+				c.emit(node, OpConstant, c.addConstant(v))
+			default:
+				panic(fmt.Errorf("invalid import value type: %T", v))
+			}
+		} else if c.allowFileImport {
+			moduleName := node.ModuleName
+			if !strings.HasSuffix(moduleName, ".tengo") {
+				moduleName += ".tengo"
+			}
+
+			modulePath, err := filepath.Abs(moduleName)
+			if err != nil {
+				return c.errorf(node, "module file path error: %s", err.Error())
+			}
+
+			if err := c.checkCyclicImports(node, modulePath); err != nil {
+				return err
+			}
+
+			moduleSrc, err := ioutil.ReadFile(moduleName)
+			if err != nil {
+				return c.errorf(node, "module file read error: %s", err.Error())
+			}
+
+			compiled, err := c.compileModule(node, moduleName, modulePath, moduleSrc)
+			if err != nil {
+				return err
+			}
+			c.emit(node, OpConstant, c.addConstant(compiled))
 			c.emit(node, OpCall, 0)
+		} else {
+			return c.errorf(node, "module '%s' not found", node.ModuleName)
 		}
 
 	case *ast.ExportStmt:
@@ -535,7 +585,7 @@ func (c *Compiler) Compile(node ast.Node) error {
 		}
 
 		c.emit(node, OpImmutable)
-		c.emit(node, OpReturnValue)
+		c.emit(node, OpReturn, 1)
 
 	case *ast.ErrorExpr:
 		if err := c.Compile(node.Expr); err != nil {
@@ -594,22 +644,28 @@ func (c *Compiler) Bytecode() *Bytecode {
 	}
 }
 
-// SetModuleLoader sets or replaces the current module loader.
-// Note that the module loader is used for user modules,
-// not for the standard modules.
-func (c *Compiler) SetModuleLoader(moduleLoader ModuleLoader) {
-	c.moduleLoader = moduleLoader
+// EnableFileImport enables or disables module loading from local files.
+// Local file modules are disabled by default.
+func (c *Compiler) EnableFileImport(enable bool) {
+	c.allowFileImport = enable
 }
 
-func (c *Compiler) fork(file *source.File, moduleName string, symbolTable *SymbolTable) *Compiler {
-	child := NewCompiler(file, symbolTable, nil, c.builtinModules, c.trace)
-	child.moduleName = moduleName       // name of the module to compile
-	child.parent = c                    // parent to set to current compiler
-	child.moduleLoader = c.moduleLoader // share module loader
+func (c *Compiler) fork(file *source.File, modulePath string, symbolTable *SymbolTable) *Compiler {
+	child := NewCompiler(file, symbolTable, nil, c.modules, c.trace)
+	child.modulePath = modulePath // module file path
+	child.parent = c              // parent to set to current compiler
 
 	return child
 }
 
+func (c *Compiler) error(node ast.Node, err error) error {
+	return &Error{
+		fileSet: c.file.Set(),
+		node:    node,
+		error:   err,
+	}
+}
+
 func (c *Compiler) errorf(node ast.Node, format string, args ...interface{}) error {
 	return &Error{
 		fileSet: c.file.Set(),
@@ -641,33 +697,6 @@ func (c *Compiler) addInstruction(b []byte) int {
 	return posNewIns
 }
 
-func (c *Compiler) setLastInstruction(op Opcode, pos int) {
-	c.scopes[c.scopeIndex].lastInstructions[1] = c.scopes[c.scopeIndex].lastInstructions[0]
-
-	c.scopes[c.scopeIndex].lastInstructions[0].Opcode = op
-	c.scopes[c.scopeIndex].lastInstructions[0].Position = pos
-}
-
-func (c *Compiler) lastInstructionIs(op Opcode) bool {
-	if len(c.currentInstructions()) == 0 {
-		return false
-	}
-
-	return c.scopes[c.scopeIndex].lastInstructions[0].Opcode == op
-}
-
-func (c *Compiler) removeLastInstruction() {
-	lastPos := c.scopes[c.scopeIndex].lastInstructions[0].Position
-
-	if c.trace != nil {
-		c.printTrace(fmt.Sprintf("DELET %s",
-			FormatInstructions(c.scopes[c.scopeIndex].instructions[lastPos:], lastPos)[0]))
-	}
-
-	c.scopes[c.scopeIndex].instructions = c.currentInstructions()[:lastPos]
-	c.scopes[c.scopeIndex].lastInstructions[0] = c.scopes[c.scopeIndex].lastInstructions[1]
-}
-
 func (c *Compiler) replaceInstruction(pos int, inst []byte) {
 	copy(c.currentInstructions()[pos:], inst)
 
@@ -684,6 +713,92 @@ func (c *Compiler) changeOperand(opPos int, operand ...int) {
 	c.replaceInstruction(opPos, inst)
 }
 
+// optimizeFunc performs some code-level optimization for the current function instructions
+// it removes unreachable (dead code) instructions and adds "returns" instruction if needed.
+func (c *Compiler) optimizeFunc(node ast.Node) {
+	// any instructions between RETURN and the function end
+	// or instructions between RETURN and jump target position
+	// are considered as unreachable.
+
+	// pass 1. identify all jump destinations
+	dsts := make(map[int]bool)
+	iterateInstructions(c.scopes[c.scopeIndex].instructions, func(pos int, opcode Opcode, operands []int) bool {
+		switch opcode {
+		case OpJump, OpJumpFalsy, OpAndJump, OpOrJump:
+			dsts[operands[0]] = true
+		}
+
+		return true
+	})
+
+	var newInsts []byte
+
+	// pass 2. eliminate dead code
+	posMap := make(map[int]int) // old position to new position
+	var dstIdx int
+	var deadCode bool
+	iterateInstructions(c.scopes[c.scopeIndex].instructions, func(pos int, opcode Opcode, operands []int) bool {
+		switch {
+		case opcode == OpReturn:
+			if deadCode {
+				return true
+			}
+			deadCode = true
+		case dsts[pos]:
+			dstIdx++
+			deadCode = false
+		case deadCode:
+			return true
+		}
+
+		posMap[pos] = len(newInsts)
+		newInsts = append(newInsts, MakeInstruction(opcode, operands...)...)
+		return true
+	})
+
+	// pass 3. update jump positions
+	var lastOp Opcode
+	var appendReturn bool
+	endPos := len(c.scopes[c.scopeIndex].instructions)
+	iterateInstructions(newInsts, func(pos int, opcode Opcode, operands []int) bool {
+		switch opcode {
+		case OpJump, OpJumpFalsy, OpAndJump, OpOrJump:
+			newDst, ok := posMap[operands[0]]
+			if ok {
+				copy(newInsts[pos:], MakeInstruction(opcode, newDst))
+			} else if endPos == operands[0] {
+				// there's a jump instruction that jumps to the end of function
+				// compiler should append "return".
+				appendReturn = true
+			} else {
+				panic(fmt.Errorf("invalid jump position: %d", newDst))
+			}
+		}
+		lastOp = opcode
+		return true
+	})
+	if lastOp != OpReturn {
+		appendReturn = true
+	}
+
+	// pass 4. update source map
+	newSourceMap := make(map[int]source.Pos)
+	for pos, srcPos := range c.scopes[c.scopeIndex].sourceMap {
+		newPos, ok := posMap[pos]
+		if ok {
+			newSourceMap[newPos] = srcPos
+		}
+	}
+
+	c.scopes[c.scopeIndex].instructions = newInsts
+	c.scopes[c.scopeIndex].sourceMap = newSourceMap
+
+	// append "return"
+	if appendReturn {
+		c.emit(node, OpReturn, 0)
+	}
+}
+
 func (c *Compiler) emit(node ast.Node, opcode Opcode, operands ...int) int {
 	filePos := source.NoPos
 	if node != nil {
@@ -693,7 +808,6 @@ func (c *Compiler) emit(node ast.Node, opcode Opcode, operands ...int) int {
 	inst := MakeInstruction(opcode, operands...)
 	pos := c.addInstruction(inst)
 	c.scopes[c.scopeIndex].sourceMap[pos] = filePos
-	c.setLastInstruction(opcode, pos)
 
 	if c.trace != nil {
 		c.printTrace(fmt.Sprintf("EMIT  %s",

vendor/github.com/d5/tengo/compiler/compiler_assign.go

@@ -51,27 +51,27 @@ func (c *Compiler) compileAssign(node ast.Node, lhs, rhs []ast.Expr, op token.To
 
 	switch op {
 	case token.AddAssign:
-		c.emit(node, OpAdd)
+		c.emit(node, OpBinaryOp, int(token.Add))
 	case token.SubAssign:
-		c.emit(node, OpSub)
+		c.emit(node, OpBinaryOp, int(token.Sub))
 	case token.MulAssign:
-		c.emit(node, OpMul)
+		c.emit(node, OpBinaryOp, int(token.Mul))
 	case token.QuoAssign:
-		c.emit(node, OpDiv)
+		c.emit(node, OpBinaryOp, int(token.Quo))
 	case token.RemAssign:
-		c.emit(node, OpRem)
+		c.emit(node, OpBinaryOp, int(token.Rem))
 	case token.AndAssign:
-		c.emit(node, OpBAnd)
+		c.emit(node, OpBinaryOp, int(token.And))
 	case token.OrAssign:
-		c.emit(node, OpBOr)
+		c.emit(node, OpBinaryOp, int(token.Or))
 	case token.AndNotAssign:
-		c.emit(node, OpBAndNot)
+		c.emit(node, OpBinaryOp, int(token.AndNot))
 	case token.XorAssign:
-		c.emit(node, OpBXor)
+		c.emit(node, OpBinaryOp, int(token.Xor))
 	case token.ShlAssign:
-		c.emit(node, OpBShiftLeft)
+		c.emit(node, OpBinaryOp, int(token.Shl))
 	case token.ShrAssign:
-		c.emit(node, OpBShiftRight)
+		c.emit(node, OpBinaryOp, int(token.Shr))
 	}
 
 	// compile selector expressions (right to left)

vendor/github.com/d5/tengo/compiler/compiler_module.go

@@ -1,72 +1,31 @@
 package compiler
 
 import (
-	"io/ioutil"
-	"strings"
-
 	"github.com/d5/tengo/compiler/ast"
 	"github.com/d5/tengo/compiler/parser"
 	"github.com/d5/tengo/objects"
 )
 
-func (c *Compiler) compileModule(expr *ast.ImportExpr) (*objects.CompiledFunction, error) {
-	compiledModule, exists := c.loadCompiledModule(expr.ModuleName)
-	if exists {
-		return compiledModule, nil
-	}
-
-	moduleName := expr.ModuleName
-
-	// read module source from loader
-	var moduleSrc []byte
-	if c.moduleLoader == nil {
-		// default loader: read from local file
-		if !strings.HasSuffix(moduleName, ".tengo") {
-			moduleName += ".tengo"
-		}
-
-		if err := c.checkCyclicImports(expr, moduleName); err != nil {
-			return nil, err
-		}
-
-		var err error
-		moduleSrc, err = ioutil.ReadFile(moduleName)
-		if err != nil {
-			return nil, c.errorf(expr, "module file read error: %s", err.Error())
-		}
-	} else {
-		if err := c.checkCyclicImports(expr, moduleName); err != nil {
-			return nil, err
-		}
-
-		var err error
-		moduleSrc, err = c.moduleLoader(moduleName)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	compiledModule, err := c.doCompileModule(moduleName, moduleSrc)
-	if err != nil {
-		return nil, err
-	}
-
-	c.storeCompiledModule(moduleName, compiledModule)
-
-	return compiledModule, nil
-}
-
-func (c *Compiler) checkCyclicImports(node ast.Node, moduleName string) error {
-	if c.moduleName == moduleName {
-		return c.errorf(node, "cyclic module import: %s", moduleName)
+func (c *Compiler) checkCyclicImports(node ast.Node, modulePath string) error {
+	if c.modulePath == modulePath {
+		return c.errorf(node, "cyclic module import: %s", modulePath)
 	} else if c.parent != nil {
-		return c.parent.checkCyclicImports(node, moduleName)
+		return c.parent.checkCyclicImports(node, modulePath)
 	}
 
 	return nil
 }
 
-func (c *Compiler) doCompileModule(moduleName string, src []byte) (*objects.CompiledFunction, error) {
+func (c *Compiler) compileModule(node ast.Node, moduleName, modulePath string, src []byte) (*objects.CompiledFunction, error) {
+	if err := c.checkCyclicImports(node, modulePath); err != nil {
+		return nil, err
+	}
+
+	compiledModule, exists := c.loadCompiledModule(modulePath)
+	if exists {
+		return compiledModule, nil
+	}
+
 	modFile := c.file.Set().AddFile(moduleName, -1, len(src))
 	p := parser.NewParser(modFile, src, nil)
 	file, err := p.ParseFile()
@@ -77,47 +36,44 @@ func (c *Compiler) doCompileModule(moduleName string, src []byte) (*objects.Comp
 	symbolTable := NewSymbolTable()
 
 	// inherit builtin functions
-	for idx, fn := range objects.Builtins {
-		s, _, ok := c.symbolTable.Resolve(fn.Name)
-		if ok && s.Scope == ScopeBuiltin {
-			symbolTable.DefineBuiltin(idx, fn.Name)
-		}
+	for _, sym := range c.symbolTable.BuiltinSymbols() {
+		symbolTable.DefineBuiltin(sym.Index, sym.Name)
 	}
 
 	// no global scope for the module
 	symbolTable = symbolTable.Fork(false)
 
 	// compile module
-	moduleCompiler := c.fork(modFile, moduleName, symbolTable)
+	moduleCompiler := c.fork(modFile, modulePath, symbolTable)
 	if err := moduleCompiler.Compile(file); err != nil {
 		return nil, err
 	}
 
-	// add OpReturn (== export undefined) if export is missing
-	if !moduleCompiler.lastInstructionIs(OpReturnValue) {
-		moduleCompiler.emit(nil, OpReturn)
-	}
+	// code optimization
+	moduleCompiler.optimizeFunc(node)
 
 	compiledFunc := moduleCompiler.Bytecode().MainFunction
 	compiledFunc.NumLocals = symbolTable.MaxSymbols()
 
+	c.storeCompiledModule(modulePath, compiledFunc)
+
 	return compiledFunc, nil
 }
 
-func (c *Compiler) loadCompiledModule(moduleName string) (mod *objects.CompiledFunction, ok bool) {
+func (c *Compiler) loadCompiledModule(modulePath string) (mod *objects.CompiledFunction, ok bool) {
 	if c.parent != nil {
-		return c.parent.loadCompiledModule(moduleName)
+		return c.parent.loadCompiledModule(modulePath)
 	}
 
-	mod, ok = c.compiledModules[moduleName]
+	mod, ok = c.compiledModules[modulePath]
 
 	return
 }
 
-func (c *Compiler) storeCompiledModule(moduleName string, module *objects.CompiledFunction) {
+func (c *Compiler) storeCompiledModule(modulePath string, module *objects.CompiledFunction) {
 	if c.parent != nil {
-		c.parent.storeCompiledModule(moduleName, module)
+		c.parent.storeCompiledModule(modulePath, module)
 	}
 
-	c.compiledModules[moduleName] = module
+	c.compiledModules[modulePath] = module
 }

vendor/github.com/d5/tengo/compiler/instructions.go

@@ -57,3 +57,16 @@ func FormatInstructions(b []byte, posOffset int) []string {
 
 	return out
 }
+
+func iterateInstructions(b []byte, fn func(pos int, opcode Opcode, operands []int) bool) {
+	for i := 0; i < len(b); i++ {
+		numOperands := OpcodeOperands[Opcode(b[i])]
+		operands, read := ReadOperands(numOperands, b[i+1:])
+
+		if !fn(i, b[i], operands) {
+			break
+		}
+
+		i += read
+	}
+}

vendor/github.com/d5/tengo/compiler/opcodes.go

@@ -5,173 +5,137 @@ type Opcode = byte
 
 // List of opcodes
 const (
-	OpConstant         Opcode = iota // Load constant
-	OpAdd                            // Add
-	OpSub                            // Sub
-	OpMul                            // Multiply
-	OpDiv                            // Divide
-	OpRem                            // Remainder
-	OpBAnd                           // bitwise AND
-	OpBOr                            // bitwise OR
-	OpBXor                           // bitwise XOR
-	OpBShiftLeft                     // bitwise shift left
-	OpBShiftRight                    // bitwise shift right
-	OpBAndNot                        // bitwise AND NOT
-	OpBComplement                    // bitwise complement
-	OpPop                            // Pop
-	OpTrue                           // Push true
-	OpFalse                          // Push false
-	OpEqual                          // Equal ==
-	OpNotEqual                       // Not equal !=
-	OpGreaterThan                    // Greater than >=
-	OpGreaterThanEqual               // Greater than or equal to >=
-	OpMinus                          // Minus -
-	OpLNot                           // Logical not !
-	OpJumpFalsy                      // Jump if falsy
-	OpAndJump                        // Logical AND jump
-	OpOrJump                         // Logical OR jump
-	OpJump                           // Jump
-	OpNull                           // Push null
-	OpArray                          // Array object
-	OpMap                            // Map object
-	OpError                          // Error object
-	OpImmutable                      // Immutable object
-	OpIndex                          // Index operation
-	OpSliceIndex                     // Slice operation
-	OpCall                           // Call function
-	OpReturn                         // Return
-	OpReturnValue                    // Return value
-	OpGetGlobal                      // Get global variable
-	OpSetGlobal                      // Set global variable
-	OpSetSelGlobal                   // Set global variable using selectors
-	OpGetLocal                       // Get local variable
-	OpSetLocal                       // Set local variable
-	OpDefineLocal                    // Define local variable
-	OpSetSelLocal                    // Set local variable using selectors
-	OpGetFree                        // Get free variables
-	OpSetFree                        // Set free variables
-	OpSetSelFree                     // Set free variables using selectors
-	OpGetBuiltin                     // Get builtin function
-	OpGetBuiltinModule               // Get builtin module
-	OpClosure                        // Push closure
-	OpIteratorInit                   // Iterator init
-	OpIteratorNext                   // Iterator next
-	OpIteratorKey                    // Iterator key
-	OpIteratorValue                  // Iterator value
+	OpConstant      Opcode = iota // Load constant
+	OpBComplement                 // bitwise complement
+	OpPop                         // Pop
+	OpTrue                        // Push true
+	OpFalse                       // Push false
+	OpEqual                       // Equal ==
+	OpNotEqual                    // Not equal !=
+	OpMinus                       // Minus -
+	OpLNot                        // Logical not !
+	OpJumpFalsy                   // Jump if falsy
+	OpAndJump                     // Logical AND jump
+	OpOrJump                      // Logical OR jump
+	OpJump                        // Jump
+	OpNull                        // Push null
+	OpArray                       // Array object
+	OpMap                         // Map object
+	OpError                       // Error object
+	OpImmutable                   // Immutable object
+	OpIndex                       // Index operation
+	OpSliceIndex                  // Slice operation
+	OpCall                        // Call function
+	OpReturn                      // Return
+	OpGetGlobal                   // Get global variable
+	OpSetGlobal                   // Set global variable
+	OpSetSelGlobal                // Set global variable using selectors
+	OpGetLocal                    // Get local variable
+	OpSetLocal                    // Set local variable
+	OpDefineLocal                 // Define local variable
+	OpSetSelLocal                 // Set local variable using selectors
+	OpGetFreePtr                  // Get free variable pointer object
+	OpGetFree                     // Get free variables
+	OpSetFree                     // Set free variables
+	OpGetLocalPtr                 // Get local variable as a pointer
+	OpSetSelFree                  // Set free variables using selectors
+	OpGetBuiltin                  // Get builtin function
+	OpClosure                     // Push closure
+	OpIteratorInit                // Iterator init
+	OpIteratorNext                // Iterator next
+	OpIteratorKey                 // Iterator key
+	OpIteratorValue               // Iterator value
+	OpBinaryOp                    // Binary Operation
 )
 
 // OpcodeNames is opcode names.
 var OpcodeNames = [...]string{
-	OpConstant:         "CONST",
-	OpPop:              "POP",
-	OpTrue:             "TRUE",
-	OpFalse:            "FALSE",
-	OpAdd:              "ADD",
-	OpSub:              "SUB",
-	OpMul:              "MUL",
-	OpDiv:              "DIV",
-	OpRem:              "REM",
-	OpBAnd:             "AND",
-	OpBOr:              "OR",
-	OpBXor:             "XOR",
-	OpBAndNot:          "ANDN",
-	OpBShiftLeft:       "SHL",
-	OpBShiftRight:      "SHR",
-	OpBComplement:      "NEG",
-	OpEqual:            "EQL",
-	OpNotEqual:         "NEQ",
-	OpGreaterThan:      "GTR",
-	OpGreaterThanEqual: "GEQ",
-	OpMinus:            "NEG",
-	OpLNot:             "NOT",
-	OpJumpFalsy:        "JMPF",
-	OpAndJump:          "ANDJMP",
-	OpOrJump:           "ORJMP",
-	OpJump:             "JMP",
-	OpNull:             "NULL",
-	OpGetGlobal:        "GETG",
-	OpSetGlobal:        "SETG",
-	OpSetSelGlobal:     "SETSG",
-	OpArray:            "ARR",
-	OpMap:              "MAP",
-	OpError:            "ERROR",
-	OpImmutable:        "IMMUT",
-	OpIndex:            "INDEX",
-	OpSliceIndex:       "SLICE",
-	OpCall:             "CALL",
-	OpReturn:           "RET",
-	OpReturnValue:      "RETVAL",
-	OpGetLocal:         "GETL",
-	OpSetLocal:         "SETL",
-	OpDefineLocal:      "DEFL",
-	OpSetSelLocal:      "SETSL",
-	OpGetBuiltin:       "BUILTIN",
-	OpGetBuiltinModule: "BLTMOD",
-	OpClosure:          "CLOSURE",
-	OpGetFree:          "GETF",
-	OpSetFree:          "SETF",
-	OpSetSelFree:       "SETSF",
-	OpIteratorInit:     "ITER",
-	OpIteratorNext:     "ITNXT",
-	OpIteratorKey:      "ITKEY",
-	OpIteratorValue:    "ITVAL",
+	OpConstant:      "CONST",
+	OpPop:           "POP",
+	OpTrue:          "TRUE",
+	OpFalse:         "FALSE",
+	OpBComplement:   "NEG",
+	OpEqual:         "EQL",
+	OpNotEqual:      "NEQ",
+	OpMinus:         "NEG",
+	OpLNot:          "NOT",
+	OpJumpFalsy:     "JMPF",
+	OpAndJump:       "ANDJMP",
+	OpOrJump:        "ORJMP",
+	OpJump:          "JMP",
+	OpNull:          "NULL",
+	OpGetGlobal:     "GETG",
+	OpSetGlobal:     "SETG",
+	OpSetSelGlobal:  "SETSG",
+	OpArray:         "ARR",
+	OpMap:           "MAP",
+	OpError:         "ERROR",
+	OpImmutable:     "IMMUT",
+	OpIndex:         "INDEX",
+	OpSliceIndex:    "SLICE",
+	OpCall:          "CALL",
+	OpReturn:        "RET",
+	OpGetLocal:      "GETL",
+	OpSetLocal:      "SETL",
+	OpDefineLocal:   "DEFL",
+	OpSetSelLocal:   "SETSL",
+	OpGetBuiltin:    "BUILTIN",
+	OpClosure:       "CLOSURE",
+	OpGetFreePtr:    "GETFP",
+	OpGetFree:       "GETF",
+	OpSetFree:       "SETF",
+	OpGetLocalPtr:   "GETLP",
+	OpSetSelFree:    "SETSF",
+	OpIteratorInit:  "ITER",
+	OpIteratorNext:  "ITNXT",
+	OpIteratorKey:   "ITKEY",
+	OpIteratorValue: "ITVAL",
+	OpBinaryOp:      "BINARYOP",
 }
 
 // OpcodeOperands is the number of operands.
 var OpcodeOperands = [...][]int{
-	OpConstant:         {2},
-	OpPop:              {},
-	OpTrue:             {},
-	OpFalse:            {},
-	OpAdd:              {},
-	OpSub:              {},
-	OpMul:              {},
-	OpDiv:              {},
-	OpRem:              {},
-	OpBAnd:             {},
-	OpBOr:              {},
-	OpBXor:             {},
-	OpBAndNot:          {},
-	OpBShiftLeft:       {},
-	OpBShiftRight:      {},
-	OpBComplement:      {},
-	OpEqual:            {},
-	OpNotEqual:         {},
-	OpGreaterThan:      {},
-	OpGreaterThanEqual: {},
-	OpMinus:            {},
-	OpLNot:             {},
-	OpJumpFalsy:        {2},
-	OpAndJump:          {2},
-	OpOrJump:           {2},
-	OpJump:             {2},
-	OpNull:             {},
-	OpGetGlobal:        {2},
-	OpSetGlobal:        {2},
-	OpSetSelGlobal:     {2, 1},
-	OpArray:            {2},
-	OpMap:              {2},
-	OpError:            {},
-	OpImmutable:        {},
-	OpIndex:            {},
-	OpSliceIndex:       {},
-	OpCall:             {1},
-	OpReturn:           {},
-	OpReturnValue:      {},
-	OpGetLocal:         {1},
-	OpSetLocal:         {1},
-	OpDefineLocal:      {1},
-	OpSetSelLocal:      {1, 1},
-	OpGetBuiltin:       {1},
-	OpGetBuiltinModule: {},
-	OpClosure:          {2, 1},
-	OpGetFree:          {1},
-	OpSetFree:          {1},
-	OpSetSelFree:       {1, 1},
-	OpIteratorInit:     {},
-	OpIteratorNext:     {},
-	OpIteratorKey:      {},
-	OpIteratorValue:    {},
+	OpConstant:      {2},
+	OpPop:           {},
+	OpTrue:          {},
+	OpFalse:         {},
+	OpBComplement:   {},
+	OpEqual:         {},
+	OpNotEqual:      {},
+	OpMinus:         {},
+	OpLNot:          {},
+	OpJumpFalsy:     {2},
+	OpAndJump:       {2},
+	OpOrJump:        {2},
+	OpJump:          {2},
+	OpNull:          {},
+	OpGetGlobal:     {2},
+	OpSetGlobal:     {2},
+	OpSetSelGlobal:  {2, 1},
+	OpArray:         {2},
+	OpMap:           {2},
+	OpError:         {},
+	OpImmutable:     {},
+	OpIndex:         {},
+	OpSliceIndex:    {},
+	OpCall:          {1},
+	OpReturn:        {1},
+	OpGetLocal:      {1},
+	OpSetLocal:      {1},
+	OpDefineLocal:   {1},
+	OpSetSelLocal:   {1, 1},
+	OpGetBuiltin:    {1},
+	OpClosure:       {2, 1},
+	OpGetFreePtr:    {1},
+	OpGetFree:       {1},
+	OpSetFree:       {1},
+	OpGetLocalPtr:   {1},
+	OpSetSelFree:    {1, 1},
+	OpIteratorInit:  {},
+	OpIteratorNext:  {},
+	OpIteratorKey:   {},
+	OpIteratorValue: {},
+	OpBinaryOp:      {1},
 }
 
 // ReadOperands reads operands from the bytecode.

vendor/github.com/d5/tengo/compiler/parser/parse_file.go

@@ -1,28 +0,0 @@
-package parser
-
-import (
-	"io"
-
-	"github.com/d5/tengo/compiler/ast"
-	"github.com/d5/tengo/compiler/source"
-)
-
-// ParseFile parses a file with a given src.
-func ParseFile(file *source.File, src []byte, trace io.Writer) (res *ast.File, err error) {
-	p := NewParser(file, src, trace)
-
-	defer func() {
-		if e := recover(); e != nil {
-			if _, ok := e.(bailout); !ok {
-				panic(e)
-			}
-		}
-
-		p.errors.Sort()
-		err = p.errors.Err()
-	}()
-
-	res, err = p.ParseFile()
-
-	return
-}

vendor/github.com/d5/tengo/compiler/parser/parse_source.go

@@ -12,5 +12,6 @@ func ParseSource(filename string, src []byte, trace io.Writer) (res *ast.File, e
 	fileSet := source.NewFileSet()
 	file := fileSet.AddFile(filename, -1, len(src))
 
-	return ParseFile(file, src, trace)
+	p := NewParser(file, src, trace)
+	return p.ParseFile()
 }

vendor/github.com/d5/tengo/compiler/parser/parser.go

@@ -57,7 +57,18 @@ func NewParser(file *source.File, src []byte, trace io.Writer) *Parser {
 }
 
 // ParseFile parses the source and returns an AST file unit.
-func (p *Parser) ParseFile() (*ast.File, error) {
+func (p *Parser) ParseFile() (file *ast.File, err error) {
+	defer func() {
+		if e := recover(); e != nil {
+			if _, ok := e.(bailout); !ok {
+				panic(e)
+			}
+		}
+
+		p.errors.Sort()
+		err = p.errors.Err()
+	}()
+
 	if p.trace {
 		defer un(trace(p, "File"))
 	}
@@ -71,10 +82,12 @@ func (p *Parser) ParseFile() (*ast.File, error) {
 		return nil, p.errors.Err()
 	}
 
-	return &ast.File{
+	file = &ast.File{
 		InputFile: p.file,
 		Stmts:     stmts,
-	}, nil
+	}
+
+	return
 }
 
 func (p *Parser) parseExpr() ast.Expr {
@@ -1002,16 +1015,26 @@ func (p *Parser) parseMapElementLit() *ast.MapElementLit {
 		defer un(trace(p, "MapElementLit"))
 	}
 
-	// key: read identifier token but it's not actually an identifier
-	ident := p.parseIdent()
+	pos := p.pos
+	name := "_"
+
+	if p.token == token.Ident {
+		name = p.tokenLit
+	} else if p.token == token.String {
+		v, _ := strconv.Unquote(p.tokenLit)
+		name = v
+	} else {
+		p.errorExpected(pos, "map key")
+	}
+
+	p.next()
 
 	colonPos := p.expect(token.Colon)
-
 	valueExpr := p.parseExpr()
 
 	return &ast.MapElementLit{
-		Key:      ident.Name,
-		KeyPos:   ident.NamePos,
+		Key:      name,
+		KeyPos:   pos,
 		ColonPos: colonPos,
 		Value:    valueExpr,
 	}

vendor/github.com/d5/tengo/compiler/symbol_table.go

@@ -2,12 +2,13 @@ package compiler
 
 // SymbolTable represents a symbol table.
 type SymbolTable struct {
-	parent        *SymbolTable
-	block         bool
-	store         map[string]*Symbol
-	numDefinition int
-	maxDefinition int
-	freeSymbols   []*Symbol
+	parent         *SymbolTable
+	block          bool
+	store          map[string]*Symbol
+	numDefinition  int
+	maxDefinition  int
+	freeSymbols    []*Symbol
+	builtinSymbols []*Symbol
 }
 
 // NewSymbolTable creates a SymbolTable.
@@ -37,6 +38,10 @@ func (t *SymbolTable) Define(name string) *Symbol {
 
 // DefineBuiltin adds a symbol for builtin function.
 func (t *SymbolTable) DefineBuiltin(index int, name string) *Symbol {
+	if t.parent != nil {
+		return t.parent.DefineBuiltin(index, name)
+	}
+
 	symbol := &Symbol{
 		Name:  name,
 		Index: index,
@@ -45,6 +50,8 @@ func (t *SymbolTable) DefineBuiltin(index int, name string) *Symbol {
 
 	t.store[name] = symbol
 
+	t.builtinSymbols = append(t.builtinSymbols, symbol)
+
 	return symbol
 }
 
@@ -57,9 +64,7 @@ func (t *SymbolTable) Resolve(name string) (symbol *Symbol, depth int, ok bool)
 			return
 		}
 
-		if !t.block {
-			depth++
-		}
+		depth++
 
 		// if symbol is defined in parent table and if it's not global/builtin
 		// then it's free variable.
@@ -101,6 +106,15 @@ func (t *SymbolTable) FreeSymbols() []*Symbol {
 	return t.freeSymbols
 }
 
+// BuiltinSymbols returns builtin symbols for the scope.
+func (t *SymbolTable) BuiltinSymbols() []*Symbol {
+	if t.parent != nil {
+		return t.parent.BuiltinSymbols()
+	}
+
+	return t.builtinSymbols
+}
+
 // Names returns the name of all the symbols.
 func (t *SymbolTable) Names() []string {
 	var names []string

vendor/github.com/d5/tengo/objects/break.go

@@ -1,37 +0,0 @@
-package objects
-
-import "github.com/d5/tengo/compiler/token"
-
-// Break represents a break statement.
-type Break struct{}
-
-// TypeName returns the name of the type.
-func (o *Break) TypeName() string {
-	return "break"
-}
-
-func (o *Break) String() string {
-	return "<break>"
-}
-
-// BinaryOp returns another object that is the result of
-// a given binary operator and a right-hand side object.
-func (o *Break) BinaryOp(op token.Token, rhs Object) (Object, error) {
-	return nil, ErrInvalidOperator
-}
-
-// Copy returns a copy of the type.
-func (o *Break) Copy() Object {
-	return &Break{}
-}
-
-// IsFalsy returns true if the value of the type is falsy.
-func (o *Break) IsFalsy() bool {
-	return false
-}
-
-// Equals returns true if the value of the type
-// is equal to the value of another object.
-func (o *Break) Equals(x Object) bool {
-	return false
-}

vendor/github.com/d5/tengo/objects/builtin_convert.go

@@ -1,5 +1,7 @@
 package objects
 
+import "github.com/d5/tengo"
+
 func builtinString(args ...Object) (Object, error) {
 	argsLen := len(args)
 	if !(argsLen == 1 || argsLen == 2) {
@@ -12,6 +14,10 @@ func builtinString(args ...Object) (Object, error) {
 
 	v, ok := ToString(args[0])
 	if ok {
+		if len(v) > tengo.MaxStringLen {
+			return nil, ErrStringLimit
+		}
+
 		return &String{Value: v}, nil
 	}
 
@@ -117,11 +123,19 @@ func builtinBytes(args ...Object) (Object, error) {
 
 	// bytes(N) => create a new bytes with given size N
 	if n, ok := args[0].(*Int); ok {
+		if n.Value > int64(tengo.MaxBytesLen) {
+			return nil, ErrBytesLimit
+		}
+
 		return &Bytes{Value: make([]byte, int(n.Value))}, nil
 	}
 
 	v, ok := ToByteSlice(args[0])
 	if ok {
+		if len(v) > tengo.MaxBytesLen {
+			return nil, ErrBytesLimit
+		}
+
 		return &Bytes{Value: v}, nil
 	}
 

vendor/github.com/d5/tengo/objects/builtin_json.go

@@ -1,54 +0,0 @@
-package objects
-
-import (
-	"encoding/json"
-)
-
-// to_json(v object) => bytes
-func builtinToJSON(args ...Object) (Object, error) {
-	if len(args) != 1 {
-		return nil, ErrWrongNumArguments
-	}
-
-	res, err := json.Marshal(objectToInterface(args[0]))
-	if err != nil {
-		return &Error{Value: &String{Value: err.Error()}}, nil
-	}
-
-	return &Bytes{Value: res}, nil
-}
-
-// from_json(data string/bytes) => object
-func builtinFromJSON(args ...Object) (Object, error) {
-	if len(args) != 1 {
-		return nil, ErrWrongNumArguments
-	}
-
-	var target interface{}
-
-	switch o := args[0].(type) {
-	case *Bytes:
-		err := json.Unmarshal(o.Value, &target)
-		if err != nil {
-			return &Error{Value: &String{Value: err.Error()}}, nil
-		}
-	case *String:
-		err := json.Unmarshal([]byte(o.Value), &target)
-		if err != nil {
-			return &Error{Value: &String{Value: err.Error()}}, nil
-		}
-	default:
-		return nil, ErrInvalidArgumentType{
-			Name:     "first",
-			Expected: "bytes/string",
-			Found:    args[0].TypeName(),
-		}
-	}
-
-	res, err := FromInterface(target)
-	if err != nil {
-		return nil, err
-	}
-
-	return res, nil
-}

vendor/github.com/d5/tengo/objects/builtin_module.go

@@ -0,0 +1,23 @@
+package objects
+
+// BuiltinModule is an importable module that's written in Go.
+type BuiltinModule struct {
+	Attrs map[string]Object
+}
+
+// Import returns an immutable map for the module.
+func (m *BuiltinModule) Import(moduleName string) (interface{}, error) {
+	return m.AsImmutableMap(moduleName), nil
+}
+
+// AsImmutableMap converts builtin module into an immutable map.
+func (m *BuiltinModule) AsImmutableMap(moduleName string) *ImmutableMap {
+	attrs := make(map[string]Object, len(m.Attrs))
+	for k, v := range m.Attrs {
+		attrs[k] = v.Copy()
+	}
+
+	attrs["__module_name__"] = &String{Value: moduleName}
+
+	return &ImmutableMap{Value: attrs}
+}

vendor/github.com/d5/tengo/objects/builtin_print.go

@@ -1,75 +0,0 @@
-package objects
-
-import (
-	"fmt"
-)
-
-// print(args...)
-func builtinPrint(args ...Object) (Object, error) {
-	for _, arg := range args {
-		if str, ok := arg.(*String); ok {
-			fmt.Println(str.Value)
-		} else {
-			fmt.Println(arg.String())
-		}
-	}
-
-	return nil, nil
-}
-
-// printf("format", args...)
-func builtinPrintf(args ...Object) (Object, error) {
-	numArgs := len(args)
-	if numArgs == 0 {
-		return nil, ErrWrongNumArguments
-	}
-
-	format, ok := args[0].(*String)
-	if !ok {
-		return nil, ErrInvalidArgumentType{
-			Name:     "format",
-			Expected: "string",
-			Found:    args[0].TypeName(),
-		}
-	}
-	if numArgs == 1 {
-		fmt.Print(format)
-		return nil, nil
-	}
-
-	formatArgs := make([]interface{}, numArgs-1, numArgs-1)
-	for idx, arg := range args[1:] {
-		formatArgs[idx] = objectToInterface(arg)
-	}
-
-	fmt.Printf(format.Value, formatArgs...)
-
-	return nil, nil
-}
-
-// sprintf("format", args...)
-func builtinSprintf(args ...Object) (Object, error) {
-	numArgs := len(args)
-	if numArgs == 0 {
-		return nil, ErrWrongNumArguments
-	}
-
-	format, ok := args[0].(*String)
-	if !ok {
-		return nil, ErrInvalidArgumentType{
-			Name:     "format",
-			Expected: "string",
-			Found:    args[0].TypeName(),
-		}
-	}
-	if numArgs == 1 {
-		return format, nil // okay to return 'format' directly as String is immutable
-	}
-
-	formatArgs := make([]interface{}, numArgs-1, numArgs-1)
-	for idx, arg := range args[1:] {
-		formatArgs[idx] = objectToInterface(arg)
-	}
-
-	return &String{Value: fmt.Sprintf(format.Value, formatArgs...)}, nil
-}

vendor/github.com/d5/tengo/objects/builtin_type_checks.go

@@ -181,3 +181,15 @@ func builtinIsCallable(args ...Object) (Object, error) {
 
 	return FalseValue, nil
 }
+
+func builtinIsIterable(args ...Object) (Object, error) {
+	if len(args) != 1 {
+		return nil, ErrWrongNumArguments
+	}
+
+	if _, ok := args[0].(Iterable); ok {
+		return TrueValue, nil
+	}
+
+	return FalseValue, nil
+}

vendor/github.com/d5/tengo/objects/builtins.go

@@ -1,135 +1,114 @@
 package objects
 
-// NamedBuiltinFunc is a named builtin function.
-type NamedBuiltinFunc struct {
-	Name string
-	Func CallableFunc
-}
-
 // Builtins contains all default builtin functions.
-var Builtins = []NamedBuiltinFunc{
+// Use GetBuiltinFunctions instead of accessing Builtins directly.
+var Builtins = []*BuiltinFunction{
 	{
-		Name: "print",
-		Func: builtinPrint,
+		Name:  "len",
+		Value: builtinLen,
 	},
 	{
-		Name: "printf",
-		Func: builtinPrintf,
+		Name:  "copy",
+		Value: builtinCopy,
 	},
 	{
-		Name: "sprintf",
-		Func: builtinSprintf,
+		Name:  "append",
+		Value: builtinAppend,
 	},
 	{
-		Name: "len",
-		Func: builtinLen,
+		Name:  "string",
+		Value: builtinString,
 	},
 	{
-		Name: "copy",
-		Func: builtinCopy,
+		Name:  "int",
+		Value: builtinInt,
 	},
 	{
-		Name: "append",
-		Func: builtinAppend,
+		Name:  "bool",
+		Value: builtinBool,
 	},
 	{
-		Name: "string",
-		Func: builtinString,
+		Name:  "float",
+		Value: builtinFloat,
 	},
 	{
-		Name: "int",
-		Func: builtinInt,
+		Name:  "char",
+		Value: builtinChar,
 	},
 	{
-		Name: "bool",
-		Func: builtinBool,
+		Name:  "bytes",
+		Value: builtinBytes,
 	},
 	{
-		Name: "float",
-		Func: builtinFloat,
+		Name:  "time",
+		Value: builtinTime,
 	},
 	{
-		Name: "char",
-		Func: builtinChar,
+		Name:  "is_int",
+		Value: builtinIsInt,
 	},
 	{
-		Name: "bytes",
-		Func: builtinBytes,
+		Name:  "is_float",
+		Value: builtinIsFloat,
 	},
 	{
-		Name: "time",
-		Func: builtinTime,
+		Name:  "is_string",
+		Value: builtinIsString,
 	},
 	{
-		Name: "is_int",
-		Func: builtinIsInt,
+		Name:  "is_bool",
+		Value: builtinIsBool,
 	},
 	{
-		Name: "is_float",
-		Func: builtinIsFloat,
+		Name:  "is_char",
+		Value: builtinIsChar,
 	},
 	{
-		Name: "is_string",
-		Func: builtinIsString,
+		Name:  "is_bytes",
+		Value: builtinIsBytes,
 	},
 	{
-		Name: "is_bool",
-		Func: builtinIsBool,
+		Name:  "is_array",
+		Value: builtinIsArray,
 	},
 	{
-		Name: "is_char",
-		Func: builtinIsChar,
+		Name:  "is_immutable_array",
+		Value: builtinIsImmutableArray,
 	},
 	{
-		Name: "is_bytes",
-		Func: builtinIsBytes,
+		Name:  "is_map",
+		Value: builtinIsMap,
 	},
 	{
-		Name: "is_array",
-		Func: builtinIsArray,
+		Name:  "is_immutable_map",
+		Value: builtinIsImmutableMap,
 	},
 	{
-		Name: "is_immutable_array",
-		Func: builtinIsImmutableArray,
+		Name:  "is_iterable",
+		Value: builtinIsIterable,
 	},
 	{
-		Name: "is_map",
-		Func: builtinIsMap,
+		Name:  "is_time",
+		Value: builtinIsTime,
 	},
 	{
-		Name: "is_immutable_map",
-		Func: builtinIsImmutableMap,
+		Name:  "is_error",
+		Value: builtinIsError,
 	},
 	{
-		Name: "is_time",
-		Func: builtinIsTime,
+		Name:  "is_undefined",
+		Value: builtinIsUndefined,
 	},
 	{
-		Name: "is_error",
-		Func: builtinIsError,
+		Name:  "is_function",
+		Value: builtinIsFunction,
 	},
 	{
-		Name: "is_undefined",
-		Func: builtinIsUndefined,
+		Name:  "is_callable",
+		Value: builtinIsCallable,
 	},
 	{
-		Name: "is_function",
-		Func: builtinIsFunction,
-	},
-	{
-		Name: "is_callable",
-		Func: builtinIsCallable,
-	},
-	{
-		Name: "to_json",
-		Func: builtinToJSON,
-	},
-	{
-		Name: "from_json",
-		Func: builtinFromJSON,
-	},
-	{
-		Name: "type_name",
-		Func: builtinTypeName,
+		Name:  "type_name",
+		Value: builtinTypeName,
 	},
 }

vendor/github.com/d5/tengo/objects/bytes.go

@@ -3,6 +3,7 @@ package objects
 import (
 	"bytes"
 
+	"github.com/d5/tengo"
 	"github.com/d5/tengo/compiler/token"
 )
 
@@ -27,6 +28,10 @@ func (o *Bytes) BinaryOp(op token.Token, rhs Object) (Object, error) {
 	case token.Add:
 		switch rhs := rhs.(type) {
 		case *Bytes:
+			if len(o.Value)+len(rhs.Value) > tengo.MaxBytesLen {
+				return nil, ErrBytesLimit
+			}
+
 			return &Bytes{Value: append(o.Value, rhs.Value...)}, nil
 		}
 	}
@@ -74,3 +79,11 @@ func (o *Bytes) IndexGet(index Object) (res Object, err error) {
 
 	return
 }
+
+// Iterate creates a bytes iterator.
+func (o *Bytes) Iterate() Iterator {
+	return &BytesIterator{
+		v: o.Value,
+		l: len(o.Value),
+	}
+}

vendor/github.com/d5/tengo/objects/bytes_iterator.go

@@ -0,0 +1,57 @@
+package objects
+
+import "github.com/d5/tengo/compiler/token"
+
+// BytesIterator represents an iterator for a string.
+type BytesIterator struct {
+	v []byte
+	i int
+	l int
+}
+
+// TypeName returns the name of the type.
+func (i *BytesIterator) TypeName() string {
+	return "bytes-iterator"
+}
+
+func (i *BytesIterator) String() string {
+	return "<bytes-iterator>"
+}
+
+// BinaryOp returns another object that is the result of
+// a given binary operator and a right-hand side object.
+func (i *BytesIterator) BinaryOp(op token.Token, rhs Object) (Object, error) {
+	return nil, ErrInvalidOperator
+}
+
+// IsFalsy returns true if the value of the type is falsy.
+func (i *BytesIterator) IsFalsy() bool {
+	return true
+}
+
+// Equals returns true if the value of the type
+// is equal to the value of another object.
+func (i *BytesIterator) Equals(Object) bool {
+	return false
+}
+
+// Copy returns a copy of the type.
+func (i *BytesIterator) Copy() Object {
+	return &BytesIterator{v: i.v, i: i.i, l: i.l}
+}
+
+// Next returns true if there are more elements to iterate.
+func (i *BytesIterator) Next() bool {
+	i.i++
+	return i.i <= i.l
+}
+
+// Key returns the key or index value of the current element.
+func (i *BytesIterator) Key() Object {
+	return &Int{Value: int64(i.i - 1)}
+}
+
+// Value returns the value of the current element.
+func (i *BytesIterator) Value() Object {
+	return &Int{Value: int64(i.v[i.i-1])}
+}

vendor/github.com/d5/tengo/objects/callable_func.go

@@ -1,4 +1,4 @@
 package objects
 
 // CallableFunc is a function signature for the callable functions.
-type CallableFunc func(args ...Object) (ret Object, err error)
+type CallableFunc = func(args ...Object) (ret Object, err error)

vendor/github.com/d5/tengo/objects/closure.go

@@ -7,7 +7,7 @@ import (
 // Closure represents a function closure.
 type Closure struct {
 	Fn   *CompiledFunction
-	Free []*Object
+	Free []*ObjectPtr
 }
 
 // TypeName returns the name of the type.
@@ -29,7 +29,7 @@ func (o *Closure) BinaryOp(op token.Token, rhs Object) (Object, error) {
 func (o *Closure) Copy() Object {
 	return &Closure{
 		Fn:   o.Fn.Copy().(*CompiledFunction),
-		Free: append([]*Object{}, o.Free...), // DO NOT Copy() of elements; these are variable pointers
+		Free: append([]*ObjectPtr{}, o.Free...), // DO NOT Copy() of elements; these are variable pointers
 	}
 }
 

vendor/github.com/d5/tengo/objects/compiled_function.go

@@ -47,3 +47,14 @@ func (o *CompiledFunction) IsFalsy() bool {
 func (o *CompiledFunction) Equals(x Object) bool {
 	return false
 }
+
+// SourcePos returns the source position of the instruction at ip.
+func (o *CompiledFunction) SourcePos(ip int) source.Pos {
+	for ip >= 0 {
+		if p, ok := o.SourceMap[ip]; ok {
+			return p
+		}
+		ip--
+	}
+	return source.NoPos
+}

vendor/github.com/d5/tengo/objects/continue.go

@@ -1,38 +0,0 @@
-package objects
-
-import "github.com/d5/tengo/compiler/token"
-
-// Continue represents a continue statement.
-type Continue struct {
-}
-
-// TypeName returns the name of the type.
-func (o *Continue) TypeName() string {
-	return "continue"
-}
-
-func (o *Continue) String() string {
-	return "<continue>"
-}
-
-// BinaryOp returns another object that is the result of
-// a given binary operator and a right-hand side object.
-func (o *Continue) BinaryOp(op token.Token, rhs Object) (Object, error) {
-	return nil, ErrInvalidOperator
-}
-
-// Copy returns a copy of the type.
-func (o *Continue) Copy() Object {
-	return &Continue{}
-}
-
-// IsFalsy returns true if the value of the type is falsy.
-func (o *Continue) IsFalsy() bool {
-	return false
-}
-
-// Equals returns true if the value of the type
-// is equal to the value of another object.
-func (o *Continue) Equals(x Object) bool {
-	return false
-}

vendor/github.com/d5/tengo/objects/conversion.go

@@ -1,9 +1,12 @@
 package objects
 
 import (
+	"errors"
 	"fmt"
 	"strconv"
 	"time"
+
+	"github.com/d5/tengo"
 )
 
 // ToString will try to convert object o to string value.
@@ -156,8 +159,8 @@ func ToTime(o Object) (v time.Time, ok bool) {
 	return
 }
 
-// objectToInterface attempts to convert an object o to an interface{} value
-func objectToInterface(o Object) (res interface{}) {
+// ToInterface attempts to convert an object o to an interface{} value
+func ToInterface(o Object) (res interface{}) {
 	switch o := o.(type) {
 	case *Int:
 		res = o.Value
@@ -174,13 +177,29 @@ func objectToInterface(o Object) (res interface{}) {
 	case *Array:
 		res = make([]interface{}, len(o.Value))
 		for i, val := range o.Value {
-			res.([]interface{})[i] = objectToInterface(val)
+			res.([]interface{})[i] = ToInterface(val)
+		}
+	case *ImmutableArray:
+		res = make([]interface{}, len(o.Value))
+		for i, val := range o.Value {
+			res.([]interface{})[i] = ToInterface(val)
 		}
 	case *Map:
 		res = make(map[string]interface{})
 		for key, v := range o.Value {
-			res.(map[string]interface{})[key] = objectToInterface(v)
+			res.(map[string]interface{})[key] = ToInterface(v)
 		}
+	case *ImmutableMap:
+		res = make(map[string]interface{})
+		for key, v := range o.Value {
+			res.(map[string]interface{})[key] = ToInterface(v)
+		}
+	case *Time:
+		res = o.Value
+	case *Error:
+		res = errors.New(o.String())
+	case *Undefined:
+		res = nil
 	case Object:
 		return o
 	}
@@ -194,6 +213,9 @@ func FromInterface(v interface{}) (Object, error) {
 	case nil:
 		return UndefinedValue, nil
 	case string:
+		if len(v) > tengo.MaxStringLen {
+			return nil, ErrStringLimit
+		}
 		return &String{Value: v}, nil
 	case int64:
 		return &Int{Value: v}, nil
@@ -211,6 +233,9 @@ func FromInterface(v interface{}) (Object, error) {
 	case float64:
 		return &Float{Value: v}, nil
 	case []byte:
+		if len(v) > tengo.MaxBytesLen {
+			return nil, ErrBytesLimit
+		}
 		return &Bytes{Value: v}, nil
 	case error:
 		return &Error{Value: &String{Value: v.Error()}}, nil
@@ -243,6 +268,8 @@ func FromInterface(v interface{}) (Object, error) {
 		return &Time{Value: v}, nil
 	case Object:
 		return v, nil
+	case CallableFunc:
+		return &UserFunction{Value: v}, nil
 	}
 
 	return nil, fmt.Errorf("cannot convert to object: %T", v)

vendor/github.com/d5/tengo/objects/count_objects.go

@@ -0,0 +1,31 @@
+package objects
+
+// CountObjects returns the number of objects that a given object o contains.
+// For scalar value types, it will always be 1. For compound value types,
+// this will include its elements and all of their elements recursively.
+func CountObjects(o Object) (c int) {
+	c = 1
+
+	switch o := o.(type) {
+	case *Array:
+		for _, v := range o.Value {
+			c += CountObjects(v)
+		}
+	case *ImmutableArray:
+		for _, v := range o.Value {
+			c += CountObjects(v)
+		}
+	case *Map:
+		for _, v := range o.Value {
+			c += CountObjects(v)
+		}
+	case *ImmutableMap:
+		for _, v := range o.Value {
+			c += CountObjects(v)
+		}
+	case *Error:
+		c += CountObjects(o.Value)
+	}
+
+	return
+}

vendor/github.com/d5/tengo/objects/errors.go

@@ -20,6 +20,12 @@ var ErrInvalidOperator = errors.New("invalid operator")
 // ErrWrongNumArguments represents a wrong number of arguments error.
 var ErrWrongNumArguments = errors.New("wrong number of arguments")
 
+// ErrBytesLimit represents an error where the size of bytes value exceeds the limit.
+var ErrBytesLimit = errors.New("exceeding bytes size limit")
+
+// ErrStringLimit represents an error where the size of string value exceeds the limit.
+var ErrStringLimit = errors.New("exceeding string size limit")
+
 // ErrInvalidArgumentType represents an invalid argument value type error.
 type ErrInvalidArgumentType struct {
 	Name     string

vendor/github.com/d5/tengo/objects/importable.go

@@ -0,0 +1,7 @@
+package objects
+
+// Importable interface represents importable module instance.
+type Importable interface {
+	// Import should return either an Object or module source code ([]byte).
+	Import(moduleName string) (interface{}, error)
+}

vendor/github.com/d5/tengo/objects/module_map.go

@@ -0,0 +1,77 @@
+package objects
+
+// ModuleMap represents a set of named modules.
+// Use NewModuleMap to create a new module map.
+type ModuleMap struct {
+	m map[string]Importable
+}
+
+// NewModuleMap creates a new module map.
+func NewModuleMap() *ModuleMap {
+	return &ModuleMap{
+		m: make(map[string]Importable),
+	}
+}
+
+// Add adds an import module.
+func (m *ModuleMap) Add(name string, module Importable) {
+	m.m[name] = module
+}
+
+// AddBuiltinModule adds a builtin module.
+func (m *ModuleMap) AddBuiltinModule(name string, attrs map[string]Object) {
+	m.m[name] = &BuiltinModule{Attrs: attrs}
+}
+
+// AddSourceModule adds a source module.
+func (m *ModuleMap) AddSourceModule(name string, src []byte) {
+	m.m[name] = &SourceModule{Src: src}
+}
+
+// Remove removes a named module.
+func (m *ModuleMap) Remove(name string) {
+	delete(m.m, name)
+}
+
+// Get returns an import module identified by name.
+// It returns if the name is not found.
+func (m *ModuleMap) Get(name string) Importable {
+	return m.m[name]
+}
+
+// GetBuiltinModule returns a builtin module identified by name.
+// It returns if the name is not found or the module is not a builtin module.
+func (m *ModuleMap) GetBuiltinModule(name string) *BuiltinModule {
+	mod, _ := m.m[name].(*BuiltinModule)
+	return mod
+}
+
+// GetSourceModule returns a source module identified by name.
+// It returns if the name is not found or the module is not a source module.
+func (m *ModuleMap) GetSourceModule(name string) *SourceModule {
+	mod, _ := m.m[name].(*SourceModule)
+	return mod
+}
+
+// Copy creates a copy of the module map.
+func (m *ModuleMap) Copy() *ModuleMap {
+	c := &ModuleMap{
+		m: make(map[string]Importable),
+	}
+	for name, mod := range m.m {
+		c.m[name] = mod
+	}
+	return c
+}
+
+// Len returns the number of named modules.
+func (m *ModuleMap) Len() int {
+	return len(m.m)
+}
+
+// AddMap adds named modules from another module map.
+func (m *ModuleMap) AddMap(o *ModuleMap) {
+	for name, mod := range o.m {
+		m.m[name] = mod
+	}
+}

vendor/github.com/d5/tengo/objects/object_ptr.go

@@ -0,0 +1,41 @@
+package objects
+
+import (
+	"github.com/d5/tengo/compiler/token"
+)
+
+// ObjectPtr represents a free variable.
+type ObjectPtr struct {
+	Value *Object
+}
+
+func (o *ObjectPtr) String() string {
+	return "free-var"
+}
+
+// TypeName returns the name of the type.
+func (o *ObjectPtr) TypeName() string {
+	return "<free-var>"
+}
+
+// BinaryOp returns another object that is the result of
+// a given binary operator and a right-hand side object.
+func (o *ObjectPtr) BinaryOp(op token.Token, rhs Object) (Object, error) {
+	return nil, ErrInvalidOperator
+}
+
+// Copy returns a copy of the type.
+func (o *ObjectPtr) Copy() Object {
+	return o
+}
+
+// IsFalsy returns true if the value of the type is falsy.
+func (o *ObjectPtr) IsFalsy() bool {
+	return o.Value == nil
+}
+
+// Equals returns true if the value of the type
+// is equal to the value of another object.
+func (o *ObjectPtr) Equals(x Object) bool {
+	return o == x
+}

vendor/github.com/d5/tengo/objects/return_value.go

@@ -1,39 +0,0 @@
-package objects
-
-import "github.com/d5/tengo/compiler/token"
-
-// ReturnValue represents a value that is being returned.
-type ReturnValue struct {
-	Value Object
-}
-
-// TypeName returns the name of the type.
-func (o *ReturnValue) TypeName() string {
-	return "return-value"
-}
-
-func (o *ReturnValue) String() string {
-	return "<return-value>"
-}
-
-// BinaryOp returns another object that is the result of
-// a given binary operator and a right-hand side object.
-func (o *ReturnValue) BinaryOp(op token.Token, rhs Object) (Object, error) {
-	return nil, ErrInvalidOperator
-}
-
-// Copy returns a copy of the type.
-func (o *ReturnValue) Copy() Object {
-	return &ReturnValue{Value: o.Copy()}
-}
-
-// IsFalsy returns true if the value of the type is falsy.
-func (o *ReturnValue) IsFalsy() bool {
-	return false
-}
-
-// Equals returns true if the value of the type
-// is equal to the value of another object.
-func (o *ReturnValue) Equals(x Object) bool {
-	return false
-}

vendor/github.com/d5/tengo/objects/source_module.go

@@ -0,0 +1,11 @@
+package objects
+
+// SourceModule is an importable module that's written in Tengo.
+type SourceModule struct {
+	Src []byte
+}
+
+// Import returns a module source code.
+func (m *SourceModule) Import(_ string) (interface{}, error) {
+	return m.Src, nil
+}

vendor/github.com/d5/tengo/objects/string.go

@@ -3,6 +3,7 @@ package objects
 import (
 	"strconv"
 
+	"github.com/d5/tengo"
 	"github.com/d5/tengo/compiler/token"
 )
 
@@ -28,9 +29,16 @@ func (o *String) BinaryOp(op token.Token, rhs Object) (Object, error) {
 	case token.Add:
 		switch rhs := rhs.(type) {
 		case *String:
+			if len(o.Value)+len(rhs.Value) > tengo.MaxStringLen {
+				return nil, ErrStringLimit
+			}
 			return &String{Value: o.Value + rhs.Value}, nil
 		default:
-			return &String{Value: o.Value + rhs.String()}, nil
+			rhsStr := rhs.String()
+			if len(o.Value)+len(rhsStr) > tengo.MaxStringLen {
+				return nil, ErrStringLimit
+			}
+			return &String{Value: o.Value + rhsStr}, nil
 		}
 	}
 

vendor/github.com/d5/tengo/objects/user_function.go

@@ -6,8 +6,9 @@ import (
 
 // UserFunction represents a user function.
 type UserFunction struct {
-	Name  string
-	Value CallableFunc
+	Name       string
+	Value      CallableFunc
+	EncodingID string
 }
 
 // TypeName returns the name of the type.

vendor/github.com/d5/tengo/runtime/errors.go

@@ -6,3 +6,6 @@ import (
 
 // ErrStackOverflow is a stack overflow error.
 var ErrStackOverflow = errors.New("stack overflow")
+
+// ErrObjectAllocLimit is an objects allocation limit error.
+var ErrObjectAllocLimit = errors.New("object allocation limit exceeded")

vendor/github.com/d5/tengo/runtime/frame.go

@@ -7,7 +7,7 @@ import (
 // Frame represents a function call frame.
 type Frame struct {
 	fn          *objects.CompiledFunction
-	freeVars    []*objects.Object
+	freeVars    []*objects.ObjectPtr
 	ip          int
 	basePointer int
 }

vendor/github.com/d5/tengo/script/compiled.go

@@ -3,6 +3,7 @@ package script
 import (
 	"context"
 	"fmt"
+	"sync"
 
 	"github.com/d5/tengo/compiler"
 	"github.com/d5/tengo/objects"
@@ -12,26 +13,39 @@ import (
 // Compiled is a compiled instance of the user script.
 // Use Script.Compile() to create Compiled object.
 type Compiled struct {
-	symbolTable *compiler.SymbolTable
-	machine     *runtime.VM
+	globalIndexes map[string]int // global symbol name to index
+	bytecode      *compiler.Bytecode
+	globals       []objects.Object
+	maxAllocs     int64
+	lock          sync.RWMutex
 }
 
 // Run executes the compiled script in the virtual machine.
 func (c *Compiled) Run() error {
-	return c.machine.Run()
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	v := runtime.NewVM(c.bytecode, c.globals, c.maxAllocs)
+
+	return v.Run()
 }
 
 // RunContext is like Run but includes a context.
 func (c *Compiled) RunContext(ctx context.Context) (err error) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	v := runtime.NewVM(c.bytecode, c.globals, c.maxAllocs)
+
 	ch := make(chan error, 1)
 
 	go func() {
-		ch <- c.machine.Run()
+		ch <- v.Run()
 	}()
 
 	select {
 	case <-ctx.Done():
-		c.machine.Abort()
+		v.Abort()
 		<-ch
 		err = ctx.Err()
 	case err = <-ch:
@@ -40,30 +54,58 @@ func (c *Compiled) RunContext(ctx context.Context) (err error) {
 	return
 }
 
+// Clone creates a new copy of Compiled.
+// Cloned copies are safe for concurrent use by multiple goroutines.
+func (c *Compiled) Clone() *Compiled {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	clone := &Compiled{
+		globalIndexes: c.globalIndexes,
+		bytecode:      c.bytecode,
+		globals:       make([]objects.Object, len(c.globals)),
+		maxAllocs:     c.maxAllocs,
+	}
+
+	// copy global objects
+	for idx, g := range c.globals {
+		if g != nil {
+			clone.globals[idx] = g
+		}
+	}
+
+	return clone
+}
+
 // IsDefined returns true if the variable name is defined (has value) before or after the execution.
 func (c *Compiled) IsDefined(name string) bool {
-	symbol, _, ok := c.symbolTable.Resolve(name)
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+
+	idx, ok := c.globalIndexes[name]
 	if !ok {
 		return false
 	}
 
-	v := c.machine.Globals()[symbol.Index]
+	v := c.globals[idx]
 	if v == nil {
 		return false
 	}
 
-	return *v != objects.UndefinedValue
+	return v != objects.UndefinedValue
 }
 
 // Get returns a variable identified by the name.
 func (c *Compiled) Get(name string) *Variable {
-	value := &objects.UndefinedValue
+	c.lock.RLock()
+	defer c.lock.RUnlock()
 
-	symbol, _, ok := c.symbolTable.Resolve(name)
-	if ok && symbol.Scope == compiler.ScopeGlobal {
-		value = c.machine.Globals()[symbol.Index]
+	value := objects.UndefinedValue
+
+	if idx, ok := c.globalIndexes[name]; ok {
+		value = c.globals[idx]
 		if value == nil {
-			value = &objects.UndefinedValue
+			value = objects.UndefinedValue
 		}
 	}
 
@@ -75,20 +117,21 @@ func (c *Compiled) Get(name string) *Variable {
 
 // GetAll returns all the variables that are defined by the compiled script.
 func (c *Compiled) GetAll() []*Variable {
-	var vars []*Variable
-	for _, name := range c.symbolTable.Names() {
-		symbol, _, ok := c.symbolTable.Resolve(name)
-		if ok && symbol.Scope == compiler.ScopeGlobal {
-			value := c.machine.Globals()[symbol.Index]
-			if value == nil {
-				value = &objects.UndefinedValue
-			}
+	c.lock.RLock()
+	defer c.lock.RUnlock()
 
-			vars = append(vars, &Variable{
-				name:  name,
-				value: value,
-			})
+	var vars []*Variable
+
+	for name, idx := range c.globalIndexes {
+		value := c.globals[idx]
+		if value == nil {
+			value = objects.UndefinedValue
 		}
+
+		vars = append(vars, &Variable{
+			name:  name,
+			value: value,
+		})
 	}
 
 	return vars
@@ -97,17 +140,20 @@ func (c *Compiled) GetAll() []*Variable {
 // Set replaces the value of a global variable identified by the name.
 // An error will be returned if the name was not defined during compilation.
 func (c *Compiled) Set(name string, value interface{}) error {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
 	obj, err := objects.FromInterface(value)
 	if err != nil {
 		return err
 	}
 
-	symbol, _, ok := c.symbolTable.Resolve(name)
-	if !ok || symbol.Scope != compiler.ScopeGlobal {
+	idx, ok := c.globalIndexes[name]
+	if !ok {
 		return fmt.Errorf("'%s' is not defined", name)
 	}
 
-	c.machine.Globals()[symbol.Index] = &obj
+	c.globals[idx] = obj
 
 	return nil
 }

vendor/github.com/d5/tengo/script/conversion.go

@@ -1,33 +0,0 @@
-package script
-
-import (
-	"github.com/d5/tengo/objects"
-)
-
-func objectToInterface(o objects.Object) interface{} {
-	switch val := o.(type) {
-	case *objects.Array:
-		return val.Value
-	case *objects.Map:
-		return val.Value
-	case *objects.Int:
-		return val.Value
-	case *objects.Float:
-		return val.Value
-	case *objects.Bool:
-		if val == objects.TrueValue {
-			return true
-		}
-		return false
-	case *objects.Char:
-		return val.Value
-	case *objects.String:
-		return val.Value
-	case *objects.Bytes:
-		return val.Value
-	case *objects.Undefined:
-		return nil
-	}
-
-	return o
-}

vendor/github.com/d5/tengo/script/script.go

@@ -9,23 +9,25 @@ import (
 	"github.com/d5/tengo/compiler/source"
 	"github.com/d5/tengo/objects"
 	"github.com/d5/tengo/runtime"
-	"github.com/d5/tengo/stdlib"
 )
 
 // Script can simplify compilation and execution of embedded scripts.
 type Script struct {
-	variables         map[string]*Variable
-	removedBuiltins   map[string]bool
-	removedStdModules map[string]bool
-	userModuleLoader  compiler.ModuleLoader
-	input             []byte
+	variables        map[string]*Variable
+	modules          *objects.ModuleMap
+	input            []byte
+	maxAllocs        int64
+	maxConstObjects  int
+	enableFileImport bool
 }
 
 // New creates a Script instance with an input script.
 func New(input []byte) *Script {
 	return &Script{
-		variables: make(map[string]*Variable),
-		input:     input,
+		variables:       make(map[string]*Variable),
+		input:           input,
+		maxAllocs:       -1,
+		maxConstObjects: -1,
 	}
 }
 
@@ -38,7 +40,7 @@ func (s *Script) Add(name string, value interface{}) error {
 
 	s.variables[name] = &Variable{
 		name:  name,
-		value: &obj,
+		value: obj,
 	}
 
 	return nil
@@ -56,32 +58,31 @@ func (s *Script) Remove(name string) bool {
 	return true
 }
 
-// DisableBuiltinFunction disables a builtin function.
-func (s *Script) DisableBuiltinFunction(name string) {
-	if s.removedBuiltins == nil {
-		s.removedBuiltins = make(map[string]bool)
-	}
-
-	s.removedBuiltins[name] = true
+// SetImports sets import modules.
+func (s *Script) SetImports(modules *objects.ModuleMap) {
+	s.modules = modules
 }
 
-// DisableStdModule disables a standard library module.
-func (s *Script) DisableStdModule(name string) {
-	if s.removedStdModules == nil {
-		s.removedStdModules = make(map[string]bool)
-	}
-
-	s.removedStdModules[name] = true
+// SetMaxAllocs sets the maximum number of objects allocations during the run time.
+// Compiled script will return runtime.ErrObjectAllocLimit error if it exceeds this limit.
+func (s *Script) SetMaxAllocs(n int64) {
+	s.maxAllocs = n
 }
 
-// SetUserModuleLoader sets the user module loader for the compiler.
-func (s *Script) SetUserModuleLoader(loader compiler.ModuleLoader) {
-	s.userModuleLoader = loader
+// SetMaxConstObjects sets the maximum number of objects in the compiled constants.
+func (s *Script) SetMaxConstObjects(n int) {
+	s.maxConstObjects = n
+}
+
+// EnableFileImport enables or disables module loading from local files.
+// Local file modules are disabled by default.
+func (s *Script) EnableFileImport(enable bool) {
+	s.enableFileImport = enable
 }
 
 // Compile compiles the script with all the defined variables, and, returns Compiled object.
 func (s *Script) Compile() (*Compiled, error) {
-	symbolTable, stdModules, globals, err := s.prepCompile()
+	symbolTable, globals, err := s.prepCompile()
 	if err != nil {
 		return nil, err
 	}
@@ -92,22 +93,44 @@ func (s *Script) Compile() (*Compiled, error) {
 	p := parser.NewParser(srcFile, s.input, nil)
 	file, err := p.ParseFile()
 	if err != nil {
-		return nil, fmt.Errorf("parse error: %s", err.Error())
-	}
-
-	c := compiler.NewCompiler(srcFile, symbolTable, nil, stdModules, nil)
-
-	if s.userModuleLoader != nil {
-		c.SetModuleLoader(s.userModuleLoader)
+		return nil, err
 	}
 
+	c := compiler.NewCompiler(srcFile, symbolTable, nil, s.modules, nil)
+	c.EnableFileImport(s.enableFileImport)
 	if err := c.Compile(file); err != nil {
 		return nil, err
 	}
 
+	// reduce globals size
+	globals = globals[:symbolTable.MaxSymbols()+1]
+
+	// global symbol names to indexes
+	globalIndexes := make(map[string]int, len(globals))
+	for _, name := range symbolTable.Names() {
+		symbol, _, _ := symbolTable.Resolve(name)
+		if symbol.Scope == compiler.ScopeGlobal {
+			globalIndexes[name] = symbol.Index
+		}
+	}
+
+	// remove duplicates from constants
+	bytecode := c.Bytecode()
+	bytecode.RemoveDuplicates()
+
+	// check the constant objects limit
+	if s.maxConstObjects >= 0 {
+		cnt := bytecode.CountObjects()
+		if cnt > s.maxConstObjects {
+			return nil, fmt.Errorf("exceeding constant objects limit: %d", cnt)
+		}
+	}
+
 	return &Compiled{
-		symbolTable: symbolTable,
-		machine:     runtime.NewVM(c.Bytecode(), globals, nil),
+		globalIndexes: globalIndexes,
+		bytecode:      bytecode,
+		globals:       globals,
+		maxAllocs:     s.maxAllocs,
 	}, nil
 }
 
@@ -136,7 +159,7 @@ func (s *Script) RunContext(ctx context.Context) (compiled *Compiled, err error)
 	return
 }
 
-func (s *Script) prepCompile() (symbolTable *compiler.SymbolTable, stdModules map[string]bool, globals []*objects.Object, err error) {
+func (s *Script) prepCompile() (symbolTable *compiler.SymbolTable, globals []objects.Object, err error) {
 	var names []string
 	for name := range s.variables {
 		names = append(names, name)
@@ -144,19 +167,10 @@ func (s *Script) prepCompile() (symbolTable *compiler.SymbolTable, stdModules ma
 
 	symbolTable = compiler.NewSymbolTable()
 	for idx, fn := range objects.Builtins {
-		if !s.removedBuiltins[fn.Name] {
-			symbolTable.DefineBuiltin(idx, fn.Name)
-		}
+		symbolTable.DefineBuiltin(idx, fn.Name)
 	}
 
-	stdModules = make(map[string]bool)
-	for name := range stdlib.Modules {
-		if !s.removedStdModules[name] {
-			stdModules[name] = true
-		}
-	}
-
-	globals = make([]*objects.Object, runtime.GlobalsSize, runtime.GlobalsSize)
+	globals = make([]objects.Object, runtime.GlobalsSize)
 
 	for idx, name := range names {
 		symbol := symbolTable.Define(name)

vendor/github.com/d5/tengo/script/variable.go

@@ -9,7 +9,7 @@ import (
 // Variable is a user-defined variable for the script.
 type Variable struct {
 	name  string
-	value *objects.Object
+	value objects.Object
 }
 
 // NewVariable creates a Variable.
@@ -21,7 +21,7 @@ func NewVariable(name string, value interface{}) (*Variable, error) {
 
 	return &Variable{
 		name:  name,
-		value: &obj,
+		value: obj,
 	}, nil
 }
 
@@ -32,18 +32,18 @@ func (v *Variable) Name() string {
 
 // Value returns an empty interface of the variable value.
 func (v *Variable) Value() interface{} {
-	return objectToInterface(*v.value)
+	return objects.ToInterface(v.value)
 }
 
 // ValueType returns the name of the value type.
 func (v *Variable) ValueType() string {
-	return (*v.value).TypeName()
+	return v.value.TypeName()
 }
 
 // Int returns int value of the variable value.
 // It returns 0 if the value is not convertible to int.
 func (v *Variable) Int() int {
-	c, _ := objects.ToInt(*v.value)
+	c, _ := objects.ToInt(v.value)
 
 	return c
 }
@@ -51,7 +51,7 @@ func (v *Variable) Int() int {
 // Int64 returns int64 value of the variable value.
 // It returns 0 if the value is not convertible to int64.
 func (v *Variable) Int64() int64 {
-	c, _ := objects.ToInt64(*v.value)
+	c, _ := objects.ToInt64(v.value)
 
 	return c
 }
@@ -59,7 +59,7 @@ func (v *Variable) Int64() int64 {
 // Float returns float64 value of the variable value.
 // It returns 0.0 if the value is not convertible to float64.
 func (v *Variable) Float() float64 {
-	c, _ := objects.ToFloat64(*v.value)
+	c, _ := objects.ToFloat64(v.value)
 
 	return c
 }
@@ -67,7 +67,7 @@ func (v *Variable) Float() float64 {
 // Char returns rune value of the variable value.
 // It returns 0 if the value is not convertible to rune.
 func (v *Variable) Char() rune {
-	c, _ := objects.ToRune(*v.value)
+	c, _ := objects.ToRune(v.value)
 
 	return c
 }
@@ -75,7 +75,7 @@ func (v *Variable) Char() rune {
 // Bool returns bool value of the variable value.
 // It returns 0 if the value is not convertible to bool.
 func (v *Variable) Bool() bool {
-	c, _ := objects.ToBool(*v.value)
+	c, _ := objects.ToBool(v.value)
 
 	return c
 }
@@ -83,11 +83,11 @@ func (v *Variable) Bool() bool {
 // Array returns []interface value of the variable value.
 // It returns 0 if the value is not convertible to []interface.
 func (v *Variable) Array() []interface{} {
-	switch val := (*v.value).(type) {
+	switch val := v.value.(type) {
 	case *objects.Array:
 		var arr []interface{}
 		for _, e := range val.Value {
-			arr = append(arr, objectToInterface(e))
+			arr = append(arr, objects.ToInterface(e))
 		}
 		return arr
 	}
@@ -98,11 +98,11 @@ func (v *Variable) Array() []interface{} {
 // Map returns map[string]interface{} value of the variable value.
 // It returns 0 if the value is not convertible to map[string]interface{}.
 func (v *Variable) Map() map[string]interface{} {
-	switch val := (*v.value).(type) {
+	switch val := v.value.(type) {
 	case *objects.Map:
 		kv := make(map[string]interface{})
 		for mk, mv := range val.Value {
-			kv[mk] = objectToInterface(mv)
+			kv[mk] = objects.ToInterface(mv)
 		}
 		return kv
 	}
@@ -113,7 +113,7 @@ func (v *Variable) Map() map[string]interface{} {
 // String returns string value of the variable value.
 // It returns 0 if the value is not convertible to string.
 func (v *Variable) String() string {
-	c, _ := objects.ToString(*v.value)
+	c, _ := objects.ToString(v.value)
 
 	return c
 }
@@ -121,7 +121,7 @@ func (v *Variable) String() string {
 // Bytes returns a byte slice of the variable value.
 // It returns nil if the value is not convertible to byte slice.
 func (v *Variable) Bytes() []byte {
-	c, _ := objects.ToByteSlice(*v.value)
+	c, _ := objects.ToByteSlice(v.value)
 
 	return c
 }
@@ -129,7 +129,7 @@ func (v *Variable) Bytes() []byte {
 // Error returns an error if the underlying value is error object.
 // If not, this returns nil.
 func (v *Variable) Error() error {
-	err, ok := (*v.value).(*objects.Error)
+	err, ok := v.value.(*objects.Error)
 	if ok {
 		return errors.New(err.String())
 	}
@@ -140,10 +140,10 @@ func (v *Variable) Error() error {
 // Object returns an underlying Object of the variable value.
 // Note that returned Object is a copy of an actual Object used in the script.
 func (v *Variable) Object() objects.Object {
-	return *v.value
+	return v.value
 }
 
 // IsUndefined returns true if the underlying value is undefined.
 func (v *Variable) IsUndefined() bool {
-	return *v.value == objects.UndefinedValue
+	return v.value == objects.UndefinedValue
 }

vendor/github.com/d5/tengo/stdlib/builtin_modules.go

@@ -0,0 +1,14 @@
+package stdlib
+
+import "github.com/d5/tengo/objects"
+
+// BuiltinModules are builtin type standard library modules.
+var BuiltinModules = map[string]map[string]objects.Object{
+	"math":  mathModule,
+	"os":    osModule,
+	"text":  textModule,
+	"times": timesModule,
+	"rand":  randModule,
+	"fmt":   fmtModule,
+	"json":  jsonModule,
+}

vendor/github.com/d5/tengo/stdlib/fmt.go

@@ -0,0 +1,116 @@
+package stdlib
+
+import (
+	"fmt"
+
+	"github.com/d5/tengo"
+	"github.com/d5/tengo/objects"
+)
+
+var fmtModule = map[string]objects.Object{
+	"print":   &objects.UserFunction{Name: "print", Value: fmtPrint},
+	"printf":  &objects.UserFunction{Name: "printf", Value: fmtPrintf},
+	"println": &objects.UserFunction{Name: "println", Value: fmtPrintln},
+	"sprintf": &objects.UserFunction{Name: "sprintf", Value: fmtSprintf},
+}
+
+func fmtPrint(args ...objects.Object) (ret objects.Object, err error) {
+	printArgs, err := getPrintArgs(args...)
+	if err != nil {
+		return nil, err
+	}
+
+	_, _ = fmt.Print(printArgs...)
+
+	return nil, nil
+}
+
+func fmtPrintf(args ...objects.Object) (ret objects.Object, err error) {
+	numArgs := len(args)
+	if numArgs == 0 {
+		return nil, objects.ErrWrongNumArguments
+	}
+
+	format, ok := args[0].(*objects.String)
+	if !ok {
+		return nil, objects.ErrInvalidArgumentType{
+			Name:     "format",
+			Expected: "string",
+			Found:    args[0].TypeName(),
+		}
+	}
+	if numArgs == 1 {
+		fmt.Print(format)
+		return nil, nil
+	}
+
+	formatArgs := make([]interface{}, numArgs-1, numArgs-1)
+	for idx, arg := range args[1:] {
+		formatArgs[idx] = objects.ToInterface(arg)
+	}
+
+	fmt.Printf(format.Value, formatArgs...)
+
+	return nil, nil
+}
+
+func fmtPrintln(args ...objects.Object) (ret objects.Object, err error) {
+	printArgs, err := getPrintArgs(args...)
+	if err != nil {
+		return nil, err
+	}
+
+	printArgs = append(printArgs, "\n")
+	_, _ = fmt.Print(printArgs...)
+
+	return nil, nil
+}
+
+func fmtSprintf(args ...objects.Object) (ret objects.Object, err error) {
+	numArgs := len(args)
+	if numArgs == 0 {
+		return nil, objects.ErrWrongNumArguments
+	}
+
+	format, ok := args[0].(*objects.String)
+	if !ok {
+		return nil, objects.ErrInvalidArgumentType{
+			Name:     "format",
+			Expected: "string",
+			Found:    args[0].TypeName(),
+		}
+	}
+	if numArgs == 1 {
+		return format, nil // okay to return 'format' directly as String is immutable
+	}
+
+	formatArgs := make([]interface{}, numArgs-1, numArgs-1)
+	for idx, arg := range args[1:] {
+		formatArgs[idx] = objects.ToInterface(arg)
+	}
+
+	s := fmt.Sprintf(format.Value, formatArgs...)
+
+	if len(s) > tengo.MaxStringLen {
+		return nil, objects.ErrStringLimit
+	}
+
+	return &objects.String{Value: s}, nil
+}
+
+func getPrintArgs(args ...objects.Object) ([]interface{}, error) {
+	var printArgs []interface{}
+	l := 0
+	for _, arg := range args {
+		s, _ := objects.ToString(arg)
+		slen := len(s)
+		if l+slen > tengo.MaxStringLen { // make sure length does not exceed the limit
+			return nil, objects.ErrStringLimit
+		}
+		l += slen
+
+		printArgs = append(printArgs, s)
+	}
+
+	return printArgs, nil
+}

vendor/github.com/d5/tengo/stdlib/func_typedefs.go

@@ -3,6 +3,7 @@ package stdlib
 import (
 	"fmt"
 
+	"github.com/d5/tengo"
 	"github.com/d5/tengo/objects"
 )
 
@@ -124,7 +125,13 @@ func FuncARS(fn func() string) objects.CallableFunc {
 			return nil, objects.ErrWrongNumArguments
 		}
 
-		return &objects.String{Value: fn()}, nil
+		s := fn()
+
+		if len(s) > tengo.MaxStringLen {
+			return nil, objects.ErrStringLimit
+		}
+
+		return &objects.String{Value: s}, nil
 	}
 }
 
@@ -141,6 +148,10 @@ func FuncARSE(fn func() (string, error)) objects.CallableFunc {
 			return wrapError(err), nil
 		}
 
+		if len(res) > tengo.MaxStringLen {
+			return nil, objects.ErrStringLimit
+		}
+
 		return &objects.String{Value: res}, nil
 	}
 }
@@ -158,6 +169,10 @@ func FuncARYE(fn func() ([]byte, error)) objects.CallableFunc {
 			return wrapError(err), nil
 		}
 
+		if len(res) > tengo.MaxBytesLen {
+			return nil, objects.ErrBytesLimit
+		}
+
 		return &objects.Bytes{Value: res}, nil
 	}
 }
@@ -183,8 +198,12 @@ func FuncARSs(fn func() []string) objects.CallableFunc {
 		}
 
 		arr := &objects.Array{}
-		for _, osArg := range fn() {
-			arr.Value = append(arr.Value, &objects.String{Value: osArg})
+		for _, elem := range fn() {
+			if len(elem) > tengo.MaxStringLen {
+				return nil, objects.ErrStringLimit
+			}
+
+			arr.Value = append(arr.Value, &objects.String{Value: elem})
 		}
 
 		return arr, nil
@@ -493,7 +512,13 @@ func FuncASRS(fn func(string) string) objects.CallableFunc {
 			}
 		}
 
-		return &objects.String{Value: fn(s1)}, nil
+		s := fn(s1)
+
+		if len(s) > tengo.MaxStringLen {
+			return nil, objects.ErrStringLimit
+		}
+
+		return &objects.String{Value: s}, nil
 	}
 }
 
@@ -516,8 +541,12 @@ func FuncASRSs(fn func(string) []string) objects.CallableFunc {
 		res := fn(s1)
 
 		arr := &objects.Array{}
-		for _, osArg := range res {
-			arr.Value = append(arr.Value, &objects.String{Value: osArg})
+		for _, elem := range res {
+			if len(elem) > tengo.MaxStringLen {
+				return nil, objects.ErrStringLimit
+			}
+
+			arr.Value = append(arr.Value, &objects.String{Value: elem})
 		}
 
 		return arr, nil
@@ -546,6 +575,10 @@ func FuncASRSE(fn func(string) (string, error)) objects.CallableFunc {
 			return wrapError(err), nil
 		}
 
+		if len(res) > tengo.MaxStringLen {
+			return nil, objects.ErrStringLimit
+		}
+
 		return &objects.String{Value: res}, nil
 	}
 }
@@ -628,6 +661,10 @@ func FuncASSRSs(fn func(string, string) []string) objects.CallableFunc {
 
 		arr := &objects.Array{}
 		for _, res := range fn(s1, s2) {
+			if len(res) > tengo.MaxStringLen {
+				return nil, objects.ErrStringLimit
+			}
+
 			arr.Value = append(arr.Value, &objects.String{Value: res})
 		}
 
@@ -671,6 +708,10 @@ func FuncASSIRSs(fn func(string, string, int) []string) objects.CallableFunc {
 
 		arr := &objects.Array{}
 		for _, res := range fn(s1, s2, i3) {
+			if len(res) > tengo.MaxStringLen {
+				return nil, objects.ErrStringLimit
+			}
+
 			arr.Value = append(arr.Value, &objects.String{Value: res})
 		}
 
@@ -732,7 +773,13 @@ func FuncASSRS(fn func(string, string) string) objects.CallableFunc {
 			}
 		}
 
-		return &objects.String{Value: fn(s1, s2)}, nil
+		s := fn(s1, s2)
+
+		if len(s) > tengo.MaxStringLen {
+			return nil, objects.ErrStringLimit
+		}
+
+		return &objects.String{Value: s}, nil
 	}
 }
 
@@ -819,7 +866,12 @@ func FuncASsSRS(fn func([]string, string) string) objects.CallableFunc {
 			}
 		}
 
-		return &objects.String{Value: fn(ss1, s2)}, nil
+		s := fn(ss1, s2)
+		if len(s) > tengo.MaxStringLen {
+			return nil, objects.ErrStringLimit
+		}
+
+		return &objects.String{Value: s}, nil
 	}
 }
 
@@ -909,7 +961,13 @@ func FuncASIRS(fn func(string, int) string) objects.CallableFunc {
 			}
 		}
 
-		return &objects.String{Value: fn(s1, i2)}, nil
+		s := fn(s1, i2)
+
+		if len(s) > tengo.MaxStringLen {
+			return nil, objects.ErrStringLimit
+		}
+
+		return &objects.String{Value: s}, nil
 	}
 }
 
@@ -1028,6 +1086,10 @@ func FuncAIRSsE(fn func(int) ([]string, error)) objects.CallableFunc {
 
 		arr := &objects.Array{}
 		for _, r := range res {
+			if len(r) > tengo.MaxStringLen {
+				return nil, objects.ErrStringLimit
+			}
+
 			arr.Value = append(arr.Value, &objects.String{Value: r})
 		}
 
@@ -1052,6 +1114,12 @@ func FuncAIRS(fn func(int) string) objects.CallableFunc {
 			}
 		}
 
-		return &objects.String{Value: fn(i1)}, nil
+		s := fn(i1)
+
+		if len(s) > tengo.MaxStringLen {
+			return nil, objects.ErrStringLimit
+		}
+
+		return &objects.String{Value: s}, nil
 	}
 }

vendor/github.com/d5/tengo/stdlib/gensrcmods.go

@@ -0,0 +1,53 @@
+// +build ignore
+
+package main
+
+import (
+	"bytes"
+	"io/ioutil"
+	"log"
+	"regexp"
+	"strconv"
+)
+
+var tengoModFileRE = regexp.MustCompile(`^srcmod_(\w+).tengo$`)
+
+func main() {
+	modules := make(map[string]string)
+
+	// enumerate all Tengo module files
+	files, err := ioutil.ReadDir(".")
+	if err != nil {
+		log.Fatal(err)
+	}
+	for _, file := range files {
+		m := tengoModFileRE.FindStringSubmatch(file.Name())
+		if m != nil {
+			modName := m[1]
+
+			src, err := ioutil.ReadFile(file.Name())
+			if err != nil {
+				log.Fatalf("file '%s' read error: %s", file.Name(), err.Error())
+			}
+
+			modules[modName] = string(src)
+		}
+	}
+
+	var out bytes.Buffer
+	out.WriteString(`// Code generated using gensrcmods.go; DO NOT EDIT.
+
+package stdlib
+
+// SourceModules are source type standard library modules.
+var SourceModules = map[string]string{` + "\n")
+	for modName, modSrc := range modules {
+		out.WriteString("\t\"" + modName + "\": " + strconv.Quote(modSrc) + ",\n")
+	}
+	out.WriteString("}\n")
+
+	const target = "source_modules.go"
+	if err := ioutil.WriteFile(target, out.Bytes(), 0644); err != nil {
+		log.Fatal(err)
+	}
+}

vendor/github.com/d5/tengo/stdlib/json.go

@@ -0,0 +1,126 @@
+package stdlib
+
+import (
+	"bytes"
+	gojson "encoding/json"
+
+	"github.com/d5/tengo/objects"
+	"github.com/d5/tengo/stdlib/json"
+)
+
+var jsonModule = map[string]objects.Object{
+	"decode":      &objects.UserFunction{Name: "decode", Value: jsonDecode},
+	"encode":      &objects.UserFunction{Name: "encode", Value: jsonEncode},
+	"indent":      &objects.UserFunction{Name: "encode", Value: jsonIndent},
+	"html_escape": &objects.UserFunction{Name: "html_escape", Value: jsonHTMLEscape},
+}
+
+func jsonDecode(args ...objects.Object) (ret objects.Object, err error) {
+	if len(args) != 1 {
+		return nil, objects.ErrWrongNumArguments
+	}
+
+	switch o := args[0].(type) {
+	case *objects.Bytes:
+		v, err := json.Decode(o.Value)
+		if err != nil {
+			return &objects.Error{Value: &objects.String{Value: err.Error()}}, nil
+		}
+		return v, nil
+	case *objects.String:
+		v, err := json.Decode([]byte(o.Value))
+		if err != nil {
+			return &objects.Error{Value: &objects.String{Value: err.Error()}}, nil
+		}
+		return v, nil
+	default:
+		return nil, objects.ErrInvalidArgumentType{
+			Name:     "first",
+			Expected: "bytes/string",
+			Found:    args[0].TypeName(),
+		}
+	}
+}
+
+func jsonEncode(args ...objects.Object) (ret objects.Object, err error) {
+	if len(args) != 1 {
+		return nil, objects.ErrWrongNumArguments
+	}
+
+	b, err := json.Encode(args[0])
+	if err != nil {
+		return &objects.Error{Value: &objects.String{Value: err.Error()}}, nil
+	}
+
+	return &objects.Bytes{Value: b}, nil
+}
+
+func jsonIndent(args ...objects.Object) (ret objects.Object, err error) {
+	if len(args) != 3 {
+		return nil, objects.ErrWrongNumArguments
+	}
+
+	prefix, ok := objects.ToString(args[1])
+	if !ok {
+		return nil, objects.ErrInvalidArgumentType{
+			Name:     "prefix",
+			Expected: "string(compatible)",
+			Found:    args[1].TypeName(),
+		}
+	}
+
+	indent, ok := objects.ToString(args[2])
+	if !ok {
+		return nil, objects.ErrInvalidArgumentType{
+			Name:     "indent",
+			Expected: "string(compatible)",
+			Found:    args[2].TypeName(),
+		}
+	}
+
+	switch o := args[0].(type) {
+	case *objects.Bytes:
+		var dst bytes.Buffer
+		err := gojson.Indent(&dst, o.Value, prefix, indent)
+		if err != nil {
+			return &objects.Error{Value: &objects.String{Value: err.Error()}}, nil
+		}
+		return &objects.Bytes{Value: dst.Bytes()}, nil
+	case *objects.String:
+		var dst bytes.Buffer
+		err := gojson.Indent(&dst, []byte(o.Value), prefix, indent)
+		if err != nil {
+			return &objects.Error{Value: &objects.String{Value: err.Error()}}, nil
+		}
+		return &objects.Bytes{Value: dst.Bytes()}, nil
+	default:
+		return nil, objects.ErrInvalidArgumentType{
+			Name:     "first",
+			Expected: "bytes/string",
+			Found:    args[0].TypeName(),
+		}
+	}
+}
+
+func jsonHTMLEscape(args ...objects.Object) (ret objects.Object, err error) {
+	if len(args) != 1 {
+		return nil, objects.ErrWrongNumArguments
+	}
+
+	switch o := args[0].(type) {
+	case *objects.Bytes:
+		var dst bytes.Buffer
+		gojson.HTMLEscape(&dst, o.Value)
+		return &objects.Bytes{Value: dst.Bytes()}, nil
+	case *objects.String:
+		var dst bytes.Buffer
+		gojson.HTMLEscape(&dst, []byte(o.Value))
+		return &objects.Bytes{Value: dst.Bytes()}, nil
+	default:
+		return nil, objects.ErrInvalidArgumentType{
+			Name:     "first",
+			Expected: "bytes/string",
+			Found:    args[0].TypeName(),
+		}
+	}
+}

vendor/github.com/d5/tengo/stdlib/json/decode.go

@@ -0,0 +1,374 @@
+// A modified version of Go's JSON implementation.
+
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"strconv"
+	"unicode"
+	"unicode/utf16"
+	"unicode/utf8"
+
+	"github.com/d5/tengo/objects"
+)
+
+// Decode parses the JSON-encoded data and returns the result object.
+func Decode(data []byte) (objects.Object, error) {
+	var d decodeState
+	err := checkValid(data, &d.scan)
+	if err != nil {
+		return nil, err
+	}
+
+	d.init(data)
+	d.scan.reset()
+	d.scanWhile(scanSkipSpace)
+
+	return d.value()
+}
+
+// decodeState represents the state while decoding a JSON value.
+type decodeState struct {
+	data   []byte
+	off    int // next read offset in data
+	opcode int // last read result
+	scan   scanner
+}
+
+// readIndex returns the position of the last byte read.
+func (d *decodeState) readIndex() int {
+	return d.off - 1
+}
+
+const phasePanicMsg = "JSON decoder out of sync - data changing underfoot?"
+
+func (d *decodeState) init(data []byte) *decodeState {
+	d.data = data
+	d.off = 0
+	return d
+}
+
+// scanNext processes the byte at d.data[d.off].
+func (d *decodeState) scanNext() {
+	if d.off < len(d.data) {
+		d.opcode = d.scan.step(&d.scan, d.data[d.off])
+		d.off++
+	} else {
+		d.opcode = d.scan.eof()
+		d.off = len(d.data) + 1 // mark processed EOF with len+1
+	}
+}
+
+// scanWhile processes bytes in d.data[d.off:] until it
+// receives a scan code not equal to op.
+func (d *decodeState) scanWhile(op int) {
+	s, data, i := &d.scan, d.data, d.off
+	for i < len(data) {
+		newOp := s.step(s, data[i])
+		i++
+		if newOp != op {
+			d.opcode = newOp
+			d.off = i
+			return
+		}
+	}
+
+	d.off = len(data) + 1 // mark processed EOF with len+1
+	d.opcode = d.scan.eof()
+}
+
+func (d *decodeState) value() (objects.Object, error) {
+	switch d.opcode {
+	default:
+		panic(phasePanicMsg)
+
+	case scanBeginArray:
+		o, err := d.array()
+		if err != nil {
+			return nil, err
+		}
+
+		d.scanNext()
+
+		return o, nil
+
+	case scanBeginObject:
+		o, err := d.object()
+		if err != nil {
+			return nil, err
+		}
+
+		d.scanNext()
+
+		return o, nil
+
+	case scanBeginLiteral:
+		return d.literal()
+	}
+}
+
+func (d *decodeState) array() (objects.Object, error) {
+	var arr []objects.Object
+	for {
+		// Look ahead for ] - can only happen on first iteration.
+		d.scanWhile(scanSkipSpace)
+		if d.opcode == scanEndArray {
+			break
+		}
+
+		o, err := d.value()
+		if err != nil {
+			return nil, err
+		}
+		arr = append(arr, o)
+
+		// Next token must be , or ].
+		if d.opcode == scanSkipSpace {
+			d.scanWhile(scanSkipSpace)
+		}
+		if d.opcode == scanEndArray {
+			break
+		}
+		if d.opcode != scanArrayValue {
+			panic(phasePanicMsg)
+		}
+	}
+
+	return &objects.Array{Value: arr}, nil
+}
+
+func (d *decodeState) object() (objects.Object, error) {
+	m := make(map[string]objects.Object)
+	for {
+		// Read opening " of string key or closing }.
+		d.scanWhile(scanSkipSpace)
+		if d.opcode == scanEndObject {
+			// closing } - can only happen on first iteration.
+			break
+		}
+		if d.opcode != scanBeginLiteral {
+			panic(phasePanicMsg)
+		}
+
+		// Read string key.
+		start := d.readIndex()
+		d.scanWhile(scanContinue)
+		item := d.data[start:d.readIndex()]
+		key, ok := unquote(item)
+		if !ok {
+			panic(phasePanicMsg)
+		}
+
+		// Read : before value.
+		if d.opcode == scanSkipSpace {
+			d.scanWhile(scanSkipSpace)
+		}
+		if d.opcode != scanObjectKey {
+			panic(phasePanicMsg)
+		}
+		d.scanWhile(scanSkipSpace)
+
+		// Read value.
+		o, err := d.value()
+		if err != nil {
+			return nil, err
+		}
+
+		m[key] = o
+
+		// Next token must be , or }.
+		if d.opcode == scanSkipSpace {
+			d.scanWhile(scanSkipSpace)
+		}
+		if d.opcode == scanEndObject {
+			break
+		}
+		if d.opcode != scanObjectValue {
+			panic(phasePanicMsg)
+		}
+	}
+
+	return &objects.Map{Value: m}, nil
+}
+
+func (d *decodeState) literal() (objects.Object, error) {
+	// All bytes inside literal return scanContinue op code.
+	start := d.readIndex()
+	d.scanWhile(scanContinue)
+
+	item := d.data[start:d.readIndex()]
+
+	switch c := item[0]; c {
+	case 'n': // null
+		return objects.UndefinedValue, nil
+
+	case 't', 'f': // true, false
+		if c == 't' {
+			return objects.TrueValue, nil
+		}
+		return objects.FalseValue, nil
+
+	case '"': // string
+		s, ok := unquote(item)
+		if !ok {
+			panic(phasePanicMsg)
+		}
+		return &objects.String{Value: s}, nil
+
+	default: // number
+		if c != '-' && (c < '0' || c > '9') {
+			panic(phasePanicMsg)
+		}
+
+		n, _ := strconv.ParseFloat(string(item), 10)
+		return &objects.Float{Value: n}, nil
+	}
+}
+
+// getu4 decodes \uXXXX from the beginning of s, returning the hex value,
+// or it returns -1.
+func getu4(s []byte) rune {
+	if len(s) < 6 || s[0] != '\\' || s[1] != 'u' {
+		return -1
+	}
+	var r rune
+	for _, c := range s[2:6] {
+		switch {
+		case '0' <= c && c <= '9':
+			c = c - '0'
+		case 'a' <= c && c <= 'f':
+			c = c - 'a' + 10
+		case 'A' <= c && c <= 'F':
+			c = c - 'A' + 10
+		default:
+			return -1
+		}
+		r = r*16 + rune(c)
+	}
+	return r
+}
+
+// unquote converts a quoted JSON string literal s into an actual string t.
+// The rules are different than for Go, so cannot use strconv.Unquote.
+func unquote(s []byte) (t string, ok bool) {
+	s, ok = unquoteBytes(s)
+	t = string(s)
+	return
+}
+
+func unquoteBytes(s []byte) (t []byte, ok bool) {
+	if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' {
+		return
+	}
+	s = s[1 : len(s)-1]
+
+	// Check for unusual characters. If there are none,
+	// then no unquoting is needed, so return a slice of the
+	// original bytes.
+	r := 0
+	for r < len(s) {
+		c := s[r]
+		if c == '\\' || c == '"' || c < ' ' {
+			break
+		}
+		if c < utf8.RuneSelf {
+			r++
+			continue
+		}
+		rr, size := utf8.DecodeRune(s[r:])
+		if rr == utf8.RuneError && size == 1 {
+			break
+		}
+		r += size
+	}
+	if r == len(s) {
+		return s, true
+	}
+
+	b := make([]byte, len(s)+2*utf8.UTFMax)
+	w := copy(b, s[0:r])
+	for r < len(s) {
+		// Out of room? Can only happen if s is full of
+		// malformed UTF-8 and we're replacing each
+		// byte with RuneError.
+		if w >= len(b)-2*utf8.UTFMax {
+			nb := make([]byte, (len(b)+utf8.UTFMax)*2)
+			copy(nb, b[0:w])
+			b = nb
+		}
+		switch c := s[r]; {
+		case c == '\\':
+			r++
+			if r >= len(s) {
+				return
+			}
+			switch s[r] {
+			default:
+				return
+			case '"', '\\', '/', '\'':
+				b[w] = s[r]
+				r++
+				w++
+			case 'b':
+				b[w] = '\b'
+				r++
+				w++
+			case 'f':
+				b[w] = '\f'
+				r++
+				w++
+			case 'n':
+				b[w] = '\n'
+				r++
+				w++
+			case 'r':
+				b[w] = '\r'
+				r++
+				w++
+			case 't':
+				b[w] = '\t'
+				r++
+				w++
+			case 'u':
+				r--
+				rr := getu4(s[r:])
+				if rr < 0 {
+					return
+				}
+				r += 6
+				if utf16.IsSurrogate(rr) {
+					rr1 := getu4(s[r:])
+					if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar {
+						// A valid pair; consume.
+						r += 6
+						w += utf8.EncodeRune(b[w:], dec)
+						break
+					}
+					// Invalid surrogate; fall back to replacement rune.
+					rr = unicode.ReplacementChar
+				}
+				w += utf8.EncodeRune(b[w:], rr)
+			}
+
+		// Quote, control characters are invalid.
+		case c == '"', c < ' ':
+			return
+
+		// ASCII
+		case c < utf8.RuneSelf:
+			b[w] = c
+			r++
+			w++
+
+		// Coerce to well-formed UTF-8.
+		default:
+			rr, size := utf8.DecodeRune(s[r:])
+			r += size
+			w += utf8.EncodeRune(b[w:], rr)
+		}
+	}
+	return b[0:w], true
+}

vendor/github.com/d5/tengo/stdlib/json/encode.go

@@ -0,0 +1,147 @@
+// A modified version of Go's JSON implementation.
+
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"encoding/base64"
+	"errors"
+	"math"
+	"strconv"
+
+	"github.com/d5/tengo/objects"
+)
+
+// Encode returns the JSON encoding of the object.
+func Encode(o objects.Object) ([]byte, error) {
+	var b []byte
+
+	switch o := o.(type) {
+	case *objects.Array:
+		b = append(b, '[')
+		len1 := len(o.Value) - 1
+		for idx, elem := range o.Value {
+			eb, err := Encode(elem)
+			if err != nil {
+				return nil, err
+			}
+			b = append(b, eb...)
+			if idx < len1 {
+				b = append(b, ',')
+			}
+		}
+		b = append(b, ']')
+	case *objects.ImmutableArray:
+		b = append(b, '[')
+		len1 := len(o.Value) - 1
+		for idx, elem := range o.Value {
+			eb, err := Encode(elem)
+			if err != nil {
+				return nil, err
+			}
+			b = append(b, eb...)
+			if idx < len1 {
+				b = append(b, ',')
+			}
+		}
+		b = append(b, ']')
+	case *objects.Map:
+		b = append(b, '{')
+		len1 := len(o.Value) - 1
+		idx := 0
+		for key, value := range o.Value {
+			b = strconv.AppendQuote(b, key)
+			b = append(b, ':')
+			eb, err := Encode(value)
+			if err != nil {
+				return nil, err
+			}
+			b = append(b, eb...)
+			if idx < len1 {
+				b = append(b, ',')
+			}
+			idx++
+		}
+		b = append(b, '}')
+	case *objects.ImmutableMap:
+		b = append(b, '{')
+		len1 := len(o.Value) - 1
+		idx := 0
+		for key, value := range o.Value {
+			b = strconv.AppendQuote(b, key)
+			b = append(b, ':')
+			eb, err := Encode(value)
+			if err != nil {
+				return nil, err
+			}
+			b = append(b, eb...)
+			if idx < len1 {
+				b = append(b, ',')
+			}
+			idx++
+		}
+		b = append(b, '}')
+	case *objects.Bool:
+		if o.IsFalsy() {
+			b = strconv.AppendBool(b, false)
+		} else {
+			b = strconv.AppendBool(b, true)
+		}
+	case *objects.Bytes:
+		b = append(b, '"')
+		encodedLen := base64.StdEncoding.EncodedLen(len(o.Value))
+		dst := make([]byte, encodedLen)
+		base64.StdEncoding.Encode(dst, o.Value)
+		b = append(b, dst...)
+		b = append(b, '"')
+	case *objects.Char:
+		b = strconv.AppendInt(b, int64(o.Value), 10)
+	case *objects.Float:
+		var y []byte
+
+		f := o.Value
+		if math.IsInf(f, 0) || math.IsNaN(f) {
+			return nil, errors.New("unsupported float value")
+		}
+
+		// Convert as if by ES6 number to string conversion.
+		// This matches most other JSON generators.
+		abs := math.Abs(f)
+		fmt := byte('f')
+		if abs != 0 {
+			if abs < 1e-6 || abs >= 1e21 {
+				fmt = 'e'
+			}
+		}
+		y = strconv.AppendFloat(y, f, fmt, -1, 64)
+		if fmt == 'e' {
+			// clean up e-09 to e-9
+			n := len(y)
+			if n >= 4 && y[n-4] == 'e' && y[n-3] == '-' && y[n-2] == '0' {
+				y[n-2] = y[n-1]
+				y = y[:n-1]
+			}
+		}
+
+		b = append(b, y...)
+	case *objects.Int:
+		b = strconv.AppendInt(b, o.Value, 10)
+	case *objects.String:
+		b = strconv.AppendQuote(b, o.Value)
+	case *objects.Time:
+		y, err := o.Value.MarshalJSON()
+		if err != nil {
+			return nil, err
+		}
+		b = append(b, y...)
+	case *objects.Undefined:
+		b = append(b, "null"...)
+	default:
+		// unknown type: ignore
+	}
+
+	return b, nil
+}

vendor/github.com/d5/tengo/stdlib/json/scanner.go

@@ -0,0 +1,559 @@
+// A modified version of Go's JSON implementation.
+
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import "strconv"
+
+func checkValid(data []byte, scan *scanner) error {
+	scan.reset()
+	for _, c := range data {
+		scan.bytes++
+		if scan.step(scan, c) == scanError {
+			return scan.err
+		}
+	}
+	if scan.eof() == scanError {
+		return scan.err
+	}
+	return nil
+}
+
+// A SyntaxError is a description of a JSON syntax error.
+type SyntaxError struct {
+	msg    string // description of error
+	Offset int64  // error occurred after reading Offset bytes
+}
+
+func (e *SyntaxError) Error() string { return e.msg }
+
+// A scanner is a JSON scanning state machine.
+// Callers call scan.reset() and then pass bytes in one at a time
+// by calling scan.step(&scan, c) for each byte.
+// The return value, referred to as an opcode, tells the
+// caller about significant parsing events like beginning
+// and ending literals, objects, and arrays, so that the
+// caller can follow along if it wishes.
+// The return value scanEnd indicates that a single top-level
+// JSON value has been completed, *before* the byte that
+// just got passed in.  (The indication must be delayed in order
+// to recognize the end of numbers: is 123 a whole value or
+// the beginning of 12345e+6?).
+type scanner struct {
+	// The step is a func to be called to execute the next transition.
+	// Also tried using an integer constant and a single func
+	// with a switch, but using the func directly was 10% faster
+	// on a 64-bit Mac Mini, and it's nicer to read.
+	step func(*scanner, byte) int
+
+	// Reached end of top-level value.
+	endTop bool
+
+	// Stack of what we're in the middle of - array values, object keys, object values.
+	parseState []int
+
+	// Error that happened, if any.
+	err error
+
+	// total bytes consumed, updated by decoder.Decode
+	bytes int64
+}
+
+// These values are returned by the state transition functions
+// assigned to scanner.state and the method scanner.eof.
+// They give details about the current state of the scan that
+// callers might be interested to know about.
+// It is okay to ignore the return value of any particular
+// call to scanner.state: if one call returns scanError,
+// every subsequent call will return scanError too.
+const (
+	// Continue.
+	scanContinue     = iota // uninteresting byte
+	scanBeginLiteral        // end implied by next result != scanContinue
+	scanBeginObject         // begin object
+	scanObjectKey           // just finished object key (string)
+	scanObjectValue         // just finished non-last object value
+	scanEndObject           // end object (implies scanObjectValue if possible)
+	scanBeginArray          // begin array
+	scanArrayValue          // just finished array value
+	scanEndArray            // end array (implies scanArrayValue if possible)
+	scanSkipSpace           // space byte; can skip; known to be last "continue" result
+
+	// Stop.
+	scanEnd   // top-level value ended *before* this byte; known to be first "stop" result
+	scanError // hit an error, scanner.err.
+)
+
+// These values are stored in the parseState stack.
+// They give the current state of a composite value
+// being scanned. If the parser is inside a nested value
+// the parseState describes the nested state, outermost at entry 0.
+const (
+	parseObjectKey   = iota // parsing object key (before colon)
+	parseObjectValue        // parsing object value (after colon)
+	parseArrayValue         // parsing array value
+)
+
+// reset prepares the scanner for use.
+// It must be called before calling s.step.
+func (s *scanner) reset() {
+	s.step = stateBeginValue
+	s.parseState = s.parseState[0:0]
+	s.err = nil
+	s.endTop = false
+}
+
+// eof tells the scanner that the end of input has been reached.
+// It returns a scan status just as s.step does.
+func (s *scanner) eof() int {
+	if s.err != nil {
+		return scanError
+	}
+	if s.endTop {
+		return scanEnd
+	}
+	s.step(s, ' ')
+	if s.endTop {
+		return scanEnd
+	}
+	if s.err == nil {
+		s.err = &SyntaxError{"unexpected end of JSON input", s.bytes}
+	}
+	return scanError
+}
+
+// pushParseState pushes a new parse state p onto the parse stack.
+func (s *scanner) pushParseState(p int) {
+	s.parseState = append(s.parseState, p)
+}
+
+// popParseState pops a parse state (already obtained) off the stack
+// and updates s.step accordingly.
+func (s *scanner) popParseState() {
+	n := len(s.parseState) - 1
+	s.parseState = s.parseState[0:n]
+	if n == 0 {
+		s.step = stateEndTop
+		s.endTop = true
+	} else {
+		s.step = stateEndValue
+	}
+}
+
+func isSpace(c byte) bool {
+	return c == ' ' || c == '\t' || c == '\r' || c == '\n'
+}
+
+// stateBeginValueOrEmpty is the state after reading `[`.
+func stateBeginValueOrEmpty(s *scanner, c byte) int {
+	if c <= ' ' && isSpace(c) {
+		return scanSkipSpace
+	}
+	if c == ']' {
+		return stateEndValue(s, c)
+	}
+	return stateBeginValue(s, c)
+}
+
+// stateBeginValue is the state at the beginning of the input.
+func stateBeginValue(s *scanner, c byte) int {
+	if c <= ' ' && isSpace(c) {
+		return scanSkipSpace
+	}
+	switch c {
+	case '{':
+		s.step = stateBeginStringOrEmpty
+		s.pushParseState(parseObjectKey)
+		return scanBeginObject
+	case '[':
+		s.step = stateBeginValueOrEmpty
+		s.pushParseState(parseArrayValue)
+		return scanBeginArray
+	case '"':
+		s.step = stateInString
+		return scanBeginLiteral
+	case '-':
+		s.step = stateNeg
+		return scanBeginLiteral
+	case '0': // beginning of 0.123
+		s.step = state0
+		return scanBeginLiteral
+	case 't': // beginning of true
+		s.step = stateT
+		return scanBeginLiteral
+	case 'f': // beginning of false
+		s.step = stateF
+		return scanBeginLiteral
+	case 'n': // beginning of null
+		s.step = stateN
+		return scanBeginLiteral
+	}
+	if '1' <= c && c <= '9' { // beginning of 1234.5
+		s.step = state1
+		return scanBeginLiteral
+	}
+	return s.error(c, "looking for beginning of value")
+}
+
+// stateBeginStringOrEmpty is the state after reading `{`.
+func stateBeginStringOrEmpty(s *scanner, c byte) int {
+	if c <= ' ' && isSpace(c) {
+		return scanSkipSpace
+	}
+	if c == '}' {
+		n := len(s.parseState)
+		s.parseState[n-1] = parseObjectValue
+		return stateEndValue(s, c)
+	}
+	return stateBeginString(s, c)
+}
+
+// stateBeginString is the state after reading `{"key": value,`.
+func stateBeginString(s *scanner, c byte) int {
+	if c <= ' ' && isSpace(c) {
+		return scanSkipSpace
+	}
+	if c == '"' {
+		s.step = stateInString
+		return scanBeginLiteral
+	}
+	return s.error(c, "looking for beginning of object key string")
+}
+
+// stateEndValue is the state after completing a value,
+// such as after reading `{}` or `true` or `["x"`.
+func stateEndValue(s *scanner, c byte) int {
+	n := len(s.parseState)
+	if n == 0 {
+		// Completed top-level before the current byte.
+		s.step = stateEndTop
+		s.endTop = true
+		return stateEndTop(s, c)
+	}
+	if c <= ' ' && isSpace(c) {
+		s.step = stateEndValue
+		return scanSkipSpace
+	}
+	ps := s.parseState[n-1]
+	switch ps {
+	case parseObjectKey:
+		if c == ':' {
+			s.parseState[n-1] = parseObjectValue
+			s.step = stateBeginValue
+			return scanObjectKey
+		}
+		return s.error(c, "after object key")
+	case parseObjectValue:
+		if c == ',' {
+			s.parseState[n-1] = parseObjectKey
+			s.step = stateBeginString
+			return scanObjectValue
+		}
+		if c == '}' {
+			s.popParseState()
+			return scanEndObject
+		}
+		return s.error(c, "after object key:value pair")
+	case parseArrayValue:
+		if c == ',' {
+			s.step = stateBeginValue
+			return scanArrayValue
+		}
+		if c == ']' {
+			s.popParseState()
+			return scanEndArray
+		}
+		return s.error(c, "after array element")
+	}
+	return s.error(c, "")
+}
+
+// stateEndTop is the state after finishing the top-level value,
+// such as after reading `{}` or `[1,2,3]`.
+// Only space characters should be seen now.
+func stateEndTop(s *scanner, c byte) int {
+	if !isSpace(c) {
+		// Complain about non-space byte on next call.
+		s.error(c, "after top-level value")
+	}
+	return scanEnd
+}
+
+// stateInString is the state after reading `"`.
+func stateInString(s *scanner, c byte) int {
+	if c == '"' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	if c == '\\' {
+		s.step = stateInStringEsc
+		return scanContinue
+	}
+	if c < 0x20 {
+		return s.error(c, "in string literal")
+	}
+	return scanContinue
+}
+
+// stateInStringEsc is the state after reading `"\` during a quoted string.
+func stateInStringEsc(s *scanner, c byte) int {
+	switch c {
+	case 'b', 'f', 'n', 'r', 't', '\\', '/', '"':
+		s.step = stateInString
+		return scanContinue
+	case 'u':
+		s.step = stateInStringEscU
+		return scanContinue
+	}
+	return s.error(c, "in string escape code")
+}
+
+// stateInStringEscU is the state after reading `"\u` during a quoted string.
+func stateInStringEscU(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInStringEscU1
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateInStringEscU1 is the state after reading `"\u1` during a quoted string.
+func stateInStringEscU1(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInStringEscU12
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateInStringEscU12 is the state after reading `"\u12` during a quoted string.
+func stateInStringEscU12(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInStringEscU123
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateInStringEscU123 is the state after reading `"\u123` during a quoted string.
+func stateInStringEscU123(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInString
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateNeg is the state after reading `-` during a number.
+func stateNeg(s *scanner, c byte) int {
+	if c == '0' {
+		s.step = state0
+		return scanContinue
+	}
+	if '1' <= c && c <= '9' {
+		s.step = state1
+		return scanContinue
+	}
+	return s.error(c, "in numeric literal")
+}
+
+// state1 is the state after reading a non-zero integer during a number,
+// such as after reading `1` or `100` but not `0`.
+func state1(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' {
+		s.step = state1
+		return scanContinue
+	}
+	return state0(s, c)
+}
+
+// state0 is the state after reading `0` during a number.
+func state0(s *scanner, c byte) int {
+	if c == '.' {
+		s.step = stateDot
+		return scanContinue
+	}
+	if c == 'e' || c == 'E' {
+		s.step = stateE
+		return scanContinue
+	}
+	return stateEndValue(s, c)
+}
+
+// stateDot is the state after reading the integer and decimal point in a number,
+// such as after reading `1.`.
+func stateDot(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' {
+		s.step = stateDot0
+		return scanContinue
+	}
+	return s.error(c, "after decimal point in numeric literal")
+}
+
+// stateDot0 is the state after reading the integer, decimal point, and subsequent
+// digits of a number, such as after reading `3.14`.
+func stateDot0(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' {
+		return scanContinue
+	}
+	if c == 'e' || c == 'E' {
+		s.step = stateE
+		return scanContinue
+	}
+	return stateEndValue(s, c)
+}
+
+// stateE is the state after reading the mantissa and e in a number,
+// such as after reading `314e` or `0.314e`.
+func stateE(s *scanner, c byte) int {
+	if c == '+' || c == '-' {
+		s.step = stateESign
+		return scanContinue
+	}
+	return stateESign(s, c)
+}
+
+// stateESign is the state after reading the mantissa, e, and sign in a number,
+// such as after reading `314e-` or `0.314e+`.
+func stateESign(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' {
+		s.step = stateE0
+		return scanContinue
+	}
+	return s.error(c, "in exponent of numeric literal")
+}
+
+// stateE0 is the state after reading the mantissa, e, optional sign,
+// and at least one digit of the exponent in a number,
+// such as after reading `314e-2` or `0.314e+1` or `3.14e0`.
+func stateE0(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' {
+		return scanContinue
+	}
+	return stateEndValue(s, c)
+}
+
+// stateT is the state after reading `t`.
+func stateT(s *scanner, c byte) int {
+	if c == 'r' {
+		s.step = stateTr
+		return scanContinue
+	}
+	return s.error(c, "in literal true (expecting 'r')")
+}
+
+// stateTr is the state after reading `tr`.
+func stateTr(s *scanner, c byte) int {
+	if c == 'u' {
+		s.step = stateTru
+		return scanContinue
+	}
+	return s.error(c, "in literal true (expecting 'u')")
+}
+
+// stateTru is the state after reading `tru`.
+func stateTru(s *scanner, c byte) int {
+	if c == 'e' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	return s.error(c, "in literal true (expecting 'e')")
+}
+
+// stateF is the state after reading `f`.
+func stateF(s *scanner, c byte) int {
+	if c == 'a' {
+		s.step = stateFa
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 'a')")
+}
+
+// stateFa is the state after reading `fa`.
+func stateFa(s *scanner, c byte) int {
+	if c == 'l' {
+		s.step = stateFal
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 'l')")
+}
+
+// stateFal is the state after reading `fal`.
+func stateFal(s *scanner, c byte) int {
+	if c == 's' {
+		s.step = stateFals
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 's')")
+}
+
+// stateFals is the state after reading `fals`.
+func stateFals(s *scanner, c byte) int {
+	if c == 'e' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 'e')")
+}
+
+// stateN is the state after reading `n`.
+func stateN(s *scanner, c byte) int {
+	if c == 'u' {
+		s.step = stateNu
+		return scanContinue
+	}
+	return s.error(c, "in literal null (expecting 'u')")
+}
+
+// stateNu is the state after reading `nu`.
+func stateNu(s *scanner, c byte) int {
+	if c == 'l' {
+		s.step = stateNul
+		return scanContinue
+	}
+	return s.error(c, "in literal null (expecting 'l')")
+}
+
+// stateNul is the state after reading `nul`.
+func stateNul(s *scanner, c byte) int {
+	if c == 'l' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	return s.error(c, "in literal null (expecting 'l')")
+}
+
+// stateError is the state after reaching a syntax error,
+// such as after reading `[1}` or `5.1.2`.
+func stateError(s *scanner, c byte) int {
+	return scanError
+}
+
+// error records an error and switches to the error state.
+func (s *scanner) error(c byte, context string) int {
+	s.step = stateError
+	s.err = &SyntaxError{"invalid character " + quoteChar(c) + " " + context, s.bytes}
+	return scanError
+}
+
+// quoteChar formats c as a quoted character literal
+func quoteChar(c byte) string {
+	// special cases - different from quoted strings
+	if c == '\'' {
+		return `'\''`
+	}
+	if c == '"' {
+		return `'"'`
+	}
+
+	// use quoted string with different quotation marks
+	s := strconv.Quote(string(c))
+	return "'" + s[1:len(s)-1] + "'"
+}

vendor/github.com/d5/tengo/stdlib/os.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"os/exec"
 
+	"github.com/d5/tengo"
 	"github.com/d5/tengo/objects"
 )
 
@@ -39,14 +40,14 @@ var osModule = map[string]objects.Object{
 	"seek_set":            &objects.Int{Value: int64(io.SeekStart)},
 	"seek_cur":            &objects.Int{Value: int64(io.SeekCurrent)},
 	"seek_end":            &objects.Int{Value: int64(io.SeekEnd)},
-	"args":                &objects.UserFunction{Value: osArgs},                                           // args() => array(string)
+	"args":                &objects.UserFunction{Name: "args", Value: osArgs},                             // args() => array(string)
 	"chdir":               &objects.UserFunction{Name: "chdir", Value: FuncASRE(os.Chdir)},                // chdir(dir string) => error
-	"chmod":               osFuncASFmRE(os.Chmod),                                                         // chmod(name string, mode int) => error
+	"chmod":               osFuncASFmRE("chmod", os.Chmod),                                                // chmod(name string, mode int) => error
 	"chown":               &objects.UserFunction{Name: "chown", Value: FuncASIIRE(os.Chown)},              // chown(name string, uid int, gid int) => error
 	"clearenv":            &objects.UserFunction{Name: "clearenv", Value: FuncAR(os.Clearenv)},            // clearenv()
 	"environ":             &objects.UserFunction{Name: "environ", Value: FuncARSs(os.Environ)},            // environ() => array(string)
 	"exit":                &objects.UserFunction{Name: "exit", Value: FuncAIR(os.Exit)},                   // exit(code int)
-	"expand_env":          &objects.UserFunction{Name: "expand_env", Value: FuncASRS(os.ExpandEnv)},       // expand_env(s string) => string
+	"expand_env":          &objects.UserFunction{Name: "expand_env", Value: osExpandEnv},                  // expand_env(s string) => string
 	"getegid":             &objects.UserFunction{Name: "getegid", Value: FuncARI(os.Getegid)},             // getegid() => int
 	"getenv":              &objects.UserFunction{Name: "getenv", Value: FuncASRS(os.Getenv)},              // getenv(s string) => string
 	"geteuid":             &objects.UserFunction{Name: "geteuid", Value: FuncARI(os.Geteuid)},             // geteuid() => int
@@ -60,9 +61,9 @@ var osModule = map[string]objects.Object{
 	"hostname":            &objects.UserFunction{Name: "hostname", Value: FuncARSE(os.Hostname)},          // hostname() => string/error
 	"lchown":              &objects.UserFunction{Name: "lchown", Value: FuncASIIRE(os.Lchown)},            // lchown(name string, uid int, gid int) => error
 	"link":                &objects.UserFunction{Name: "link", Value: FuncASSRE(os.Link)},                 // link(oldname string, newname string) => error
-	"lookup_env":          &objects.UserFunction{Value: osLookupEnv},                                      // lookup_env(key string) => string/false
-	"mkdir":               osFuncASFmRE(os.Mkdir),                                                         // mkdir(name string, perm int) => error
-	"mkdir_all":           osFuncASFmRE(os.MkdirAll),                                                      // mkdir_all(name string, perm int) => error
+	"lookup_env":          &objects.UserFunction{Name: "lookup_env", Value: osLookupEnv},                  // lookup_env(key string) => string/false
+	"mkdir":               osFuncASFmRE("mkdir", os.Mkdir),                                                // mkdir(name string, perm int) => error
+	"mkdir_all":           osFuncASFmRE("mkdir_all", os.MkdirAll),                                         // mkdir_all(name string, perm int) => error
 	"readlink":            &objects.UserFunction{Name: "readlink", Value: FuncASRSE(os.Readlink)},         // readlink(name string) => string/error
 	"remove":              &objects.UserFunction{Name: "remove", Value: FuncASRE(os.Remove)},              // remove(name string) => error
 	"remove_all":          &objects.UserFunction{Name: "remove_all", Value: FuncASRE(os.RemoveAll)},       // remove_all(name string) => error
@@ -72,15 +73,15 @@ var osModule = map[string]objects.Object{
 	"temp_dir":            &objects.UserFunction{Name: "temp_dir", Value: FuncARS(os.TempDir)},            // temp_dir() => string
 	"truncate":            &objects.UserFunction{Name: "truncate", Value: FuncASI64RE(os.Truncate)},       // truncate(name string, size int) => error
 	"unsetenv":            &objects.UserFunction{Name: "unsetenv", Value: FuncASRE(os.Unsetenv)},          // unsetenv(key string) => error
-	"create":              &objects.UserFunction{Value: osCreate},                                         // create(name string) => imap(file)/error
-	"open":                &objects.UserFunction{Value: osOpen},                                           // open(name string) => imap(file)/error
-	"open_file":           &objects.UserFunction{Value: osOpenFile},                                       // open_file(name string, flag int, perm int) => imap(file)/error
-	"find_process":        &objects.UserFunction{Value: osFindProcess},                                    // find_process(pid int) => imap(process)/error
-	"start_process":       &objects.UserFunction{Value: osStartProcess},                                   // start_process(name string, argv array(string), dir string, env array(string)) => imap(process)/error
+	"create":              &objects.UserFunction{Name: "create", Value: osCreate},                         // create(name string) => imap(file)/error
+	"open":                &objects.UserFunction{Name: "open", Value: osOpen},                             // open(name string) => imap(file)/error
+	"open_file":           &objects.UserFunction{Name: "open_file", Value: osOpenFile},                    // open_file(name string, flag int, perm int) => imap(file)/error
+	"find_process":        &objects.UserFunction{Name: "find_process", Value: osFindProcess},              // find_process(pid int) => imap(process)/error
+	"start_process":       &objects.UserFunction{Name: "start_process", Value: osStartProcess},            // start_process(name string, argv array(string), dir string, env array(string)) => imap(process)/error
 	"exec_look_path":      &objects.UserFunction{Name: "exec_look_path", Value: FuncASRSE(exec.LookPath)}, // exec_look_path(file) => string/error
-	"exec":                &objects.UserFunction{Value: osExec},                                           // exec(name, args...) => command
-	"stat":                &objects.UserFunction{Value: osStat},                                           // stat(name) => imap(fileinfo)/error
-	"read_file":           &objects.UserFunction{Value: osReadFile},                                       // readfile(name) => array(byte)/error
+	"exec":                &objects.UserFunction{Name: "exec", Value: osExec},                             // exec(name, args...) => command
+	"stat":                &objects.UserFunction{Name: "stat", Value: osStat},                             // stat(name) => imap(fileinfo)/error
+	"read_file":           &objects.UserFunction{Name: "read_file", Value: osReadFile},                    // readfile(name) => array(byte)/error
 }
 
 func osReadFile(args ...objects.Object) (ret objects.Object, err error) {
@@ -102,6 +103,10 @@ func osReadFile(args ...objects.Object) (ret objects.Object, err error) {
 		return wrapError(err), nil
 	}
 
+	if len(bytes) > tengo.MaxBytesLen {
+		return nil, objects.ErrBytesLimit
+	}
+
 	return &objects.Bytes{Value: bytes}, nil
 }
 
@@ -233,14 +238,19 @@ func osArgs(args ...objects.Object) (objects.Object, error) {
 
 	arr := &objects.Array{}
 	for _, osArg := range os.Args {
+		if len(osArg) > tengo.MaxStringLen {
+			return nil, objects.ErrStringLimit
+		}
+
 		arr.Value = append(arr.Value, &objects.String{Value: osArg})
 	}
 
 	return arr, nil
 }
 
-func osFuncASFmRE(fn func(string, os.FileMode) error) *objects.UserFunction {
+func osFuncASFmRE(name string, fn func(string, os.FileMode) error) *objects.UserFunction {
 	return &objects.UserFunction{
+		Name: name,
 		Value: func(args ...objects.Object) (objects.Object, error) {
 			if len(args) != 2 {
 				return nil, objects.ErrWrongNumArguments
@@ -287,9 +297,54 @@ func osLookupEnv(args ...objects.Object) (objects.Object, error) {
 		return objects.FalseValue, nil
 	}
 
+	if len(res) > tengo.MaxStringLen {
+		return nil, objects.ErrStringLimit
+	}
+
 	return &objects.String{Value: res}, nil
 }
 
+func osExpandEnv(args ...objects.Object) (objects.Object, error) {
+	if len(args) != 1 {
+		return nil, objects.ErrWrongNumArguments
+	}
+
+	s1, ok := objects.ToString(args[0])
+	if !ok {
+		return nil, objects.ErrInvalidArgumentType{
+			Name:     "first",
+			Expected: "string(compatible)",
+			Found:    args[0].TypeName(),
+		}
+	}
+
+	var vlen int
+	var failed bool
+	s := os.Expand(s1, func(k string) string {
+		if failed {
+			return ""
+		}
+
+		v := os.Getenv(k)
+
+		// this does not count the other texts that are not being replaced
+		// but the code checks the final length at the end
+		vlen += len(v)
+		if vlen > tengo.MaxStringLen {
+			failed = true
+			return ""
+		}
+
+		return v
+	})
+
+	if failed || len(s) > tengo.MaxStringLen {
+		return nil, objects.ErrStringLimit
+	}
+
+	return &objects.String{Value: s}, nil
+}
+
 func osExec(args ...objects.Object) (objects.Object, error) {
 	if len(args) == 0 {
 		return nil, objects.ErrWrongNumArguments