From 3af757ed77b5c8763b81f913ec6981671fdd6f26 Mon Sep 17 00:00:00 2001
From: Peter Smit <peter@smitmail.eu>
Date: Mon, 22 Dec 2014 11:01:52 +0200
Subject: Replace mahonia with the standard functions in the Golang
 Sub-repositories

---
 modules/base/template.go | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

(limited to 'modules/base/template.go')

diff --git a/modules/base/template.go b/modules/base/template.go
index 462269aa..9107f3e1 100644
--- a/modules/base/template.go
+++ b/modules/base/template.go
@@ -7,14 +7,15 @@ package base
 import (
 	"container/list"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"html/template"
 	"runtime"
 	"strings"
 	"time"
 
-	"github.com/gogits/gogs/modules/mahonia"
+	"golang.org/x/net/html/charset"
+	"golang.org/x/text/transform"
+
 	"github.com/gogits/gogs/modules/setting"
 	"github.com/saintfish/chardet"
 )
@@ -54,20 +55,30 @@ func DetectEncoding(content []byte) (string, error) {
 }
 
 func ToUtf8WithErr(content []byte) (error, string) {
-	charset, err := DetectEncoding(content)
+	charsetLabel, err := DetectEncoding(content)
 	if err != nil {
 		return err, ""
 	}
 
-	if charset == "utf8" {
+	if charsetLabel == "utf8" {
 		return nil, string(content)
 	}
 
-	decoder := mahonia.NewDecoder(charset)
-	if decoder != nil {
-		return nil, decoder.ConvertString(string(content))
+	encoding, _ := charset.Lookup(charsetLabel)
+
+	if encoding == nil {
+		return fmt.Errorf("unknow char decoder %s", charsetLabel), string(content)
 	}
-	return errors.New("unknow char decoder"), string(content)
+
+	result, n, err := transform.String(encoding.NewDecoder(), string(content))
+
+	// If there is an error, we concatenate the nicely decoded part and the
+	// original left over. This way we won't loose data.
+	if err != nil {
+		result = result + string(content[n:])
+	}
+
+	return err, result
 }
 
 func ToUtf8(content string) string {
-- 
cgit v1.2.3


From e6f97c78ffdb20b95c36686eb8eb8a8ce1938a7b Mon Sep 17 00:00:00 2001
From: Peter Smit <peter@smitmail.eu>
Date: Thu, 15 Jan 2015 12:40:16 +0200
Subject: Change chardet to gogits

---
 .gopmfile                | 2 +-
 modules/base/template.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'modules/base/template.go')

diff --git a/.gopmfile b/.gopmfile
index 49b41715..dd262cbb 100644
--- a/.gopmfile
+++ b/.gopmfile
@@ -13,6 +13,7 @@ github.com/codegangsta/cli = commit:7381bc4e62
 github.com/go-sql-driver/mysql = commit:8111ee3ec3
 github.com/go-xorm/core = commit:3e0fa232ab
 github.com/go-xorm/xorm = commit:58d33844ce
+github.com/gogits/chardet = commit:2404f77725
 github.com/gogits/go-gogs-client = commit:92e76d616a
 github.com/gogits/oauth2 = commit:99cbec870a
 github.com/lib/pq = commit:b021d0ef20
@@ -27,7 +28,6 @@ github.com/macaron-contrib/toolbox = commit:57127bcc89
 github.com/mattn/go-sqlite3 = commit:a80c27ba33
 github.com/nfnt/resize = commit:581d15cb53
 github.com/russross/blackfriday = commit:05b8cefd6a
-github.com/saintfish/chardet = commit:3af4cd4741
 
 [res]
 include = conf|etc|public|scripts|templates
diff --git a/modules/base/template.go b/modules/base/template.go
index 9107f3e1..d96617c0 100644
--- a/modules/base/template.go
+++ b/modules/base/template.go
@@ -17,7 +17,7 @@ import (
 	"golang.org/x/text/transform"
 
 	"github.com/gogits/gogs/modules/setting"
-	"github.com/saintfish/chardet"
+	"github.com/gogits/chardet"
 )
 
 func Str2html(raw string) template.HTML {
-- 
cgit v1.2.3