aboutsummaryrefslogtreecommitdiff
path: root/vendor/github.com/gogs/chardet/utf8.go
diff options
context:
space:
mode:
authorUnknwon <u@gogs.io>2019-10-23 23:03:17 -0700
committerGitHub <noreply@github.com>2019-10-23 23:03:17 -0700
commit613139e7bef81d3573e7988a47eb6765f3de347a (patch)
tree49de7277898d3ff47a122c072568edb8ed4c9ac9 /vendor/github.com/gogs/chardet/utf8.go
parentfb100dbf98f02e4c631d142ff0f52ec29ee2f00c (diff)
Enable Go modules (#5835)
* Remove vendor * Enable Go modules * ci: add command to fetch dependencies * ci: update setting * ci: update settings * Require Go 1.11 * Rename module name to gogs.io/gogs
Diffstat (limited to 'vendor/github.com/gogs/chardet/utf8.go')
-rw-r--r--vendor/github.com/gogs/chardet/utf8.go71
1 files changed, 0 insertions, 71 deletions
diff --git a/vendor/github.com/gogs/chardet/utf8.go b/vendor/github.com/gogs/chardet/utf8.go
deleted file mode 100644
index ae036ad9..00000000
--- a/vendor/github.com/gogs/chardet/utf8.go
+++ /dev/null
@@ -1,71 +0,0 @@
-package chardet
-
-import (
- "bytes"
-)
-
-var utf8Bom = []byte{0xEF, 0xBB, 0xBF}
-
-type recognizerUtf8 struct {
-}
-
-func newRecognizer_utf8() *recognizerUtf8 {
- return &recognizerUtf8{}
-}
-
-func (*recognizerUtf8) Match(input *recognizerInput) (output recognizerOutput) {
- output = recognizerOutput{
- Charset: "UTF-8",
- }
- hasBom := bytes.HasPrefix(input.raw, utf8Bom)
- inputLen := len(input.raw)
- var numValid, numInvalid uint32
- var trailBytes uint8
- for i := 0; i < inputLen; i++ {
- c := input.raw[i]
- if c&0x80 == 0 {
- continue
- }
- if c&0xE0 == 0xC0 {
- trailBytes = 1
- } else if c&0xF0 == 0xE0 {
- trailBytes = 2
- } else if c&0xF8 == 0xF0 {
- trailBytes = 3
- } else {
- numInvalid++
- if numInvalid > 5 {
- break
- }
- trailBytes = 0
- }
-
- for i++; i < inputLen; i++ {
- c = input.raw[i]
- if c&0xC0 != 0x80 {
- numInvalid++
- break
- }
- if trailBytes--; trailBytes == 0 {
- numValid++
- break
- }
- }
- }
-
- if hasBom && numInvalid == 0 {
- output.Confidence = 100
- } else if hasBom && numValid > numInvalid*10 {
- output.Confidence = 80
- } else if numValid > 3 && numInvalid == 0 {
- output.Confidence = 100
- } else if numValid > 0 && numInvalid == 0 {
- output.Confidence = 80
- } else if numValid == 0 && numInvalid == 0 {
- // Plain ASCII
- output.Confidence = 10
- } else if numValid > numInvalid*10 {
- output.Confidence = 25
- }
- return
-}