This project is not active, and moved to spiegel-im-spiegel/gnkf.
text - Encoding/Decoding Text Package by Golang
Declare text module
See go.mod file.
Usage of package
Import Package
import "github.com/spiegel-im-spiegel/text"
detect
encoding := detect.EncodingJa(bytes.NewBufferString("こんにちは,世界"))
fmt.Println(encoding)
// Output:
// UTF-8
decode
jisText := []byte{0x1b, 0x24, 0x42, 0x24, 0x33, 0x24, 0x73, 0x24, 0x4b, 0x24, 0x41, 0x24, 0x4f, 0x40, 0x24, 0x33, 0x26, 0x1b, 0x28, 0x42}
res, err := decode.ToUTF8ja(bytes.NewReader(jisText))
if err != nil {
fmt.Println(err)
return
}
buf := new(bytes.Buffer)
io.Copy(buf, res)
fmt.Println(buf)
// Output:
// こんにちは世界
encode
utf8Text := "こんにちは,世界\n"
res, err := encode.FromUTF8To(detect.ISO2022JP, bytes.NewBufferString(utf8Text))
if err != nil {
fmt.Println(err)
return
}
buf := new(bytes.Buffer)
io.Copy(buf, res)
fmt.Println(buf.Bytes())
// Output:
// [27 36 66 36 51 36 115 36 75 36 65 36 79 33 36 64 36 51 38 27 40 66 10]
convert
jisText := []byte{0x1b, 0x24, 0x42, 0x24, 0x33, 0x24, 0x73, 0x24, 0x4b, 0x24, 0x41, 0x24, 0x4f, 0x40, 0x24, 0x33, 0x26, 0x1b, 0x28, 0x42}
res, err := convert.FromTo(detect.ISO2022JP, detect.UTF8, bytes.NewReader(jisText))
if err != nil {
fmt.Println(err)
return
}
buf := new(bytes.Buffer)
io.Copy(buf, res)
fmt.Println(buf)
// Output:
// こんにちは世界
newline
res := newline.Reader(strings.NewReader("こんにちは\nこんにちは\rこんにちは\r\nこんにちは"), newline.LF)
buf := new(bytes.Buffer)
io.Copy(buf, res)
fmt.Println(buf)
// Output:
// こんにちは
// こんにちは
// こんにちは
// こんにちは
res := newline.String("こんにちは\nこんにちは\rこんにちは\r\nこんにちは", newline.LF)
fmt.Println(res)
// Output:
// こんにちは
// こんにちは
// こんにちは
// こんにちは
normalize
res := normalize.Reader(strings.NewReader("ペンギン"), normalize.NFKC)
buf := new(bytes.Buffer)
io.Copy(buf, res)
fmt.Println(buf)
// Output:
// ペンギン
res := normalize.String("ペンギン", normalize.NFKC)
fmt.Println(res)
// Output:
// ペンギン
width
res := width.Reader(strings.NewReader("1234567890アイウエオカキクケコABCDEFGHIJK"), width.Fold)
buf := new(bytes.Buffer)
io.Copy(buf, res)
fmt.Println(buf)
// Output:
// 1234567890アイウエオカキクケコABCDEFGHIJK
res := width.String("1234567890アイウエオカキクケコABCDEFGHIJK", width.Fold)
fmt.Printlnres)
// Output:
// 1234567890アイウエオカキクケコABCDEFGHIJK
Command Line Interface
Binaries
See latest release.
Usage
$ gonkf -h
Network Kanji Filter by Golang
Usage:
gonkf [flags]
gonkf [command]
Available Commands:
conv Convert character encoding of text
guess Guess character encoding of text
help Help about any command
norm Unicode normalization
nwline Convert newline of text
version Print the version number of gonkf
width Convert character width of text
Flags:
-h, --help help for gonkf
Use "gonkf [command] --help" for more information about a command.
guess sub-command
$ gonkf guess -h
Guess character encoding of text
Usage:
gonkf guess [flags] [text file]
Flags:
-h, --help help for guess
$ echo こんにちは。世界の国から | gonkf guess
UTF-8
conv sub-command
$ gonkf conv -h
Convert character encoding of text
Usage:
gonkf conv [flags] [text file]
Flags:
-d, --dst-encoding string encoding of dest [euc|jis|sjis|utf8] (default "utf8")
-h, --help help for conv
-o, --output string output file path
-s, --src-encoding string encoding of src [euc|jis|sjis|utf8]
$ gonkf conv -d utf8 testdata/SHIFT_JIS.txt
こんにちは。世界の国から。
norm sub-command
$ gonkf norm -h
Unicode normalization (UTF-8 text only)
Usage:
gonkf norm [flags] [text file]
Flags:
-f, --form string normalization form [nfc|nfd|nfkc|nfkd] (default "nfc")
-h, --help help for norm
-o, --output string output file path
$ echo ペンギン | gonkf norm -f NFKC
ペンギン
width sub-command
$ gonkf width -h
Convert character width of text (UTF-8 text only)
Usage:
gonkf width [flags] [text file]
Flags:
-f, --form string form of width [fold|narrow|widen] (default "fold")
-h, --help help for width
-o, --output string output file path
$ echo 1234567890アイウエオカキクケコABCDEFGHIJK | gonkf width -f fold
1234567890アイウエオカキクケコABCDEFGHIJK