Golang code-google-com-p-go-text-transform.NewReader类(方法)实例源码

//在命令行执行
func parseSgf(file string) {

	b, err := ioutil.ReadFile(file)

	rInNop := transform.NewReader(bytes.NewReader(b), encoding.Nop.NewDecoder())
	b, _ = ioutil.ReadAll(rInNop)
	NopB := string(b)

	b1 := tryFixSgf(NopB)
	c, err := sgf.ParseSgf(b1)
	if err != nil {
		log.Println("ERR")
	}
	node := c.GameTrees[0].Nodes[0]
	f := func(pName string) string {
		for _, prop := range node.Properties {
			if pName == prop.Ident {
				return strings.Join(prop.Values, ",")
			}
		}
		return ""
	}
	log.Println("dt ", f("DT"))
	log.Println("HA ", f("HA"))
	log.Println("BR ", f("BR"))
	log.Println("WR ", f("WR"))
	log.Println("PB ", f("PB"))
	log.Println("PW ", f("PW"))
	log.Println("RE ", f("RE"))
	log.Println("KM ", f("KM"))
	log.Println("====================")
}

func eucjp_to_utf8(str string) (string, error) {
	ret, err := ioutil.ReadAll(transform.NewReader(strings.NewReader(str), japanese.EUCJP.NewDecoder()))
	if err != nil {
		return "", err
	}
	return string(ret), err
}

// TestNonRepertoire tests that codes outside of an Encoding's repertoire are
// converted:
//   - to the Unicode replacement character '\ufffd' when decoding to UTF-8,
//   - to the ASCII substitute character '\x1a' when encoding from UTF-8.
func TestNonRepertoire(t *testing.T) {
	testCases := []struct {
		e          encoding.Encoding
		dSrc, eSrc string
	}{
		{charmap.Windows1252, "\x81", "갂"},
		{japanese.EUCJP, "\xfe\xfc", "갂"},
		{japanese.ISO2022JP, "\x1b$B\x7e\x7e", "갂"},
		{japanese.ShiftJIS, "\xef\xfc", "갂"},
		{korean.EUCKR, "\xfe\xfe", "א"},
		{simplifiedchinese.GBK, "\xfe\xfe", "갂"},
		{simplifiedchinese.HZGB2312, "~{z~", "갂"},
		{traditionalchinese.Big5, "\x81\x40", "갂"},
	}
	for _, tc := range testCases {
		for _, direction := range []string{"Decode", "Encode"} {
			enc, want, src := (transform.Transformer)(nil), "", ""
			if direction == "Decode" {
				enc, want, src = tc.e.NewDecoder(), "\ufffd", tc.dSrc
			} else {
				enc, want, src = tc.e.NewEncoder(), "\x1a", tc.eSrc
			}

			dst, err := ioutil.ReadAll(transform.NewReader(strings.NewReader(src), enc))
			if err != nil {
				t.Errorf("%s %v: %v", direction, tc.e, err)
				continue
			}
			if got := string(dst); got != want {
				t.Errorf("%s %v:\ngot  %q\nwant %q", direction, tc.e, got, want)
				continue
			}
		}
	}
}

func TransUTF8FromCode(src []byte, code int) ([]byte, error) {
	var rInUTF8 io.Reader
	switch code {
	case GBK:
		rInUTF8 = transform.NewReader(bytes.NewReader(src), simplifiedchinese.GBK.NewDecoder())
		break
	case GB2312:
		rInUTF8 = transform.NewReader(bytes.NewReader(src), simplifiedchinese.HZGB2312.NewDecoder())
		break
	default:
		return nil, NOT_SUPPORTED_CODE
	}

	out, _ := ioutil.ReadAll(rInUTF8)
	return out, nil
}

func Decode(s []byte) ([]byte, error) {
	f := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewDecoder())
	d, e := ioutil.ReadAll(f)
	if e != nil {
		return nil, e
	}
	return d, nil
}

func TestBasics(t *testing.T) {
	for _, tc := range basicTestCases {
		for _, direction := range []string{"Decode", "Encode"} {
			newTransformer, want, src := (func() transform.Transformer)(nil), "", ""
			wPrefix, sPrefix := "", ""
			if direction == "Decode" {
				newTransformer, want, src = tc.e.NewDecoder, tc.utf8, tc.encoded
				wPrefix, sPrefix = "", tc.encPrefix
			} else {
				newTransformer, want, src = tc.e.NewEncoder, tc.encoded, tc.utf8
				wPrefix, sPrefix = tc.encPrefix, ""
			}

			dst := make([]byte, len(wPrefix)+len(want))
			nDst, nSrc, err := newTransformer().Transform(dst, []byte(sPrefix+src), true)
			if err != nil {
				t.Errorf("%v: %s: %v", tc.e, direction, err)
				continue
			}
			if nDst != len(wPrefix)+len(want) {
				t.Errorf("%v: %s: nDst got %d, want %d",
					tc.e, direction, nDst, len(wPrefix)+len(want))
				continue
			}
			if nSrc != len(sPrefix)+len(src) {
				t.Errorf("%v: %s: nSrc got %d, want %d",
					tc.e, direction, nSrc, len(sPrefix)+len(src))
				continue
			}
			if got := string(dst); got != wPrefix+want {
				t.Errorf("%v: %s:\ngot  %q\nwant %q",
					tc.e, direction, got, wPrefix+want)
				continue
			}

			for _, n := range []int{0, 1, 2, 10, 123, 4567} {
				input := sPrefix + strings.Repeat(src, n)
				sr := strings.NewReader(input)
				g, err := ioutil.ReadAll(transform.NewReader(sr, newTransformer()))
				if err != nil {
					t.Errorf("%v: %s: ReadAll: n=%d: %v", tc.e, direction, n, err)
					continue
				}
				if len(g) == 0 && len(input) == 0 {
					// If the input is empty then the output can be empty,
					// regardless of whatever wPrefix is.
					continue
				}
				got1, want1 := string(g), wPrefix+strings.Repeat(want, n)
				if got1 != want1 {
					t.Errorf("%v: %s: ReadAll: n=%d\ngot  %q\nwant %q",
						tc.e, direction, n, trim(got1), trim(want1))
					continue
				}
			}
		}
	}
}

func BigDecodeUTF8(s []byte) ([]byte, error) {
	I := bytes.NewReader(s)
	O := transform.NewReader(I, traditionalchinese.Big5.NewDecoder())
	d, e := ioutil.ReadAll(O)
	if e != nil {
		return nil, e
	}
	return d, nil
}

func transfer(b []byte) string {
	rInUTF8 := transform.NewReader(bytes.NewReader(b), simplifiedchinese.GBK.NewDecoder())
	gbk, err := ioutil.ReadAll(rInUTF8)
	if err != nil {
		log.Println("[wanring]", err)
		return ""
	}
	return string(gbk)
}

func utf8_to_sjis(str string) string {
	iostr := strings.NewReader(str)
	rio := transform.NewReader(iostr, japanese.ShiftJIS.NewEncoder())
	ret, err := ioutil.ReadAll(rio)
	if err != nil {
		return ""
	}
	return string(ret)
}

func conUtf(s string) string {
	sr := strings.NewReader(s)
	tr := transform.NewReader(sr, charmap.Windows1251.NewDecoder())
	buf, err := ioutil.ReadAll(tr)
	if err != err {
		// обработка ошибки
	}

	return string(buf)
}

func benchmark(b *testing.B, direction string, enc encoding.Encoding) {
	_, src, newTransformer, err := load(direction, enc)
	if err != nil {
		b.Fatal(err)
	}
	b.SetBytes(int64(len(src)))
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		r := transform.NewReader(bytes.NewReader(src), newTransformer())
		io.Copy(ioutil.Discard, r)
	}
}

func utf8DocByEucjpUrl(eucjpUrl string) *goquery.Document {
	resp, err := http.Get(eucjpUrl)
	defer resp.Body.Close()
	if err != nil {
		log.Fatal(err)
	}

	doc, err := goquery.NewDocumentFromReader(transform.NewReader(resp.Body, japanese.EUCJP.NewDecoder()))
	if err != nil {
		log.Fatal("Fail to ", err.Error(), ", for: ", eucjpUrl)
	}
	return doc
}

// TestBig5CircumflexAndMacron tests the special cases listed in
// http://encoding.spec.whatwg.org/#big5
// Note that these special cases aren't preserved by round-tripping through
// decoding and encoding (since
// http://encoding.spec.whatwg.org/index-big5.txt does not have an entry for
// U+0304 or U+030C), so we can't test this in TestBasics.
func TestBig5CircumflexAndMacron(t *testing.T) {
	src := "\x88\x5f\x88\x60\x88\x61\x88\x62\x88\x63\x88\x64\x88\x65\x88\x66 " +
		"\x88\xa2\x88\xa3\x88\xa4\x88\xa5\x88\xa6"
	want := "ÓǑÒ\u00ca\u0304Ế\u00ca\u030cỀÊ " +
		"ü\u00ea\u0304ế\u00ea\u030cề"
	dst, err := ioutil.ReadAll(transform.NewReader(
		strings.NewReader(src), traditionalchinese.Big5.NewDecoder()))
	if err != nil {
		t.Fatal(err)
	}
	if got := string(dst); got != want {
		t.Fatalf("\ngot  %q\nwant %q", got, want)
	}
}

//在命令行执行
func main() {
	//	userFile := "/Users/lihexing/Downloads/002/i.sgf"
	//	userFile := "/Users/lihexing/Downloads/002/m.sgf"
	//	userFile := "/Users/lihexing/Documents/kifu/ys.sgf"
	//	userFile := "/Users/lihexing/Documents/kifu/pa.sgf"
	//	userFile := "/Users/lihexing/Documents/kifu/cl.sgf"
	//	userFile := "/Users/lihexing/Documents/kifu/wt.sgf"
	//	userFile := "/Users/lihexing/Documents/12月1日处理/棋谱处理/已上传/LG杯/11/第11届LG杯32强赛_2006-05-15_朴永训_古力_62839.sgf"
	//	userFile := "/Users/lihexing/Documents/12月1日处理/棋谱处理/未上传/分好类待上传/中国晚报杯/2003成都晚报队选拔赛_2003-12-13_沈逢春_龙霖_61565.sgf"
	//	userFile := "/Users/lihexing/Documents/棋谱待解决问题/2/日本十段战/第51期日本十段战挑战赛五番棋第五局_2013-04-26_井山裕太_结城聪_107576.sgf"
	//	userFile := "/Users/lihexing/Documents/棋谱待解决问题/2/日本十段战/第8期十段战_1969-12-18_工藤纪夫_武宫正树_101715.sgf"
	//	userFile := "/Users/lihexing/Documents/棋谱待解决问题/2/应氏杯/第二届应氏杯第一轮_1992-07-13_王铭琬_石井邦生_51138.sgf"
	//	userFile := "/Users/lihexing/Documents/kifu/第9届理光杯第一轮_2008-12-22_彭荃_廖行文_48857.sgf"
	//	userFile := "/Users/lihexing/Downloads/第9届理光杯第一轮_2008-12-22_彭荃_廖行文_48857.sgf"

	//	userFile := "/Users/lihexing/Desktop/第24届富士通杯决赛(周睿羊讲解)_2011-08-14_朴廷桓_邱峻_73705.sgf"
	//	userFile := "/Users/lihexing/Downloads/第24届富士通杯决赛(周睿羊讲解)-73705.sgf"

	//	userFile := "/Users/lihexing/Desktop/第3期名人战三次预选_1978-11-30_山部俊郎_武宫正树_61320.sgf"
	//	userFile := "/Users/lihexing/Downloads/第3期名人战三次预选-61320.sgf"

	//	userFile := "/Users/lihexing/Desktop/DT_PB_PW_BR_WR_RE_第2届北林弘通杯围棋争霸赛8强赛_2013-12-08_别中华_杨宇轩_117622.sgf"
	//	userFile := "/Users/lihexing/Desktop/第42期本因坊战循环赛（日本每日新闻主办）_1987-03-05_山城宏_林海峰_50781.sgf"//给晓光处理手数
	//	userFile := "/Users/lihexing/Downloads/2012首届北林弘通杯围棋争霸赛3、4名决定战-92553.sgf" //弘通官网下载
	userFile := "/Users/lihexing/Desktop/2012首届北林弘通杯围棋争霸赛3、4名决定战_2012-10-14_陈其洲_李应硕_92553.sgf" //爬虫下载

	file, err := os.Open(userFile)
	defer file.Close()
	if err != nil {
		fmt.Println(userFile, err)
		return
	}

	b, err := ioutil.ReadAll(file)

	rInNop := transform.NewReader(bytes.NewReader(b), encoding.Nop.NewDecoder())
	b, _ = ioutil.ReadAll(rInNop)
	NopB := string(b)
	//	log.Println("string(b) ", NopB)
	parseKifu(NopB)

	//	rInGBK := transform.NewReader(bytes.NewReader(b), simplifiedchinese.GBK.NewDecoder())
	//	b, _ = ioutil.ReadAll(rInGBK)
	//	GbkB := string(b)
	////	log.Println("rInGBK ", GbkB)
	//	parseKifu(GbkB)

}

func read(userFile string) string {

	file, err := os.Open(userFile)
	defer file.Close()
	if err != nil {
		fmt.Println(userFile, err)
		return "fileOpenERR"
	}

	b, err := ioutil.ReadAll(file)

	rInNop := transform.NewReader(bytes.NewReader(b), encoding.Nop.NewDecoder())
	b, _ = ioutil.ReadAll(rInNop)
	NoOK := string(b)
	return parseKifu(NoOK)
}

// NewReader returns an io.Reader that converts the content of r to UTF-8.
// It calls DetermineEncoding to find out what r's encoding is.
func NewReader(r io.Reader, contentType string) (io.Reader, error) {
	preview := make([]byte, 1024)
	n, err := io.ReadFull(r, preview)
	switch {
	case err == io.ErrUnexpectedEOF:
		preview = preview[:n]
		r = bytes.NewReader(preview)
	case err != nil:
		return nil, err
	default:
		r = io.MultiReader(bytes.NewReader(preview), r)
	}

	if e, _, _ := DetermineEncoding(preview, contentType); e != encoding.Nop {
		r = transform.NewReader(r, e.NewDecoder())
	}
	return r, nil
}

func ParseIEpg(r io.Reader) (*IEpg, error) {
	var err error
	iepg := &IEpg{}
	iepg.Body = make([]byte, 0)
	rio := transform.NewReader(r, japanese.ShiftJIS.NewDecoder())
	scanner := bufio.NewScanner(rio)
	kv := make(map[string]string)
	for scanner.Scan() {
		line := scanner.Text()
		if line == "" {
			for scanner.Scan() {
				line = scanner.Text()
				iepg.Body = append(iepg.Body, []byte(line)...)
			}
			break
		}
		tmp := strings.Split(line, ": ")
		if len(tmp) != 2 {
			fmt.Printf("%q", line)
			continue
		}
		kv[tmp[0]] = tmp[1]
	}
	if err := scanner.Err(); err != nil {
		return nil, fmt.Errorf("IEPG scan error: %v", err)
	}
	iepg.StationId = kv["station"]
	iepg.StationName = kv["station-name"]
	iepg.ProgramTitle = kv["program-title"]
	if kv["program-id"] != "" {
		if iepg.ProgramId, err = strconv.Atoi(kv["program-id"]); err != nil {
			return nil, fmt.Errorf("Could not parse `program-id` - %q in %v: %v ", kv["program-id"], kv, err)
		}
	}
	start_time := fmt.Sprintf(datetime_format, kv["year"], kv["month"], kv["date"], kv["start"])
	if iepg.StartAt, err = time.Parse(datetime_layout, start_time); err != nil {
		return nil, fmt.Errorf("Could not parse start time - %q in %v: %v", start_time, kv, err)
	}
	end_time := fmt.Sprintf(datetime_format, kv["year"], kv["month"], kv["date"], kv["end"])
	if iepg.EndAt, err = time.Parse(datetime_layout, end_time); err != nil {
		return nil, fmt.Errorf("Could not parse end time - %q in %v: %v", end_time, kv, err)
	}
	return iepg, nil
}

func TestFiles(t *testing.T) {
	for _, dir := range []string{"Decode", "Encode"} {
		for _, tf := range testdataFiles {
			dst, src, newTransformer, err := load(dir, tf.enc)
			if err != nil {
				t.Errorf("%s, %s: load: %v", dir, tf.enc, err)
				continue
			}
			buf := bytes.NewBuffer(nil)
			r := transform.NewReader(bytes.NewReader(src), newTransformer())
			if _, err := io.Copy(buf, r); err != nil {
				t.Errorf("%s, %s: copy: %v", dir, tf.enc, err)
				continue
			}
			if !bytes.Equal(buf.Bytes(), dst) {
				t.Errorf("%s, %s: transformed bytes did not match golden file", dir, tf.enc)
				continue
			}
		}
	}
}

func TestReplacement(t *testing.T) {
	for _, direction := range []string{"Decode", "Encode"} {
		enc, want := (transform.Transformer)(nil), ""
		if direction == "Decode" {
			enc = encoding.Replacement.NewDecoder()
			want = "\ufffd"
		} else {
			enc = encoding.Replacement.NewEncoder()
			want = "AB\x00CD\ufffdYZ"
		}
		sr := strings.NewReader("AB\x00CD\x80YZ")
		g, err := ioutil.ReadAll(transform.NewReader(sr, enc))
		if err != nil {
			t.Errorf("%s: ReadAll: %v", direction, err)
			continue
		}
		if got := string(g); got != want {
			t.Errorf("%s:\ngot  %q\nwant %q", direction, got, want)
			continue
		}
	}
}

func encodingReader(body []byte, contentType string) (io.Reader, error) {
	preview := make([]byte, 1024)
	var r io.Reader = bytes.NewReader(body)
	n, err := io.ReadFull(r, preview)
	switch {
	case err == io.ErrUnexpectedEOF:
		preview = preview[:n]
		r = bytes.NewReader(preview)
	case err != nil:
		return nil, err
	default:
		r = io.MultiReader(bytes.NewReader(preview), r)
	}

	e, _, certain := charset.DetermineEncoding(preview, contentType)
	if !certain && e == charmap.Windows1252 && utf8.Valid(body) {
		e = encoding.Nop
	}
	if e != encoding.Nop {
		r = transform.NewReader(r, e.NewDecoder())
	}
	return r, nil
}